From c0d210206fb37624888d6f7b8aba92a6e7366063 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 2 Apr 2020 18:12:43 +0530
Subject: [PATCH 001/361] feat: paytm integration

---
 .../doctype/paytm_settings/__init__.py        |  0
 .../doctype/paytm_settings/paytm_settings.js  |  8 ++++
 .../paytm_settings/paytm_settings.json        | 46 +++++++++++++++++++
 .../doctype/paytm_settings/paytm_settings.py  | 10 ++++
 .../paytm_settings/test_paytm_settings.py     | 10 ++++
 5 files changed, 74 insertions(+)
 create mode 100644 frappe/integrations/doctype/paytm_settings/__init__.py
 create mode 100644 frappe/integrations/doctype/paytm_settings/paytm_settings.js
 create mode 100644 frappe/integrations/doctype/paytm_settings/paytm_settings.json
 create mode 100644 frappe/integrations/doctype/paytm_settings/paytm_settings.py
 create mode 100644 frappe/integrations/doctype/paytm_settings/test_paytm_settings.py

diff --git a/frappe/integrations/doctype/paytm_settings/__init__.py b/frappe/integrations/doctype/paytm_settings/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.js b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
new file mode 100644
index 0000000000..d51a6e4a25
--- /dev/null
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Paytm Settings', {
+	// refresh: function(frm) {
+
+	// }
+});
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
new file mode 100644
index 0000000000..e57c8a192c
--- /dev/null
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
@@ -0,0 +1,46 @@
+{
+ "actions": [],
+ "creation": "2020-04-02 00:11:22.846697",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "merchant_id",
+  "merchant_key"
+ ],
+ "fields": [
+  {
+   "fieldname": "merchant_id",
+   "fieldtype": "Data",
+   "label": "Merchant ID"
+  },
+  {
+   "fieldname": "merchant_key",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Merchant Key"
+  }
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-04-02 00:11:22.846697",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Paytm Settings",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
new file mode 100644
index 0000000000..74d51d1dcc
--- /dev/null
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class PaytmSettings(Document):
+	pass
diff --git a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
new file mode 100644
index 0000000000..77a16c82ae
--- /dev/null
+++ b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestPaytmSettings(unittest.TestCase):
+	pass

From 8831bc6531bad3925526ea0d4e9f345cfbd89610 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Wed, 8 Apr 2020 18:49:48 +0530
Subject: [PATCH 002/361] feat: blink checkout

---
 .../doctype/paytm_settings/checksum.py        | 143 ++++++++++++++++++
 .../includes/integrations/paytm_checkout.js   |  42 +++++
 2 files changed, 185 insertions(+)
 create mode 100644 frappe/integrations/doctype/paytm_settings/checksum.py
 create mode 100644 frappe/templates/includes/integrations/paytm_checkout.js

diff --git a/frappe/integrations/doctype/paytm_settings/checksum.py b/frappe/integrations/doctype/paytm_settings/checksum.py
new file mode 100644
index 0000000000..95fb7dc164
--- /dev/null
+++ b/frappe/integrations/doctype/paytm_settings/checksum.py
@@ -0,0 +1,143 @@
+import base64
+import string
+import random
+import hashlib
+
+from Crypto.Cipher import AES
+
+
+IV = "@@@@&&&&####$$$$"
+BLOCK_SIZE = 16
+
+
+def generate_checksum(param_dict, merchant_key, salt=None):
+	params_string = __get_param_string__(param_dict)
+	salt = salt if salt else __id_generator__(4)
+	final_string = '%s|%s' % (params_string, salt)
+
+	hasher = hashlib.sha256(final_string.encode())
+	hash_string = hasher.hexdigest()
+
+	hash_string += salt
+
+	return __encode__(hash_string, IV, merchant_key)
+
+
+def generate_refund_checksum(param_dict, merchant_key, salt=None):
+	for i in param_dict:
+		if("|" in param_dict[i]):
+			param_dict = {}
+			exit()
+	params_string = __get_param_string__(param_dict)
+	salt = salt if salt else __id_generator__(4)
+	final_string = '%s|%s' % (params_string, salt)
+
+	hasher = hashlib.sha256(final_string.encode())
+	hash_string = hasher.hexdigest()
+
+	hash_string += salt
+
+	return __encode__(hash_string, IV, merchant_key)
+
+
+def generate_checksum_by_str(param_str, merchant_key, salt=None):
+	params_string = param_str
+	salt = salt if salt else __id_generator__(4)
+	final_string = '%s|%s' % (params_string, salt)
+
+	hasher = hashlib.sha256(final_string.encode())
+	hash_string = hasher.hexdigest()
+
+	hash_string += salt
+
+	return __encode__(hash_string, IV, merchant_key)
+
+
+def verify_checksum(param_dict, merchant_key, checksum):
+	# Remove checksum
+	if 'CHECKSUMHASH' in param_dict:
+		param_dict.pop('CHECKSUMHASH')
+
+	# Get salt
+	paytm_hash = __decode__(checksum, IV, merchant_key)
+	salt = paytm_hash[-4:]
+	calculated_checksum = generate_checksum(
+		param_dict, merchant_key, salt=salt)
+	return calculated_checksum == checksum
+
+
+def verify_checksum_by_str(param_str, merchant_key, checksum):
+	# Remove checksum
+	# if 'CHECKSUMHASH' in param_dict:
+		# param_dict.pop('CHECKSUMHASH')
+
+	# Get salt
+	paytm_hash = __decode__(checksum, IV, merchant_key)
+	salt = paytm_hash[-4:]
+	calculated_checksum = generate_checksum_by_str(
+		param_str, merchant_key, salt=salt)
+	return calculated_checksum == checksum
+
+
+def __id_generator__(size=6, chars=string.ascii_uppercase + string.digits + string.ascii_lowercase):
+	return ''.join(random.choice(chars) for _ in range(size))
+
+
+def __get_param_string__(params):
+	params_string = []
+	for key in sorted(params.keys()):
+		if("REFUND" in params[key] or "|" in params[key]):
+			respons_dict = {}
+			exit()
+		value = params[key]
+		params_string.append('' if value == 'null' else str(value))
+	return '|'.join(params_string)
+
+
+def __pad__(s): return s + (BLOCK_SIZE - len(s) %
+							BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE)
+
+
+def __unpad__(s): return s[0:-ord(s[-1])]
+
+
+def __encode__(to_encode, iv, key):
+	# Pad
+	to_encode = __pad__(to_encode)
+	# Encrypt
+	c = AES.new(key, AES.MODE_CBC, iv)
+	to_encode = c.encrypt(to_encode)
+	# Encode
+	to_encode = base64.b64encode(to_encode)
+	return to_encode.decode("UTF-8")
+
+
+def __decode__(to_decode, iv, key):
+	# Decode
+	to_decode = base64.b64decode(to_decode)
+	# Decrypt
+	c = AES.new(key, AES.MODE_CBC, iv)
+	to_decode = c.decrypt(to_decode)
+	if type(to_decode) == bytes:
+		# convert bytes array to str.
+		to_decode = to_decode.decode()
+	# remove pad
+	return __unpad__(to_decode)
+
+
+if __name__ == "__main__":
+	params = {
+		"MID": "mid",
+		"ORDER_ID": "order_id",
+		"CUST_ID": "cust_id",
+		"TXN_AMOUNT": "1",
+		"CHANNEL_ID": "WEB",
+		"INDUSTRY_TYPE_ID": "Retail",
+		"WEBSITE": "xxxxxxxxxxx"
+	}
+
+	print(verify_checksum(
+		params, 'xxxxxxxxxxxxxxxx',
+		"CD5ndX8VVjlzjWbbYoAtKQIlvtXPypQYOg0Fi2AUYKXZA5XSHiRF0FDj7vQu66S8MHx9NaDZ/uYm3WBOWHf+sDQAmTyxqUipA7i1nILlxrk="))
+
+	# print(generate_checksum(params, "xxxxxxxxxxxxxxxx"))
diff --git a/frappe/templates/includes/integrations/paytm_checkout.js b/frappe/templates/includes/integrations/paytm_checkout.js
new file mode 100644
index 0000000000..4cf070b60b
--- /dev/null
+++ b/frappe/templates/includes/integrations/paytm_checkout.js
@@ -0,0 +1,42 @@
+function onScriptLoad() {
+	console.log('inside on load')
+	var config = {
+		root: '',
+		flow: 'DEFAULT',
+		data: {
+			orderId: '{{ order_id}}',
+			token: '{{ token }}',
+			tokenType: 'TXN_TOKEN',
+			amount: '{{ amount }}'
+		},
+		handler: {
+			notifyMerchant: function(eventName, data) {
+				// notify about the state of the payment page ( invalid token , session expire , cancel transaction)
+				console.log('notifyMerchant handler function called');
+				console.log('eventName => ', eventName);
+				console.log('data => ', data);
+			},
+			transactionStatus: function transactionStatus(paymentStatus) {
+				// provide information to merchant about the payment status.
+				console.log('transaction status handler function called');
+				console.log('paymentStatus => ', paymentStatus);
+			}
+		}
+	};
+
+	$('.paytm-loading').addClass('hidden');
+	if (window.Paytm && window.Paytm.CheckoutJS) {
+		window.Paytm.CheckoutJS.onLoad(function excecuteAfterCompleteLoad() {
+			// initialze configuration using init method
+			window.Paytm.CheckoutJS.init(config)
+				.then(function onSuccess() {
+					// after successfully updating configuration, invoke Blink Checkout
+					window.Paytm.CheckoutJS.invoke();
+				})
+				.catch(function onError(error) {
+					console.log('inside the error window')
+					console.log('error => ', error);
+				});
+		});
+	}
+}

From 72e88b344033d8512651770c1f170324fa66d08d Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 10 Apr 2020 12:50:54 +0530
Subject: [PATCH 003/361] feat: create transaction token

---
 .../doctype/paytm_settings/paytm_settings.py  | 150 +++++++++++++++++-
 .../pages/integrations/paytm_checkout.html    |  26 +++
 .../pages/integrations/paytm_checkout.py      | 105 ++++++++++++
 3 files changed, 279 insertions(+), 2 deletions(-)
 create mode 100644 frappe/templates/pages/integrations/paytm_checkout.html
 create mode 100644 frappe/templates/pages/integrations/paytm_checkout.py

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 74d51d1dcc..1f9e508460 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -3,8 +3,154 @@
 # For license information, please see license.txt
 
 from __future__ import unicode_literals
-# import frappe
+import frappe
 from frappe.model.document import Document
+from frappe import _
+from six.moves.urllib.parse import urlencode
+from frappe.utils import get_url, call_hook_method, cint, flt
+from frappe.integrations.utils import make_get_request, make_post_request, create_request_log, create_payment_gateway
+from frappe.utils import get_request_site_address
+
+import json
+import requests
 
 class PaytmSettings(Document):
-	pass
+	supported_currencies = ["INR"]
+
+	def validate(self):
+		create_payment_gateway('Paytm')
+		call_hook_method('payment_gateway_enabled', gateway='Paytm')
+		if not self.flags.ignore_mandatory:
+			self.validate_paytm_credentails()
+
+	def validate_paytm_credentails(self):
+		if self.merchant_id and self.merchant_key:
+			pass
+			# header = {"Authorization": "Bearer {0}".format(self.get_password(fieldname="secret_key", raise_exception=False))}
+			# try:
+			# 	make_get_request(url="https://api.stripe.com/v1/charges", headers=header)
+			# except Exception:
+			# 	frappe.throw(_("Seems Publishable Key or Secret Key is wrong !!!"))
+
+	def validate_transaction_currency(self, currency):
+		if currency not in self.supported_currencies:
+			frappe.throw(_("Please select another payment method. Stripe does not support transactions in currency '{0}'").format(currency))
+
+	def get_payment_url(self, **kwargs):
+		'''Return payment url with several params'''
+		# create unique order id by making it equal to the integration request
+		integration_request = create_request_log(kwargs, "Host", "Paytm")
+		kwargs.update(dict(order_id=integration_request.name))
+
+		return get_url("./integrations/paytm_checkout?{0}".format(urlencode(kwargs)))
+
+	def create_request(self, data):
+		self.data = frappe._dict(data)
+
+		try:
+			self.integration_request = create_request_log(self.data, "Host", "Paytm")
+			return self.generate_transaction_token()
+
+		except Exception:
+			frappe.log_error(frappe.get_traceback())
+			return{
+				"redirect_to": frappe.redirect_to_message(_('Server Error'), _("It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account.")),
+				"status": 401
+			}
+
+	def generate_transaction_token(self):
+		import stripe
+		try:
+			charge = stripe.Charge.create(amount=cint(flt(self.data.amount)*100), currency=self.data.currency, source=self.data.stripe_token_id, description=self.data.description, receipt_email=self.data.payer_email)
+
+			if charge.captured == True:
+				self.integration_request.db_set('status', 'Completed', update_modified=False)
+				self.flags.status_changed_to = "Completed"
+
+			else:
+				frappe.log_error(charge.failure_message, 'Stripe Payment not completed')
+
+		except Exception:
+			frappe.log_error(frappe.get_traceback())
+
+		return self.finalize_request()
+
+
+	def finalize_request(self):
+		redirect_to = self.data.get('redirect_to') or None
+		redirect_message = self.data.get('redirect_message') or None
+		status = self.integration_request.status
+
+		if self.flags.status_changed_to == "Completed":
+			if self.data.reference_doctype and self.data.reference_docname:
+				custom_redirect_to = None
+				try:
+					custom_redirect_to = frappe.get_doc(self.data.reference_doctype,
+						self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to)
+				except Exception:
+					frappe.log_error(frappe.get_traceback())
+
+				if custom_redirect_to:
+					redirect_to = custom_redirect_to
+
+				redirect_url = 'payment-success'
+
+			if self.redirect_url:
+				redirect_url = self.redirect_url
+				redirect_to = None
+		else:
+			redirect_url = 'payment-failed'
+
+		if redirect_to:
+			redirect_url += '?' + urlencode({'redirect_to': redirect_to})
+		if redirect_message:
+			redirect_url += '&' + urlencode({'redirect_message': redirect_message})
+
+		return {
+			"redirect_to": redirect_url,
+			"status": status
+		}
+
+def generate_transaction_token(payment_details, order_id, merchant_id, merchant_key):
+
+	# initialize a dictionary
+	paytmParams = dict()
+
+	redirect_uri = get_request_site_address(True) + "?cmd=frappe.templates.pages.integrations.paytm_checkout.get_transaction_status"
+
+	# body parameters
+	paytmParams["body"] = get_paytm_params(payment_details, order_id, merchant_id, merchant_key)
+
+	checksum = generate_checksum_by_str(json.dumps(paytmParams["body"]), merchant_key)
+
+	paytmParams["head"] = {
+		"signature"  : checksum
+	}
+
+	post_data = json.dumps(paytmParams)
+
+	url = "https://securegw-stage.paytm.in/theia/api/v1/initiateTransaction?mid={0}&orderId={1}".format(merchant_id, order_id)
+
+	response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json()
+	return response['body'].get('txnToken')
+
+def get_paytm_params(payment_details, order_id, merchant_id, merchant_key):
+	return {
+		"requestType" : "Payment",
+		"mid" : merchant_id,
+		"websiteName" : "WEBSTAGING",
+		"orderId" : order_id,
+		"callbackUrl" : redirect_uri,
+		"txnAmount" : {
+			"value" : flt(payment_details['amount'], 2),
+			"currency" : "INR",
+		},
+		"userInfo" : {
+			"custId" : payment_details['payer_email'],
+		},
+	}
+
+def get_gateway_controller(doctype, docname):
+	reference_doc = frappe.get_doc(doctype, docname)
+	gateway_controller = frappe.db.get_value("Payment Gateway", reference_doc.payment_gateway, "gateway_controller")
+	return gateway_controller
\ No newline at end of file
diff --git a/frappe/templates/pages/integrations/paytm_checkout.html b/frappe/templates/pages/integrations/paytm_checkout.html
new file mode 100644
index 0000000000..2aedf31d65
--- /dev/null
+++ b/frappe/templates/pages/integrations/paytm_checkout.html
@@ -0,0 +1,26 @@
+{% extends "templates/web.html" %} {% block title %} Payment {% endblock %} {%-
+block header -%}{% endblock %} {% block script %}
+<script defer
+	type="application/javascript"
+	crossorigin="anonymous"
+	src="{{ host }}/merchantpgpui/checkoutjs/merchants/{{ mid }}.js"
+	onload="onScriptLoad();"
+></script>
+<script>
+	{% include "/templates/includes/integrations/paytm_checkout.js" %}
+</script>
+{% endblock %} {%- block page_content -%}
+<div id="paytm-checkoutjs"></div>
+<p class="lead text-center centered">
+	<span class="paytm-loading">Loading Payment System</span>
+	<span class="paytm-confirming hidden">Confirming Payment</span>
+</p>
+
+{% endblock %} {% block style %}
+<style>
+	/* header,
+	footer {
+		display: none;
+	} */
+</style>
+{% endblock %}
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
new file mode 100644
index 0000000000..a17873bc80
--- /dev/null
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -0,0 +1,105 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from frappe.utils import flt, cint
+import json
+from six import string_types
+from frappe.integrations.doctype.paytm_settings.checksum import generate_checksum_by_str
+from frappe.utils import get_request_site_address
+import requests
+
+no_cache = 1
+
+expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'reference_docname',
+	'payer_name', 'payer_email')
+
+def get_context(context):
+	context.no_cache = 1
+	merchant_id, merchant_key = get_paytm_credentials()
+
+	try:
+		doc = frappe.get_doc("Integration Request", frappe.form_dict['order_id'])
+		payment_details = json.loads(doc.data)
+		context.token = generate_transaction_token(payment_details, doc.name, merchant_id, merchant_key)
+		context.order_id = doc.name
+
+
+		for key in expected_keys:
+			context[key] = payment_details[key]
+
+		context['amount'] = flt(context['amount'], 2)
+		context.host = 'https://securegw-stage.paytm.in'
+		context.mid = merchant_id
+
+	except Exception as e:
+		frappe.log_error()
+		frappe.redirect_to_message(_('Invalid Token'),
+			_('Seems token you are using is invalid!'),
+			http_status_code=400, indicator_color='red')
+
+		frappe.local.flags.redirect_location = frappe.local.response.location
+		raise frappe.Redirect
+
+def get_paytm_credentials():
+	return frappe.db.get_value("Paytm Settings", None, ['merchant_id', 'merchant_key'])
+
+@frappe.whitelist(allow_guest=True)
+def make_payment(paytm_payment_id, options, reference_doctype, reference_docname, token):
+	data = {}
+
+	if isinstance(options, string_types):
+		data = json.loads(options)
+
+	data.update({
+		"paytm_payment_id": paytm_payment_id,
+		"reference_docname": reference_docname,
+		"reference_doctype": reference_doctype,
+		"token": token
+	})
+
+	data =  frappe.get_doc("Paytm Settings").create_request(data)
+	frappe.db.commit()
+	return data
+
+def generate_transaction_token(payment_details, order_id, merchant_id, merchant_key):
+
+	# initialize a dictionary
+	paytmParams = dict()
+
+	redirect_uri = get_request_site_address(True) + "?cmd=frappe.templates.pages.integrations.paytm_checkout.get_transaction_status"
+
+	# body parameters
+	paytmParams["body"] = {
+		"requestType" : "Payment",
+		"mid" : merchant_id,
+		"websiteName" : "WEBSTAGING",
+		"orderId" : order_id,
+		"callbackUrl" : redirect_uri,
+		"txnAmount" : {
+			"value" : flt(payment_details['amount'], 2),
+			"currency" : "INR",
+		},
+		"userInfo" : {
+			"custId" : payment_details['payer_email'],
+		},
+	}
+
+	checksum = generate_checksum_by_str(json.dumps(paytmParams["body"]), merchant_key)
+
+	paytmParams["head"] = {
+		"signature"  : checksum
+	}
+
+	print(paytmParams)
+	post_data = json.dumps(paytmParams)
+
+	url = "https://securegw-stage.paytm.in/theia/api/v1/initiateTransaction?mid={0}&orderId={1}".format(merchant_id, order_id)
+
+	response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json()
+	return response['body'].get('txnToken')
+
+@frappe.whitelist(allow_guest=True)
+def get_transaction_status():
+	print(vars(frappe.form_dict))
\ No newline at end of file

From 3f6459edbb384d1529fe350bd93bd413c4b4f136 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 16 Apr 2020 15:48:37 +0530
Subject: [PATCH 004/361] fix: paytm integration

---
 .pylintrc                                                      | 2 --
 frappe/integrations/doctype/paytm_settings/paytm_settings.json | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)
 delete mode 100644 .pylintrc

diff --git a/.pylintrc b/.pylintrc
deleted file mode 100644
index c11c0ab6a3..0000000000
--- a/.pylintrc
+++ /dev/null
@@ -1,2 +0,0 @@
-disable=access-member-before-definition
-disable=no-member
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
index e57c8a192c..eb68f0ab79 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.json
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
@@ -17,13 +17,12 @@
   {
    "fieldname": "merchant_key",
    "fieldtype": "Data",
-   "hidden": 1,
    "label": "Merchant Key"
   }
  ],
  "issingle": 1,
  "links": [],
- "modified": "2020-04-02 00:11:22.846697",
+ "modified": "2020-04-07 19:57:24.126640",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Paytm Settings",

From b0a34c602d9954f9b1d77fae918ce167a2ba799e Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 16 Apr 2020 15:51:11 +0530
Subject: [PATCH 005/361] fix: initiate payment via standard checkout

---
 .../pages/integrations/paytm_checkout.html    | 47 ++++++-----
 .../pages/integrations/paytm_checkout.py      | 78 ++-----------------
 2 files changed, 33 insertions(+), 92 deletions(-)

diff --git a/frappe/templates/pages/integrations/paytm_checkout.html b/frappe/templates/pages/integrations/paytm_checkout.html
index 2aedf31d65..772e16ecb8 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.html
+++ b/frappe/templates/pages/integrations/paytm_checkout.html
@@ -1,26 +1,35 @@
-{% extends "templates/web.html" %} {% block title %} Payment {% endblock %} {%-
-block header -%}{% endblock %} {% block script %}
-<script defer
-	type="application/javascript"
-	crossorigin="anonymous"
-	src="{{ host }}/merchantpgpui/checkoutjs/merchants/{{ mid }}.js"
-	onload="onScriptLoad();"
-></script>
-<script>
-	{% include "/templates/includes/integrations/paytm_checkout.js" %}
-</script>
-{% endblock %} {%- block page_content -%}
-<div id="paytm-checkoutjs"></div>
-<p class="lead text-center centered">
-	<span class="paytm-loading">Loading Payment System</span>
-	<span class="paytm-confirming hidden">Confirming Payment</span>
-</p>
+{% extends "templates/web.html" %}
 
-{% endblock %} {% block style %}
+{% block title %} Payment {% endblock %}
+
+{%- block header -%}
+<head>
+	<title>Merchant Checkout Page</title>
+</head>
+{% endblock %}
+
+{% block script %}
+<script defer type="text/javascript">
+	document.paytm_form.submit();
+</script>
+{% endblock %}
+
+{%- block page_content -%}
+<body>
+		<center><h1>Please do not refresh this page...</h1></center>
+		<form method="post" action="{{ url }}" name="paytm_form">
+			{% for name, value in payment_details.items() %}
+				<input type="hidden" name="{{ name }}" value="{{ value }}">
+			{% endfor %}
+		</form>
+</body>
+{% endblock %}
+
+{% block style %}
 <style>
 	/* header,
 	footer {
 		display: none;
 	} */
 </style>
-{% endblock %}
+{% endblock %}
\ No newline at end of file
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
index a17873bc80..1c61d5cab4 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.py
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -6,8 +6,8 @@ from frappe import _
 from frappe.utils import flt, cint
 import json
 from six import string_types
-from frappe.integrations.doctype.paytm_settings.checksum import generate_checksum_by_str
 from frappe.utils import get_request_site_address
+from frappe.integrations.doctype.paytm_settings.paytm_settings import get_paytm_params, get_paytm_config
 import requests
 
 no_cache = 1
@@ -17,21 +17,15 @@ expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'referen
 
 def get_context(context):
 	context.no_cache = 1
-	merchant_id, merchant_key = get_paytm_credentials()
+	paytm_config = get_paytm_config()
 
 	try:
 		doc = frappe.get_doc("Integration Request", frappe.form_dict['order_id'])
 		payment_details = json.loads(doc.data)
-		context.token = generate_transaction_token(payment_details, doc.name, merchant_id, merchant_key)
-		context.order_id = doc.name
 
+		context.payment_details = get_paytm_params(json.loads(doc.data), doc.name, paytm_config)
 
-		for key in expected_keys:
-			context[key] = payment_details[key]
-
-		context['amount'] = flt(context['amount'], 2)
-		context.host = 'https://securegw-stage.paytm.in'
-		context.mid = merchant_id
+		context.url = paytm_config.url
 
 	except Exception as e:
 		frappe.log_error()
@@ -40,66 +34,4 @@ def get_context(context):
 			http_status_code=400, indicator_color='red')
 
 		frappe.local.flags.redirect_location = frappe.local.response.location
-		raise frappe.Redirect
-
-def get_paytm_credentials():
-	return frappe.db.get_value("Paytm Settings", None, ['merchant_id', 'merchant_key'])
-
-@frappe.whitelist(allow_guest=True)
-def make_payment(paytm_payment_id, options, reference_doctype, reference_docname, token):
-	data = {}
-
-	if isinstance(options, string_types):
-		data = json.loads(options)
-
-	data.update({
-		"paytm_payment_id": paytm_payment_id,
-		"reference_docname": reference_docname,
-		"reference_doctype": reference_doctype,
-		"token": token
-	})
-
-	data =  frappe.get_doc("Paytm Settings").create_request(data)
-	frappe.db.commit()
-	return data
-
-def generate_transaction_token(payment_details, order_id, merchant_id, merchant_key):
-
-	# initialize a dictionary
-	paytmParams = dict()
-
-	redirect_uri = get_request_site_address(True) + "?cmd=frappe.templates.pages.integrations.paytm_checkout.get_transaction_status"
-
-	# body parameters
-	paytmParams["body"] = {
-		"requestType" : "Payment",
-		"mid" : merchant_id,
-		"websiteName" : "WEBSTAGING",
-		"orderId" : order_id,
-		"callbackUrl" : redirect_uri,
-		"txnAmount" : {
-			"value" : flt(payment_details['amount'], 2),
-			"currency" : "INR",
-		},
-		"userInfo" : {
-			"custId" : payment_details['payer_email'],
-		},
-	}
-
-	checksum = generate_checksum_by_str(json.dumps(paytmParams["body"]), merchant_key)
-
-	paytmParams["head"] = {
-		"signature"  : checksum
-	}
-
-	print(paytmParams)
-	post_data = json.dumps(paytmParams)
-
-	url = "https://securegw-stage.paytm.in/theia/api/v1/initiateTransaction?mid={0}&orderId={1}".format(merchant_id, order_id)
-
-	response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json()
-	return response['body'].get('txnToken')
-
-@frappe.whitelist(allow_guest=True)
-def get_transaction_status():
-	print(vars(frappe.form_dict))
\ No newline at end of file
+		raise frappe.Redirect
\ No newline at end of file

From 9d71bdcd874451181c21b384be81f8e88f390500 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 16 Apr 2020 18:53:56 +0530
Subject: [PATCH 006/361] feat: add paytm config, transaction status

---
 .../paytm_settings/paytm_settings.json        |  40 +++-
 .../doctype/paytm_settings/paytm_settings.py  | 206 ++++++++++--------
 2 files changed, 148 insertions(+), 98 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
index eb68f0ab79..0540dc4948 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.json
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
@@ -6,23 +6,55 @@
  "engine": "InnoDB",
  "field_order": [
   "merchant_id",
-  "merchant_key"
+  "merchant_key",
+  "staging",
+  "column_break_4",
+  "industry_type",
+  "website"
  ],
  "fields": [
   {
    "fieldname": "merchant_id",
    "fieldtype": "Data",
-   "label": "Merchant ID"
+   "in_list_view": 1,
+   "label": "Merchant ID",
+   "reqd": 1
   },
   {
    "fieldname": "merchant_key",
+   "fieldtype": "Password",
+   "in_list_view": 1,
+   "label": "Merchant Key",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "staging",
+   "fieldtype": "Check",
+   "label": "Staging"
+  },
+  {
+   "depends_on": "eval: !doc.staging",
+   "fieldname": "industry_type",
    "fieldtype": "Data",
-   "label": "Merchant Key"
+   "label": "Industry Type",
+   "mandatory_depends_on": "eval: !doc.staging"
+  },
+  {
+   "depends_on": "eval: !doc.staging",
+   "fieldname": "website",
+   "fieldtype": "Data",
+   "label": "Website",
+   "mandatory_depends_on": "eval: !doc.staging"
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
   }
  ],
  "issingle": 1,
  "links": [],
- "modified": "2020-04-07 19:57:24.126640",
+ "modified": "2020-04-16 14:16:19.687449",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Paytm Settings",
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 1f9e508460..10282d34ea 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -3,16 +3,18 @@
 # For license information, please see license.txt
 
 from __future__ import unicode_literals
+import json
+import requests
+from six.moves.urllib.parse import urlencode
+
 import frappe
 from frappe.model.document import Document
 from frappe import _
-from six.moves.urllib.parse import urlencode
-from frappe.utils import get_url, call_hook_method, cint, flt
+from frappe.utils import get_url, call_hook_method, cint, flt, cstr
 from frappe.integrations.utils import make_get_request, make_post_request, create_request_log, create_payment_gateway
 from frappe.utils import get_request_site_address
-
-import json
-import requests
+from frappe.integrations.doctype.paytm_settings.checksum import generate_checksum, verify_checksum
+from frappe.utils.password import get_decrypted_password
 
 class PaytmSettings(Document):
 	supported_currencies = ["INR"]
@@ -44,110 +46,126 @@ class PaytmSettings(Document):
 
 		return get_url("./integrations/paytm_checkout?{0}".format(urlencode(kwargs)))
 
-	def create_request(self, data):
-		self.data = frappe._dict(data)
+def get_paytm_config():
+	''' Returns paytm config '''
 
-		try:
-			self.integration_request = create_request_log(self.data, "Host", "Paytm")
-			return self.generate_transaction_token()
+	paytm_config = frappe.db.get_singles_dict('Paytm Settings')
+	paytm_config.update(dict(merchant_key=get_decrypted_password('Paytm Settings', 'Paytm Settings', 'merchant_key')))
 
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return{
-				"redirect_to": frappe.redirect_to_message(_('Server Error'), _("It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account.")),
-				"status": 401
-			}
+	if cint(paytm_config.staging):
+		paytm_config.update(dict(
+			website="WEBSTAGING",
+			url='https://securegw-stage.paytm.in/order/process',
+			transaction_status_url='https://securegw-stage.paytm.in/order/status',
+			industry_type_id='RETAIL'
+		))
+	else:
+		paytm_config.update(dict(
+			url='https://securegw.paytm.in/order/process',
+			transaction_status_url='https://securegw.paytm.in/order/status',
+		))
+	return paytm_config
 
-	def generate_transaction_token(self):
-		import stripe
-		try:
-			charge = stripe.Charge.create(amount=cint(flt(self.data.amount)*100), currency=self.data.currency, source=self.data.stripe_token_id, description=self.data.description, receipt_email=self.data.payer_email)
-
-			if charge.captured == True:
-				self.integration_request.db_set('status', 'Completed', update_modified=False)
-				self.flags.status_changed_to = "Completed"
-
-			else:
-				frappe.log_error(charge.failure_message, 'Stripe Payment not completed')
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-
-		return self.finalize_request()
-
-
-	def finalize_request(self):
-		redirect_to = self.data.get('redirect_to') or None
-		redirect_message = self.data.get('redirect_message') or None
-		status = self.integration_request.status
-
-		if self.flags.status_changed_to == "Completed":
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					custom_redirect_to = frappe.get_doc(self.data.reference_doctype,
-						self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to)
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-				redirect_url = 'payment-success'
-
-			if self.redirect_url:
-				redirect_url = self.redirect_url
-				redirect_to = None
-		else:
-			redirect_url = 'payment-failed'
-
-		if redirect_to:
-			redirect_url += '?' + urlencode({'redirect_to': redirect_to})
-		if redirect_message:
-			redirect_url += '&' + urlencode({'redirect_message': redirect_message})
-
-		return {
-			"redirect_to": redirect_url,
-			"status": status
-		}
-
-def generate_transaction_token(payment_details, order_id, merchant_id, merchant_key):
+def get_paytm_params(payment_details, order_id, paytm_config):
 
 	# initialize a dictionary
-	paytmParams = dict()
+	paytm_params = dict()
+	
+	# redirect_uri = get_request_site_address(True) + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.get_transaction_status"
+	redirect_uri = "http://cf9b2bb1.ngrok.io/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.get_transaction_status"
 
-	redirect_uri = get_request_site_address(True) + "?cmd=frappe.templates.pages.integrations.paytm_checkout.get_transaction_status"
+	paytm_params.update({
+		"MID" : paytm_config.merchant_id,
+		"WEBSITE" : paytm_config.website,
+		"INDUSTRY_TYPE_ID" : paytm_config.industry_type_id,
+		"CHANNEL_ID" : "WEB",
+		"ORDER_ID" : order_id,
+		"CUST_ID" : payment_details['payer_email'],
+		"EMAIL" : payment_details['payer_email'],
+		"TXN_AMOUNT" : cstr(flt(payment_details['amount'], 2)),
+		"CALLBACK_URL" : redirect_uri,
+	})
 
-	# body parameters
-	paytmParams["body"] = get_paytm_params(payment_details, order_id, merchant_id, merchant_key)
+	checksum = generate_checksum(paytm_params, paytm_config.merchant_key)
 
-	checksum = generate_checksum_by_str(json.dumps(paytmParams["body"]), merchant_key)
+	paytm_params.update({
+		"CHECKSUMHASH" : checksum
+	})
 
-	paytmParams["head"] = {
-		"signature"  : checksum
-	}
+	return paytm_params
 
-	post_data = json.dumps(paytmParams)
+@frappe.whitelist(allow_guest=True)
+def verify_transaction(**kwargs):
+	'''Verify checksum for received data in the callback and then verify the transaction'''
+	paytm_config = get_paytm_config()
+	received_data = frappe._dict(kwargs)
 
-	url = "https://securegw-stage.paytm.in/theia/api/v1/initiateTransaction?mid={0}&orderId={1}".format(merchant_id, order_id)
+	print(received_data)
+	paytm_params = {}
+	for key, value in received_data.items(): 
+		if key == 'CHECKSUMHASH':
+			paytm_checksum = value
+		else:
+			paytm_params[key] = value
+
+	# Verify checksum
+	is_valid_checksum = verify_checksum(paytm_params, paytm_config.merchant_key, paytm_checksum)
+
+	if is_valid_checksum and received_data['RESPCODE'] == '01':
+		verify_transaction_status(paytm_config, received_data['ORDERID'])
+	else:
+		frappe.respond_as_web_page("Payment Failed",
+			"Transaction failed to complete. Don't worry, in case of failure amount will get refunded to your account.",
+			http_status_code=401, indicator_color='red')
+		frappe.log_error("Order unsuccessful, received data:"+received_data, 'Paytm Payment Failed')
+
+def verify_transaction_status(paytm_config, order_id):
+	'''Verify transaction completion after checksum has been verified'''
+	paytm_params=dict(
+		MID=paytm_config.merchant_id,
+		ORDERID= order_id
+	)
+
+	checksum = generate_checksum(paytm_params, paytm_config.merchant_key)
+	paytm_params["CHECKSUMHASH"] = checksum
+
+	post_data = json.dumps(paytm_params)
+	url = paytm_config.transaction_status_url
 
 	response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json()
-	return response['body'].get('txnToken')
+	print('transaction status response')
+	print(response)
+	finalize_request(order_id, response)
+
+def finalize_request(order_id, response):
+	request = frappe.db.get_value('Integration Request', order_id)
+	redirect_to = request.data.get('redirect_to') or None
+	redirect_message = request.data.get('redirect_message') or None
+
+	if request.flags.status_changed_to == "Completed":
+		if request.data.reference_doctype and request.data.reference_docname:
+			custom_redirect_to = None
+			try:
+				custom_redirect_to = frappe.get_doc(request.data.reference_doctype,
+					request.data.reference_docname).run_method("on_payment_authorized", request.flags.status_changed_to)
+			except Exception:
+				frappe.log_error(frappe.get_traceback())
+
+			if custom_redirect_to:
+				redirect_to = custom_redirect_to
+
+			redirect_url = 'payment-success'
+	else:
+		redirect_url = 'payment-failed'
+
+	if redirect_to:
+		redirect_url += '?' + urlencode({'redirect_to': redirect_to})
+	if redirect_message:
+		redirect_url += '&' + urlencode({'redirect_message': redirect_message})
 
-def get_paytm_params(payment_details, order_id, merchant_id, merchant_key):
 	return {
-		"requestType" : "Payment",
-		"mid" : merchant_id,
-		"websiteName" : "WEBSTAGING",
-		"orderId" : order_id,
-		"callbackUrl" : redirect_uri,
-		"txnAmount" : {
-			"value" : flt(payment_details['amount'], 2),
-			"currency" : "INR",
-		},
-		"userInfo" : {
-			"custId" : payment_details['payer_email'],
-		},
+		"redirect_to": redirect_url,
+		"status": status
 	}
 
 def get_gateway_controller(doctype, docname):

From cf5da1c787e1a24ef707d8541e26e5091421ee63 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 17 Apr 2020 10:02:54 +0530
Subject: [PATCH 007/361] fix: check for transaction success in the final stage

---
 .../doctype/paytm_settings/paytm_settings.py        | 13 ++++---------
 .../templates/pages/integrations/paytm_checkout.py  |  1 -
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 10282d34ea..3382aeb6ed 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -100,7 +100,6 @@ def verify_transaction(**kwargs):
 	paytm_config = get_paytm_config()
 	received_data = frappe._dict(kwargs)
 
-	print(received_data)
 	paytm_params = {}
 	for key, value in received_data.items(): 
 		if key == 'CHECKSUMHASH':
@@ -133,16 +132,14 @@ def verify_transaction_status(paytm_config, order_id):
 	url = paytm_config.transaction_status_url
 
 	response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json()
-	print('transaction status response')
-	print(response)
 	finalize_request(order_id, response)
 
-def finalize_request(order_id, response):
+def finalize_request(order_id, transaction_response):
 	request = frappe.db.get_value('Integration Request', order_id)
 	redirect_to = request.data.get('redirect_to') or None
 	redirect_message = request.data.get('redirect_message') or None
 
-	if request.flags.status_changed_to == "Completed":
+	if transaction_response['STATUS'] == "TXN_SUCCESS":
 		if request.data.reference_doctype and request.data.reference_docname:
 			custom_redirect_to = None
 			try:
@@ -163,10 +160,8 @@ def finalize_request(order_id, response):
 	if redirect_message:
 		redirect_url += '&' + urlencode({'redirect_message': redirect_message})
 
-	return {
-		"redirect_to": redirect_url,
-		"status": status
-	}
+	frappe.local.response['type'] = 'redirect'
+	frappe.local.response['location'] = 'redirect_url'
 
 def get_gateway_controller(doctype, docname):
 	reference_doc = frappe.get_doc(doctype, docname)
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
index 1c61d5cab4..1fa947a68f 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.py
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -21,7 +21,6 @@ def get_context(context):
 
 	try:
 		doc = frappe.get_doc("Integration Request", frappe.form_dict['order_id'])
-		payment_details = json.loads(doc.data)
 
 		context.payment_details = get_paytm_params(json.loads(doc.data), doc.name, paytm_config)
 

From 0cbf937763cf96ee2fd567aba550ebf0fe29b273 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 17 Apr 2020 13:34:26 +0530
Subject: [PATCH 008/361] fix: change status of integration request on
 completion

---
 .../integrations/doctype/paytm_settings/paytm_settings.py  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 3382aeb6ed..39a9f42aa0 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -135,7 +135,7 @@ def verify_transaction_status(paytm_config, order_id):
 	finalize_request(order_id, response)
 
 def finalize_request(order_id, transaction_response):
-	request = frappe.db.get_value('Integration Request', order_id)
+	request = frappe.get_doc('Integration Request', order_id)
 	redirect_to = request.data.get('redirect_to') or None
 	redirect_message = request.data.get('redirect_message') or None
 
@@ -144,8 +144,10 @@ def finalize_request(order_id, transaction_response):
 			custom_redirect_to = None
 			try:
 				custom_redirect_to = frappe.get_doc(request.data.reference_doctype,
-					request.data.reference_docname).run_method("on_payment_authorized", request.flags.status_changed_to)
+					request.data.reference_docname).run_method("on_payment_authorized", 'Completed')
+				request.db_set('status', 'Completed')
 			except Exception:
+				request.db_set('status', 'Failed')
 				frappe.log_error(frappe.get_traceback())
 
 			if custom_redirect_to:
@@ -153,6 +155,7 @@ def finalize_request(order_id, transaction_response):
 
 			redirect_url = 'payment-success'
 	else:
+		request.db_set('status', 'Failed')
 		redirect_url = 'payment-failed'
 
 	if redirect_to:

From 6eaefb35c7137352f19f2c7d153aa5c78d740bba Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Wed, 22 Apr 2020 11:40:08 +0530
Subject: [PATCH 009/361] fix: add finalize request to complete the transaction

---
 .../doctype/paytm_settings/paytm_settings.py  | 33 ++++++++-------
 .../includes/integrations/paytm_checkout.js   | 42 -------------------
 .../pages/integrations/paytm_checkout.html    |  4 +-
 .../pages/integrations/paytm_checkout.py      |  3 --
 .../website_theme/standard/standard.json      | 20 ---------
 5 files changed, 22 insertions(+), 80 deletions(-)
 delete mode 100644 frappe/templates/includes/integrations/paytm_checkout.js
 delete mode 100644 frappe/website/website_theme/standard/standard.json

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 39a9f42aa0..394cd9c334 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -71,8 +71,8 @@ def get_paytm_params(payment_details, order_id, paytm_config):
 	# initialize a dictionary
 	paytm_params = dict()
 	
-	# redirect_uri = get_request_site_address(True) + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.get_transaction_status"
-	redirect_uri = "http://cf9b2bb1.ngrok.io/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.get_transaction_status"
+	redirect_uri = get_request_site_address(True) + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction"
+
 
 	paytm_params.update({
 		"MID" : paytm_config.merchant_id,
@@ -99,24 +99,28 @@ def verify_transaction(**kwargs):
 	'''Verify checksum for received data in the callback and then verify the transaction'''
 	paytm_config = get_paytm_config()
 	received_data = frappe._dict(kwargs)
+	is_valid_checksum = False
 
 	paytm_params = {}
 	for key, value in received_data.items(): 
 		if key == 'CHECKSUMHASH':
 			paytm_checksum = value
+		elif key == 'cmd':
+			continue
 		else:
 			paytm_params[key] = value
 
-	# Verify checksum
-	is_valid_checksum = verify_checksum(paytm_params, paytm_config.merchant_key, paytm_checksum)
+	if paytm_params and paytm_config and paytm_checksum:
+		# Verify checksum
+		is_valid_checksum = verify_checksum(paytm_params, paytm_config.merchant_key, paytm_checksum)
 
 	if is_valid_checksum and received_data['RESPCODE'] == '01':
 		verify_transaction_status(paytm_config, received_data['ORDERID'])
 	else:
 		frappe.respond_as_web_page("Payment Failed",
-			"Transaction failed to complete. Don't worry, in case of failure amount will get refunded to your account.",
+			"Transaction failed to complete. In case of any deductions, deducted amount will get refunded to your account.",
 			http_status_code=401, indicator_color='red')
-		frappe.log_error("Order unsuccessful, received data:"+received_data, 'Paytm Payment Failed')
+		frappe.log_error("Order unsuccessful. Failed Response:"+cstr(received_data), 'Paytm Payment Failed')
 
 def verify_transaction_status(paytm_config, order_id):
 	'''Verify transaction completion after checksum has been verified'''
@@ -136,15 +140,16 @@ def verify_transaction_status(paytm_config, order_id):
 
 def finalize_request(order_id, transaction_response):
 	request = frappe.get_doc('Integration Request', order_id)
-	redirect_to = request.data.get('redirect_to') or None
-	redirect_message = request.data.get('redirect_message') or None
+	transaction_data = frappe._dict(json.loads(request.data))
+	redirect_to = transaction_data.get('redirect_to') or None
+	redirect_message = transaction_data.get('redirect_message') or None
 
 	if transaction_response['STATUS'] == "TXN_SUCCESS":
-		if request.data.reference_doctype and request.data.reference_docname:
+		if transaction_data.reference_doctype and transaction_data.reference_docname:
 			custom_redirect_to = None
 			try:
-				custom_redirect_to = frappe.get_doc(request.data.reference_doctype,
-					request.data.reference_docname).run_method("on_payment_authorized", 'Completed')
+				custom_redirect_to = frappe.get_doc(transaction_data.reference_doctype,
+					transaction_data.reference_docname).run_method("on_payment_authorized", 'Completed')
 				request.db_set('status', 'Completed')
 			except Exception:
 				request.db_set('status', 'Failed')
@@ -153,10 +158,10 @@ def finalize_request(order_id, transaction_response):
 			if custom_redirect_to:
 				redirect_to = custom_redirect_to
 
-			redirect_url = 'payment-success'
+			redirect_url = '/integrations/payment-success'
 	else:
 		request.db_set('status', 'Failed')
-		redirect_url = 'payment-failed'
+		redirect_url = '/integrations/payment-failed'
 
 	if redirect_to:
 		redirect_url += '?' + urlencode({'redirect_to': redirect_to})
@@ -164,7 +169,7 @@ def finalize_request(order_id, transaction_response):
 		redirect_url += '&' + urlencode({'redirect_message': redirect_message})
 
 	frappe.local.response['type'] = 'redirect'
-	frappe.local.response['location'] = 'redirect_url'
+	frappe.local.response['location'] = redirect_url
 
 def get_gateway_controller(doctype, docname):
 	reference_doc = frappe.get_doc(doctype, docname)
diff --git a/frappe/templates/includes/integrations/paytm_checkout.js b/frappe/templates/includes/integrations/paytm_checkout.js
deleted file mode 100644
index 4cf070b60b..0000000000
--- a/frappe/templates/includes/integrations/paytm_checkout.js
+++ /dev/null
@@ -1,42 +0,0 @@
-function onScriptLoad() {
-	console.log('inside on load')
-	var config = {
-		root: '',
-		flow: 'DEFAULT',
-		data: {
-			orderId: '{{ order_id}}',
-			token: '{{ token }}',
-			tokenType: 'TXN_TOKEN',
-			amount: '{{ amount }}'
-		},
-		handler: {
-			notifyMerchant: function(eventName, data) {
-				// notify about the state of the payment page ( invalid token , session expire , cancel transaction)
-				console.log('notifyMerchant handler function called');
-				console.log('eventName => ', eventName);
-				console.log('data => ', data);
-			},
-			transactionStatus: function transactionStatus(paymentStatus) {
-				// provide information to merchant about the payment status.
-				console.log('transaction status handler function called');
-				console.log('paymentStatus => ', paymentStatus);
-			}
-		}
-	};
-
-	$('.paytm-loading').addClass('hidden');
-	if (window.Paytm && window.Paytm.CheckoutJS) {
-		window.Paytm.CheckoutJS.onLoad(function excecuteAfterCompleteLoad() {
-			// initialze configuration using init method
-			window.Paytm.CheckoutJS.init(config)
-				.then(function onSuccess() {
-					// after successfully updating configuration, invoke Blink Checkout
-					window.Paytm.CheckoutJS.invoke();
-				})
-				.catch(function onError(error) {
-					console.log('inside the error window')
-					console.log('error => ', error);
-				});
-		});
-	}
-}
diff --git a/frappe/templates/pages/integrations/paytm_checkout.html b/frappe/templates/pages/integrations/paytm_checkout.html
index 772e16ecb8..77a2045edf 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.html
+++ b/frappe/templates/pages/integrations/paytm_checkout.html
@@ -16,12 +16,14 @@
 
 {%- block page_content -%}
 <body>
-		<center><h1>Please do not refresh this page...</h1></center>
+	<div style="margin: 30vh;">
+		<center><h2>Please do not refresh this page...</h2></center>
 		<form method="post" action="{{ url }}" name="paytm_form">
 			{% for name, value in payment_details.items() %}
 				<input type="hidden" name="{{ name }}" value="{{ value }}">
 			{% endfor %}
 		</form>
+	</div>
 </body>
 {% endblock %}
 
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
index 1fa947a68f..5a55d02ce9 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.py
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -12,9 +12,6 @@ import requests
 
 no_cache = 1
 
-expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'reference_docname',
-	'payer_name', 'payer_email')
-
 def get_context(context):
 	context.no_cache = 1
 	paytm_config = get_paytm_config()
diff --git a/frappe/website/website_theme/standard/standard.json b/frappe/website/website_theme/standard/standard.json
deleted file mode 100644
index a799f25425..0000000000
--- a/frappe/website/website_theme/standard/standard.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "button_gradients": 0,
- "button_rounded_corners": 1,
- "button_shadows": 0,
- "creation": "2015-02-19 13:37:33.925909",
- "custom": 0,
- "docstatus": 0,
- "doctype": "Website Theme",
- "font_properties": "300,600",
- "font_size": "",
- "idx": 27,
- "modified": "2020-04-21 02:10:31.761219",
- "modified_by": "Administrator",
- "module": "Website",
- "name": "Standard",
- "owner": "Administrator",
- "theme": "Standard",
- "theme_scss": "$enable-shadows: false;\n$enable-gradients: false;\n$enable-rounded: true;\n\n@import \"frappe/public/scss/website\";\n\n",
- "theme_url": "/assets/css/standard_style.css"
-}
\ No newline at end of file

From 25c9ac002290d070ab596a872faeba926f8317ad Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Wed, 22 Apr 2020 16:12:49 +0530
Subject: [PATCH 010/361] chore: remove initial validation check

---
 .../integrations/doctype/paytm_settings/checksum.py |  1 -
 .../doctype/paytm_settings/paytm_settings.py        | 13 +------------
 .../templates/pages/integrations/paytm_checkout.py  |  6 +-----
 3 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/checksum.py b/frappe/integrations/doctype/paytm_settings/checksum.py
index 95fb7dc164..b2e67bb5ad 100644
--- a/frappe/integrations/doctype/paytm_settings/checksum.py
+++ b/frappe/integrations/doctype/paytm_settings/checksum.py
@@ -87,7 +87,6 @@ def __get_param_string__(params):
 	params_string = []
 	for key in sorted(params.keys()):
 		if("REFUND" in params[key] or "|" in params[key]):
-			respons_dict = {}
 			exit()
 		value = params[key]
 		params_string.append('' if value == 'null' else str(value))
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index 394cd9c334..ed71202ea6 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -11,7 +11,7 @@ import frappe
 from frappe.model.document import Document
 from frappe import _
 from frappe.utils import get_url, call_hook_method, cint, flt, cstr
-from frappe.integrations.utils import make_get_request, make_post_request, create_request_log, create_payment_gateway
+from frappe.integrations.utils import create_request_log, create_payment_gateway
 from frappe.utils import get_request_site_address
 from frappe.integrations.doctype.paytm_settings.checksum import generate_checksum, verify_checksum
 from frappe.utils.password import get_decrypted_password
@@ -22,17 +22,6 @@ class PaytmSettings(Document):
 	def validate(self):
 		create_payment_gateway('Paytm')
 		call_hook_method('payment_gateway_enabled', gateway='Paytm')
-		if not self.flags.ignore_mandatory:
-			self.validate_paytm_credentails()
-
-	def validate_paytm_credentails(self):
-		if self.merchant_id and self.merchant_key:
-			pass
-			# header = {"Authorization": "Bearer {0}".format(self.get_password(fieldname="secret_key", raise_exception=False))}
-			# try:
-			# 	make_get_request(url="https://api.stripe.com/v1/charges", headers=header)
-			# except Exception:
-			# 	frappe.throw(_("Seems Publishable Key or Secret Key is wrong !!!"))
 
 	def validate_transaction_currency(self, currency):
 		if currency not in self.supported_currencies:
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
index 5a55d02ce9..3018a9d65b 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.py
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -3,12 +3,8 @@
 from __future__ import unicode_literals
 import frappe
 from frappe import _
-from frappe.utils import flt, cint
 import json
-from six import string_types
-from frappe.utils import get_request_site_address
 from frappe.integrations.doctype.paytm_settings.paytm_settings import get_paytm_params, get_paytm_config
-import requests
 
 no_cache = 1
 
@@ -23,7 +19,7 @@ def get_context(context):
 
 		context.url = paytm_config.url
 
-	except Exception as e:
+	except Exception:
 		frappe.log_error()
 		frappe.redirect_to_message(_('Invalid Token'),
 			_('Seems token you are using is invalid!'),

From 7dbdb7c50bb37e0fa1e371032c4b2516d3089c4e Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 24 Apr 2020 11:18:35 +0530
Subject: [PATCH 011/361] fix: add pycryptodome for paytm checksum generation

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 58f923d880..2587a2eceb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -66,3 +66,4 @@ watchdog==0.8.0
 Werkzeug==0.16.1
 xlrd==1.2.0
 zxcvbn-python==4.4.24
+pycryptodome==3.9.7
\ No newline at end of file

From 7a15f6f61d121901aa08c1f0bd1c3627bda0fe34 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 24 Apr 2020 15:09:11 +0530
Subject: [PATCH 012/361] revert: add pylint and standard json again

---
 .pylintrc                                     |  2 ++
 .../website_theme/standard/standard.json      | 20 +++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 .pylintrc
 create mode 100644 frappe/website/website_theme/standard/standard.json

diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 0000000000..c11c0ab6a3
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,2 @@
+disable=access-member-before-definition
+disable=no-member
\ No newline at end of file
diff --git a/frappe/website/website_theme/standard/standard.json b/frappe/website/website_theme/standard/standard.json
new file mode 100644
index 0000000000..a799f25425
--- /dev/null
+++ b/frappe/website/website_theme/standard/standard.json
@@ -0,0 +1,20 @@
+{
+ "button_gradients": 0,
+ "button_rounded_corners": 1,
+ "button_shadows": 0,
+ "creation": "2015-02-19 13:37:33.925909",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "Website Theme",
+ "font_properties": "300,600",
+ "font_size": "",
+ "idx": 27,
+ "modified": "2020-04-21 02:10:31.761219",
+ "modified_by": "Administrator",
+ "module": "Website",
+ "name": "Standard",
+ "owner": "Administrator",
+ "theme": "Standard",
+ "theme_scss": "$enable-shadows: false;\n$enable-gradients: false;\n$enable-rounded: true;\n\n@import \"frappe/public/scss/website\";\n\n",
+ "theme_url": "/assets/css/standard_style.css"
+}
\ No newline at end of file

From 0781d048d178aea71ccfdeeeb5d6caac3db42299 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Tue, 2 Jun 2020 12:33:01 +0530
Subject: [PATCH 013/361] fix: add docs link to the settings page dashboard

---
 .../integrations/doctype/paytm_settings/paytm_settings.js   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.js b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
index d51a6e4a25..fe2ee7c952 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.js
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
@@ -2,7 +2,7 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Paytm Settings', {
-	// refresh: function(frm) {
-
-	// }
+	refresh: function(frm) {
+		frm.dashboard.set_headline(__("For more information, {0}.", [`<a href='https://erpnext.com/docs/user/manual/en/erpnext_integration/paytm-integration'>${__('Click here')}</a>`]));
+	}
 });

From 04c070d82d3bb1df4072a52f279478487a2511cc Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 4 Jun 2020 14:38:46 +0530
Subject: [PATCH 014/361] fix: add paytm settings link to the module page

---
 frappe/config/integrations.py            | 5 +++++
 frappe/templates/includes/login/login.js | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/frappe/config/integrations.py b/frappe/config/integrations.py
index f41adc9ea4..a7ac20065f 100644
--- a/frappe/config/integrations.py
+++ b/frappe/config/integrations.py
@@ -27,6 +27,11 @@ def get_data():
 					"name": "Stripe Settings",
 					"description": _("Stripe payment gateway settings"),
 				},
+				{
+					"type": "doctype",
+					"name": "Paytm Settings",
+					"description": _("Paytm payment gateway settings"),
+				},
 			]
 		},
 		{
diff --git a/frappe/templates/includes/login/login.js b/frappe/templates/includes/login/login.js
index 37b8acb328..793fcc1720 100644
--- a/frappe/templates/includes/login/login.js
+++ b/frappe/templates/includes/login/login.js
@@ -186,7 +186,7 @@ login.login_handlers = (function() {
 			} else if(data.message == 'Password Reset'){
 				window.location.href = frappe.utils.sanitise_redirect(data.redirect_to);
 			} else if(data.message=="No App") {
-				login.set_indicator("{{ _("Success") }}", 'green');
+				login.set_indicator("{{ _('Success') }}", 'green');
 				if(localStorage) {
 					var last_visited =
 						localStorage.getItem("last_visited")

From a9a8fc953fa6a90be8718a9d5c1f9b16fa890e87 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Mon, 8 Jun 2020 17:06:48 +0530
Subject: [PATCH 015/361] fix: update the checksum logic

---
 .../doctype/paytm_settings/checksum.py        | 172 ++++++------------
 .../paytm_settings/paytm_settings.json        |  40 ++--
 .../doctype/paytm_settings/paytm_settings.py  |   8 +-
 3 files changed, 85 insertions(+), 135 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/checksum.py b/frappe/integrations/doctype/paytm_settings/checksum.py
index b2e67bb5ad..32f976ae18 100644
--- a/frappe/integrations/doctype/paytm_settings/checksum.py
+++ b/frappe/integrations/doctype/paytm_settings/checksum.py
@@ -2,141 +2,79 @@ import base64
 import string
 import random
 import hashlib
+import sys
 
 from Crypto.Cipher import AES
 
 
-IV = "@@@@&&&&####$$$$"
+iv = '@@@@&&&&####$$$$'
 BLOCK_SIZE = 16
 
+if (sys.version_info > (3, 0)):
+	__pad__ = lambda s: bytes(s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE), 'utf-8')
+else:
+	__pad__ = lambda s: s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE)
 
-def generate_checksum(param_dict, merchant_key, salt=None):
-	params_string = __get_param_string__(param_dict)
-	salt = salt if salt else __id_generator__(4)
-	final_string = '%s|%s' % (params_string, salt)
+__unpad__ = lambda s: s[0:-ord(s[-1])]
 
-	hasher = hashlib.sha256(final_string.encode())
-	hash_string = hasher.hexdigest()
+def encrypt(input, key):
+	input = __pad__(input)
+	c = AES.new(key.encode("utf8"), AES.MODE_CBC, iv.encode("utf8"))
+	input = c.encrypt(input)
+	input = base64.b64encode(input)
+	return input.decode("UTF-8")
 
-	hash_string += salt
+def decrypt(encrypted, key):
+	encrypted = base64.b64decode(encrypted)
+	c = AES.new(key.encode("utf8"), AES.MODE_CBC, iv.encode("utf8"))
+	param = c.decrypt(encrypted)
+	if type(param) == bytes:
+		param = param.decode()
+	return __unpad__(param)
 
-	return __encode__(hash_string, IV, merchant_key)
+def generateSignature(params, key):
+	if not type(params) is dict and not type(params) is str:
+		raise Exception("string or dict expected, " + str(type(params)) + " given")
+	if type(params) is dict:
+		params = getStringByParams(params)
+	return generateSignatureByString(params, key)
 
+def verifySignature(params, key, checksum):
+	if not type(params) is dict and not type(params) is str:
+		raise Exception("string or dict expected, " + str(type(params)) + " given")
+	if "CHECKSUMHASH" in params:
+		del params["CHECKSUMHASH"]
+		
+	if type(params) is dict:
+		params = getStringByParams(params)
+	return verifySignatureByString(params, key, checksum)
 
-def generate_refund_checksum(param_dict, merchant_key, salt=None):
-	for i in param_dict:
-		if("|" in param_dict[i]):
-			param_dict = {}
-			exit()
-	params_string = __get_param_string__(param_dict)
-	salt = salt if salt else __id_generator__(4)
-	final_string = '%s|%s' % (params_string, salt)
+def generateSignatureByString(params, key):
+	salt = generateRandomString(4)
+	return calculateChecksum(params, key, salt)
 
-	hasher = hashlib.sha256(final_string.encode())
-	hash_string = hasher.hexdigest()
-
-	hash_string += salt
-
-	return __encode__(hash_string, IV, merchant_key)
-
-
-def generate_checksum_by_str(param_str, merchant_key, salt=None):
-	params_string = param_str
-	salt = salt if salt else __id_generator__(4)
-	final_string = '%s|%s' % (params_string, salt)
-
-	hasher = hashlib.sha256(final_string.encode())
-	hash_string = hasher.hexdigest()
-
-	hash_string += salt
-
-	return __encode__(hash_string, IV, merchant_key)
-
-
-def verify_checksum(param_dict, merchant_key, checksum):
-	# Remove checksum
-	if 'CHECKSUMHASH' in param_dict:
-		param_dict.pop('CHECKSUMHASH')
-
-	# Get salt
-	paytm_hash = __decode__(checksum, IV, merchant_key)
+def verifySignatureByString(params, key, checksum):
+	paytm_hash = decrypt(checksum, key)    
 	salt = paytm_hash[-4:]
-	calculated_checksum = generate_checksum(
-		param_dict, merchant_key, salt=salt)
-	return calculated_checksum == checksum
+	return paytm_hash == calculateHash(params, salt)
 
+def generateRandomString(length):
+	chars = string.ascii_uppercase + string.digits + string.ascii_lowercase
+	return ''.join(random.choice(chars) for _ in range(length))
 
-def verify_checksum_by_str(param_str, merchant_key, checksum):
-	# Remove checksum
-	# if 'CHECKSUMHASH' in param_dict:
-		# param_dict.pop('CHECKSUMHASH')
-
-	# Get salt
-	paytm_hash = __decode__(checksum, IV, merchant_key)
-	salt = paytm_hash[-4:]
-	calculated_checksum = generate_checksum_by_str(
-		param_str, merchant_key, salt=salt)
-	return calculated_checksum == checksum
-
-
-def __id_generator__(size=6, chars=string.ascii_uppercase + string.digits + string.ascii_lowercase):
-	return ''.join(random.choice(chars) for _ in range(size))
-
-
-def __get_param_string__(params):
+def getStringByParams(params):
 	params_string = []
 	for key in sorted(params.keys()):
-		if("REFUND" in params[key] or "|" in params[key]):
-			exit()
-		value = params[key]
-		params_string.append('' if value == 'null' else str(value))
+		value = params[key] if params[key] is not None and params[key].lower() != "null" else ""
+		params_string.append(str(value))
 	return '|'.join(params_string)
 
+def calculateHash(params, salt):
+	finalString = '%s|%s' % (params, salt)
+	hasher = hashlib.sha256(finalString.encode())
+	hashString = hasher.hexdigest() + salt
+	return hashString
 
-def __pad__(s): return s + (BLOCK_SIZE - len(s) %
-							BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE)
-
-
-def __unpad__(s): return s[0:-ord(s[-1])]
-
-
-def __encode__(to_encode, iv, key):
-	# Pad
-	to_encode = __pad__(to_encode)
-	# Encrypt
-	c = AES.new(key, AES.MODE_CBC, iv)
-	to_encode = c.encrypt(to_encode)
-	# Encode
-	to_encode = base64.b64encode(to_encode)
-	return to_encode.decode("UTF-8")
-
-
-def __decode__(to_decode, iv, key):
-	# Decode
-	to_decode = base64.b64decode(to_decode)
-	# Decrypt
-	c = AES.new(key, AES.MODE_CBC, iv)
-	to_decode = c.decrypt(to_decode)
-	if type(to_decode) == bytes:
-		# convert bytes array to str.
-		to_decode = to_decode.decode()
-	# remove pad
-	return __unpad__(to_decode)
-
-
-if __name__ == "__main__":
-	params = {
-		"MID": "mid",
-		"ORDER_ID": "order_id",
-		"CUST_ID": "cust_id",
-		"TXN_AMOUNT": "1",
-		"CHANNEL_ID": "WEB",
-		"INDUSTRY_TYPE_ID": "Retail",
-		"WEBSITE": "xxxxxxxxxxx"
-	}
-
-	print(verify_checksum(
-		params, 'xxxxxxxxxxxxxxxx',
-		"CD5ndX8VVjlzjWbbYoAtKQIlvtXPypQYOg0Fi2AUYKXZA5XSHiRF0FDj7vQu66S8MHx9NaDZ/uYm3WBOWHf+sDQAmTyxqUipA7i1nILlxrk="))
-
-	# print(generate_checksum(params, "xxxxxxxxxxxxxxxx"))
+def calculateChecksum(params, key, salt): 
+	hashString = calculateHash(params, salt)
+	return encrypt(hashString, key)
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
index 0540dc4948..93fbd0df09 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.json
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
@@ -9,7 +9,7 @@
   "merchant_key",
   "staging",
   "column_break_4",
-  "industry_type",
+  "industry_type_id",
   "website"
  ],
  "fields": [
@@ -18,43 +18,55 @@
    "fieldtype": "Data",
    "in_list_view": 1,
    "label": "Merchant ID",
-   "reqd": 1
+   "reqd": 1,
+   "show_days": 1,
+   "show_seconds": 1
   },
   {
    "fieldname": "merchant_key",
    "fieldtype": "Password",
    "in_list_view": 1,
    "label": "Merchant Key",
-   "reqd": 1
+   "reqd": 1,
+   "show_days": 1,
+   "show_seconds": 1
   },
   {
    "default": "0",
    "fieldname": "staging",
    "fieldtype": "Check",
-   "label": "Staging"
-  },
-  {
-   "depends_on": "eval: !doc.staging",
-   "fieldname": "industry_type",
-   "fieldtype": "Data",
-   "label": "Industry Type",
-   "mandatory_depends_on": "eval: !doc.staging"
+   "label": "Staging",
+   "show_days": 1,
+   "show_seconds": 1
   },
   {
    "depends_on": "eval: !doc.staging",
    "fieldname": "website",
    "fieldtype": "Data",
    "label": "Website",
-   "mandatory_depends_on": "eval: !doc.staging"
+   "mandatory_depends_on": "eval: !doc.staging",
+   "show_days": 1,
+   "show_seconds": 1
   },
   {
    "fieldname": "column_break_4",
-   "fieldtype": "Column Break"
+   "fieldtype": "Column Break",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "depends_on": "eval: !doc.staging",
+   "fieldname": "industry_type_id",
+   "fieldtype": "Data",
+   "label": "Industry Type ID",
+   "mandatory_depends_on": "eval: !doc.staging",
+   "show_days": 1,
+   "show_seconds": 1
   }
  ],
  "issingle": 1,
  "links": [],
- "modified": "2020-04-16 14:16:19.687449",
+ "modified": "2020-06-08 13:36:09.703143",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Paytm Settings",
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index ed71202ea6..c169a53246 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -13,7 +13,7 @@ from frappe import _
 from frappe.utils import get_url, call_hook_method, cint, flt, cstr
 from frappe.integrations.utils import create_request_log, create_payment_gateway
 from frappe.utils import get_request_site_address
-from frappe.integrations.doctype.paytm_settings.checksum import generate_checksum, verify_checksum
+from frappe.integrations.doctype.paytm_settings.checksum import generateSignature, verifySignature
 from frappe.utils.password import get_decrypted_password
 
 class PaytmSettings(Document):
@@ -75,7 +75,7 @@ def get_paytm_params(payment_details, order_id, paytm_config):
 		"CALLBACK_URL" : redirect_uri,
 	})
 
-	checksum = generate_checksum(paytm_params, paytm_config.merchant_key)
+	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
 
 	paytm_params.update({
 		"CHECKSUMHASH" : checksum
@@ -101,7 +101,7 @@ def verify_transaction(**kwargs):
 
 	if paytm_params and paytm_config and paytm_checksum:
 		# Verify checksum
-		is_valid_checksum = verify_checksum(paytm_params, paytm_config.merchant_key, paytm_checksum)
+		is_valid_checksum = verifySignature(paytm_params, paytm_config.merchant_key, paytm_checksum)
 
 	if is_valid_checksum and received_data['RESPCODE'] == '01':
 		verify_transaction_status(paytm_config, received_data['ORDERID'])
@@ -118,7 +118,7 @@ def verify_transaction_status(paytm_config, order_id):
 		ORDERID= order_id
 	)
 
-	checksum = generate_checksum(paytm_params, paytm_config.merchant_key)
+	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
 	paytm_params["CHECKSUMHASH"] = checksum
 
 	post_data = json.dumps(paytm_params)

From 26063d88a2ae063fb30aebd1dd370feac7a158bf Mon Sep 17 00:00:00 2001
From: Ronel Cabrera <ronelvcabrera007@gmail.com>
Date: Fri, 26 Jun 2020 12:29:38 +0800
Subject: [PATCH 016/361] feat(controls/text_editor): resize image for markdown
 editor

---
 .../js/frappe/form/controls/text_editor.js    |  5 +-
 package.json                                  |  1 +
 yarn.lock                                     | 52 ++++++++++++++++++-
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/form/controls/text_editor.js b/frappe/public/js/frappe/form/controls/text_editor.js
index 3c0f7d5110..3b887aba0e 100644
--- a/frappe/public/js/frappe/form/controls/text_editor.js
+++ b/frappe/public/js/frappe/form/controls/text_editor.js
@@ -1,5 +1,7 @@
 import Quill from 'quill';
+import ImageResize from 'quill-image-resize';
 
+Quill.register('modules/imageResize', ImageResize);
 const CodeBlockContainer = Quill.import('formats/code-block-container');
 CodeBlockContainer.tagName = 'PRE';
 Quill.register(CodeBlockContainer, true);
@@ -146,7 +148,8 @@ frappe.ui.form.ControlTextEditor = frappe.ui.form.ControlCode.extend({
 		return {
 			modules: {
 				toolbar: this.get_toolbar_options(),
-				table: true
+				table: true,
+				imageResize: {}
 			},
 			theme: 'snow'
 		};
diff --git a/package.json b/package.json
index 3167db9e13..8326e88e71 100644
--- a/package.json
+++ b/package.json
@@ -39,6 +39,7 @@
     "moment-timezone": "^0.5.28",
     "quagga": "^0.12.1",
     "quill": "2.0.0-dev.4",
+    "quill-image-resize": "^3.0.9",
     "qz-tray": "^2.0.8",
     "redis": "^2.8.0",
     "showdown": "^1.9.1",
diff --git a/yarn.lock b/yarn.lock
index 45a33c5d42..de8ba197c2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1052,7 +1052,7 @@ cliui@^5.0.0:
     strip-ansi "^5.2.0"
     wrap-ansi "^5.1.0"
 
-clone@^2.1.2:
+clone@^2.1.1, clone@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
   integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
@@ -2015,6 +2015,11 @@ eventemitter2@4.1.2:
   resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-4.1.2.tgz#0e1a8477af821a6ef3995b311bf74c23a5247f15"
   integrity sha1-DhqEd6+CGm7zmVsxG/dMI6UkfxU=
 
+eventemitter3@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-2.0.3.tgz#b5e1079b59fb5e1ba2771c0a993be060a58c99ba"
+  integrity sha1-teEHm1n7XhuidxwKmTvgYKWMmbo=
+
 eventemitter3@^4.0.0:
   version "4.0.4"
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.4.tgz#b5463ace635a083d018bdc7c917b4c5f10a85384"
@@ -2200,6 +2205,11 @@ fast-deep-equal@^3.1.1:
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz#545145077c501491e33b15ec408c294376e94ae4"
   integrity sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==
 
+fast-diff@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.1.2.tgz#4b62c42b8e03de3f848460b639079920695d0154"
+  integrity sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==
+
 fast-diff@1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03"
@@ -4433,6 +4443,11 @@ parchment@2.0.0-dev.2:
   resolved "https://registry.yarnpkg.com/parchment/-/parchment-2.0.0-dev.2.tgz#9d6fe57b3721317bd1c481ea38ffa9b287d496b8"
   integrity sha512-4fgRny4pPISoML08Zp7poi52Dff3E2G1ORTi2D/acJ/RiROdDAMDB6VcQNfBcmehrX5Wixp6dxh6JjLyE5yUNQ==
 
+parchment@^1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/parchment/-/parchment-1.1.4.tgz#aeded7ab938fe921d4c34bc339ce1168bc2ffde5"
+  integrity sha512-J5FBQt/pM2inLzg4hEWmzQx/8h8D0CiDxaG3vyp9rKrQRSDgBlhjdP5jQGgosEajXPSQouXGHOmVdgo7QmJuOg==
+
 parse-data-uri@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/parse-data-uri/-/parse-data-uri-0.2.0.tgz#bf04d851dd5c87b0ab238e5d01ace494b604b4c9"
@@ -5124,6 +5139,24 @@ quill-delta@4.2.1:
     extend "^3.0.2"
     fast-diff "1.2.0"
 
+quill-delta@^3.6.2:
+  version "3.6.3"
+  resolved "https://registry.yarnpkg.com/quill-delta/-/quill-delta-3.6.3.tgz#b19fd2b89412301c60e1ff213d8d860eac0f1032"
+  integrity sha512-wdIGBlcX13tCHOXGMVnnTVFtGRLoP0imqxM696fIPwIf5ODIYUHIvHbZcyvGlZFiFhK5XzDC2lpjbxRhnM05Tg==
+  dependencies:
+    deep-equal "^1.0.1"
+    extend "^3.0.2"
+    fast-diff "1.1.2"
+
+quill-image-resize@^3.0.9:
+  version "3.0.9"
+  resolved "https://registry.yarnpkg.com/quill-image-resize/-/quill-image-resize-3.0.9.tgz#5677fb6257560bff951153ddbdb836758e49f804"
+  integrity sha512-5Dk0nixhbFsCwSWtPU9qqqtfM2gURfaP+pbBhQvAoMJoF4p99xbAibfAI3gsZJkbWUodoK2iAPf1V5oTSpv9ww==
+  dependencies:
+    lodash "^4.17.4"
+    quill "^1.2.2"
+    raw-loader "^0.5.1"
+
 quill@2.0.0-dev.4:
   version "2.0.0-dev.4"
   resolved "https://registry.yarnpkg.com/quill/-/quill-2.0.0-dev.4.tgz#130e38efe7a16b3766d767d750c8aacc038945e7"
@@ -5136,6 +5169,18 @@ quill@2.0.0-dev.4:
     parchment "2.0.0-dev.2"
     quill-delta "4.2.1"
 
+quill@^1.2.2:
+  version "1.3.7"
+  resolved "https://registry.yarnpkg.com/quill/-/quill-1.3.7.tgz#da5b2f3a2c470e932340cdbf3668c9f21f9286e8"
+  integrity sha512-hG/DVzh/TiknWtE6QmWAF/pxoZKYxfe3J/d/+ShUWkDvvkZQVTPeVmUJVu1uE6DDooC4fWTiCLh84ul89oNz5g==
+  dependencies:
+    clone "^2.1.1"
+    deep-equal "^1.0.1"
+    eventemitter3 "^2.0.3"
+    extend "^3.0.2"
+    parchment "^1.1.4"
+    quill-delta "^3.6.2"
+
 qz-tray@^2.0.8:
   version "2.0.8"
   resolved "https://registry.yarnpkg.com/qz-tray/-/qz-tray-2.0.8.tgz#5e15d102cf3a11a31ddb332891c7f8a6af8f6ad5"
@@ -5171,6 +5216,11 @@ raw-body@^2.2.0:
     iconv-lite "0.4.24"
     unpipe "1.0.0"
 
+raw-loader@^0.5.1:
+  version "0.5.1"
+  resolved "https://registry.yarnpkg.com/raw-loader/-/raw-loader-0.5.1.tgz#0c3d0beaed8a01c966d9787bf778281252a979aa"
+  integrity sha1-DD0L6u2KAclm2Xh793goElKpeao=
+
 rc@^1.0.1, rc@^1.1.6:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"

From 99596e9c3c4112b66fc664829726dd91176bfe99 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Thu, 9 Jul 2020 11:53:44 +0530
Subject: [PATCH 017/361] fix: auto email report for group by reports

---
 frappe/core/doctype/report/report.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 967b28b8b2..3228fec13a 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -173,8 +173,6 @@ class Report(Document):
 				columns = params.get('fields')
 			elif params.get('columns'):
 				columns = params.get('columns')
-			elif params.get('fields'):
-				columns = params.get('fields')
 			else:
 				columns = [['name', self.ref_doctype]]
 				for df in frappe.get_meta(self.ref_doctype).fields:
@@ -204,12 +202,23 @@ class Report(Document):
 			if params.get('sort_by_next'):
 				order_by += ', ' + _format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')
 
+			group_by = None
+			if params.get('group_by'):
+				group_by_args = frappe._dict(params['group_by'])
+				group_by = group_by_args['group_by']
+				order_by = '_aggregate_column desc'
+
 			result = frappe.get_list(self.ref_doctype,
-				fields = [_format([c[1], c[0]]) for c in columns],
+				fields = [
+					get_group_by_field(group_by_args, c[1]) if c[0] == '_aggregate_column' and group_by_args
+					else _format([c[1], c[0]])
+					for c in columns
+				],
 				filters=_filters,
 				order_by = order_by,
 				as_list=True,
 				limit=limit,
+				group_by=group_by,
 				user=user)
 
 			_columns = []
@@ -262,3 +271,15 @@ def is_prepared_report_disabled(report):
 def get_report_module_dotted_path(module, report_name):
 	return frappe.local.module_app[scrub(module)] + "." + scrub(module) \
 		+ ".report." + scrub(report_name) + "." + scrub(report_name)
+
+def get_group_by_field(args, doctype):
+	if args['aggregate_function'] == 'count':
+		group_by_field = 'count(*) as _aggregate_column'
+	else:
+		group_by_field = '{0}(`tab{1}`.{2}) as _aggregate_column'.format(
+			args.aggregate_function,
+			doctype,
+			args.aggregate_on
+		)
+
+	return group_by_field
\ No newline at end of file

From ccde503c5a0b81887eba33e0a17069d9ba08099b Mon Sep 17 00:00:00 2001
From: marination <maricadsouza221197@gmail.com>
Date: Mon, 13 Jul 2020 19:27:46 +0530
Subject: [PATCH 018/361] fix: Total Row in Reports

---
 .../js/frappe/views/reports/query_report.js       | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index f82956adac..609e1feb46 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -861,8 +861,19 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				if (column.isHeader && !data && this.data) {
 					// totalRow doesn't have a data object
 					// proxy it using the first data object
-					// this is needed only for currency formatting
-					data = this.data[0];
+					// applied to Float, Currency fields, needed only for currency formatting.
+					// make first data column have value 'Total'
+					let index = 1;
+					if (this.datatable && this.datatable.options.checkboxColumn) index = 2;
+
+					if (column.colIndex === index && !value) {
+						value = "Total";
+						column.fieldtype = "Data"; // avoid type issues for value if Date column
+					}
+					else if (in_list(["Currency", "Float"], column.fieldtype)) {
+						// proxy for currency and float
+						data = this.data[0];
+					}
 				}
 				return frappe.format(value, column,
 					{for_print: false, always_show_decimals: true}, data);

From f057260e90a5e8c7888ac9b62431bb19d09063ef Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Tue, 14 Jul 2020 17:49:34 +0530
Subject: [PATCH 019/361] fix: reorder result according to custom columns for
 script reports

---
 frappe/desk/query_report.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 0edfd57d4f..d0a32ef076 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -67,8 +67,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 		# Reordered columns
 		columns = json.loads(report.custom_columns)
 
-		if report.report_type == 'Query Report':
-			result = reorder_data_for_custom_columns(columns, query_columns, result)
+		result = reorder_data_for_custom_columns(columns, query_columns, result, report.report_type)
 
 		result = add_data_to_custom_columns(columns, result)
 
@@ -216,15 +215,21 @@ def add_data_to_custom_columns(columns, result):
 
 	return data
 
-def reorder_data_for_custom_columns(custom_columns, columns, result):
+def reorder_data_for_custom_columns(custom_columns, columns, result, report_type):
+	custom_column_labels = [col["label"] for col in custom_columns]
+
+	if report_type == 'Query Report':
+		original_column_labels = [col.split(":")[0] for col in columns]
+	else:
+		original_column_labels = [col["label"] for col in columns]
+
 	reordered_result = []
-	columns = [col.split(":")[0] for col in columns]
 
 	for res in result:
 		r = []
-		for col in custom_columns:
+		for col_name in custom_column_labels:
 			try:
-				idx = columns.index(col.get("label"))
+				idx = original_column_labels.index(col_name)
 				r.append(res[idx])
 			except ValueError:
 				pass

From 476e625261ab4954b7429978b24840e52967c9c0 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 16 Jul 2020 16:26:20 +0530
Subject: [PATCH 020/361] fix: Add site validation for logger

---
 frappe/__init__.py     | 4 ++--
 frappe/app.py          | 5 +++--
 frappe/utils/logger.py | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index d644d2a473..8c24980189 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1559,10 +1559,10 @@ def get_doctype_app(doctype):
 
 loggers = {}
 log_level = None
-def logger(module=None, with_more_info=False):
+def logger(module=None, with_more_info=False, _site=None):
 	'''Returns a python logger that uses StreamHandler'''
 	from frappe.utils.logger import get_logger
-	return get_logger(module=module, with_more_info=with_more_info)
+	return get_logger(module=module, with_more_info=with_more_info, _site=_site)
 
 def log_error(message=None, title=_("Error")):
 	'''Log error to Error Log'''
diff --git a/frappe/app.py b/frappe/app.py
index 57db867882..83a9b8c149 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -99,8 +99,9 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		frappe.logger("frappe.web").info({
-			"site": get_site_name(request.host),
+		_site = get_site_name(request.host)
+		frappe.logger("frappe.web", _site=_site).info({
+			"site": _site,
 			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
 			"base_url": getattr(request, "base_url", "NOTFOUND"),
 			"full_path": getattr(request, "full_path", "NOTFOUND"),
diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 89e3711b0f..bea83297d2 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -17,7 +17,7 @@ default_log_level = logging.DEBUG
 site = getattr(frappe.local, 'site', None)
 
 
-def get_logger(module, with_more_info=False):
+def get_logger(module, with_more_info=False, _site=None):
 	global site
 	if module in frappe.loggers:
 		return frappe.loggers[module]
@@ -38,7 +38,7 @@ def get_logger(module, with_more_info=False):
 	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
 	logger.addHandler(handler)
 #
-	if site:
+	if site == _site:
 		SITELOG_FILENAME = os.path.join(site, 'logs', logfile)
 		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
 		site_handler.setFormatter(formatter)

From e92a612ba231eebb8dbe7ac42d24ac002a89fbe1 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 16 Jul 2020 16:26:53 +0530
Subject: [PATCH 021/361] docs(get_logger): add docstring

---
 frappe/utils/logger.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index bea83297d2..afe4335aea 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -18,7 +18,24 @@ site = getattr(frappe.local, 'site', None)
 
 
 def get_logger(module, with_more_info=False, _site=None):
+	"""Application Logger for your given module
+
+	Args:
+		module (str): Name of your logger and consequently your log file.
+		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
+		_site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None.
+
+	Returns:
+		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
+	"""
 	global site
+
+	def allow_site():
+		allow = False
+		if site: allow = True
+		if _site: allow = site == _site
+		return allow
+
 	if module in frappe.loggers:
 		return frappe.loggers[module]
 
@@ -37,8 +54,8 @@ def get_logger(module, with_more_info=False, _site=None):
 	formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s')
 	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
 	logger.addHandler(handler)
-#
-	if site == _site:
+
+	if allow_site():
 		SITELOG_FILENAME = os.path.join(site, 'logs', logfile)
 		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
 		site_handler.setFormatter(formatter)

From c87d598480e006a88a4374d0ab17d15429701441 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 17 Jul 2020 17:35:08 +0530
Subject: [PATCH 022/361] refactor: less word do trick

Co-authored-by: Chinmay Pai <chinmaydpai@gmail.com>
---
 frappe/utils/logger.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index afe4335aea..9254c0aedd 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -31,10 +31,7 @@ def get_logger(module, with_more_info=False, _site=None):
 	global site
 
 	def allow_site():
-		allow = False
-		if site: allow = True
-		if _site: allow = site == _site
-		return allow
+		return (_site and _site == site) or bool(site)
 
 	if module in frappe.loggers:
 		return frappe.loggers[module]

From 0ae7d40ebe9268e9f08928f475bda7a920aa52db Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Mon, 20 Jul 2020 23:08:50 +0530
Subject: [PATCH 023/361] fix(paytm-integration): use checksum library to
 generate/verify checksum

---
 .../doctype/paytm_settings/checksum.py        | 80 -------------------
 .../doctype/paytm_settings/paytm_settings.py  |  2 +-
 requirements.txt                              |  2 +-
 3 files changed, 2 insertions(+), 82 deletions(-)
 delete mode 100644 frappe/integrations/doctype/paytm_settings/checksum.py

diff --git a/frappe/integrations/doctype/paytm_settings/checksum.py b/frappe/integrations/doctype/paytm_settings/checksum.py
deleted file mode 100644
index 32f976ae18..0000000000
--- a/frappe/integrations/doctype/paytm_settings/checksum.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import base64
-import string
-import random
-import hashlib
-import sys
-
-from Crypto.Cipher import AES
-
-
-iv = '@@@@&&&&####$$$$'
-BLOCK_SIZE = 16
-
-if (sys.version_info > (3, 0)):
-	__pad__ = lambda s: bytes(s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE), 'utf-8')
-else:
-	__pad__ = lambda s: s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * chr(BLOCK_SIZE - len(s) % BLOCK_SIZE)
-
-__unpad__ = lambda s: s[0:-ord(s[-1])]
-
-def encrypt(input, key):
-	input = __pad__(input)
-	c = AES.new(key.encode("utf8"), AES.MODE_CBC, iv.encode("utf8"))
-	input = c.encrypt(input)
-	input = base64.b64encode(input)
-	return input.decode("UTF-8")
-
-def decrypt(encrypted, key):
-	encrypted = base64.b64decode(encrypted)
-	c = AES.new(key.encode("utf8"), AES.MODE_CBC, iv.encode("utf8"))
-	param = c.decrypt(encrypted)
-	if type(param) == bytes:
-		param = param.decode()
-	return __unpad__(param)
-
-def generateSignature(params, key):
-	if not type(params) is dict and not type(params) is str:
-		raise Exception("string or dict expected, " + str(type(params)) + " given")
-	if type(params) is dict:
-		params = getStringByParams(params)
-	return generateSignatureByString(params, key)
-
-def verifySignature(params, key, checksum):
-	if not type(params) is dict and not type(params) is str:
-		raise Exception("string or dict expected, " + str(type(params)) + " given")
-	if "CHECKSUMHASH" in params:
-		del params["CHECKSUMHASH"]
-		
-	if type(params) is dict:
-		params = getStringByParams(params)
-	return verifySignatureByString(params, key, checksum)
-
-def generateSignatureByString(params, key):
-	salt = generateRandomString(4)
-	return calculateChecksum(params, key, salt)
-
-def verifySignatureByString(params, key, checksum):
-	paytm_hash = decrypt(checksum, key)    
-	salt = paytm_hash[-4:]
-	return paytm_hash == calculateHash(params, salt)
-
-def generateRandomString(length):
-	chars = string.ascii_uppercase + string.digits + string.ascii_lowercase
-	return ''.join(random.choice(chars) for _ in range(length))
-
-def getStringByParams(params):
-	params_string = []
-	for key in sorted(params.keys()):
-		value = params[key] if params[key] is not None and params[key].lower() != "null" else ""
-		params_string.append(str(value))
-	return '|'.join(params_string)
-
-def calculateHash(params, salt):
-	finalString = '%s|%s' % (params, salt)
-	hasher = hashlib.sha256(finalString.encode())
-	hashString = hasher.hexdigest() + salt
-	return hashString
-
-def calculateChecksum(params, key, salt): 
-	hashString = calculateHash(params, salt)
-	return encrypt(hashString, key)
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index c169a53246..bfa9b6b3eb 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -13,7 +13,7 @@ from frappe import _
 from frappe.utils import get_url, call_hook_method, cint, flt, cstr
 from frappe.integrations.utils import create_request_log, create_payment_gateway
 from frappe.utils import get_request_site_address
-from frappe.integrations.doctype.paytm_settings.checksum import generateSignature, verifySignature
+from paytmchecksum import generateSignature, verifySignature
 from frappe.utils.password import get_decrypted_password
 
 class PaytmSettings(Document):
diff --git a/requirements.txt b/requirements.txt
index e6f6f9fff9..d053f8f48c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -66,5 +66,5 @@ watchdog==0.8.0
 Werkzeug==0.16.1
 xlrd==1.2.0
 zxcvbn-python==4.4.24
-pycryptodome==3.9.7
 Whoosh==2.7.4
+paytmchecksum==1.7.0
\ No newline at end of file

From 158da07c6ab6b125086d60f83082d8d208272d8f Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Tue, 21 Jul 2020 00:12:06 +0530
Subject: [PATCH 024/361] fix: make the checkout page responsive for mobile
 view

---
 .../pages/integrations/paytm_checkout.html       | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/frappe/templates/pages/integrations/paytm_checkout.html b/frappe/templates/pages/integrations/paytm_checkout.html
index 77a2045edf..168f6597e5 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.html
+++ b/frappe/templates/pages/integrations/paytm_checkout.html
@@ -16,8 +16,9 @@
 
 {%- block page_content -%}
 <body>
-	<div style="margin: 30vh;">
-		<center><h2>Please do not refresh this page...</h2></center>
+	<div class="centered">
+		<h2>Please do not refresh this page...</h2>
+
 		<form method="post" action="{{ url }}" name="paytm_form">
 			{% for name, value in payment_details.items() %}
 				<input type="hidden" name="{{ name }}" value="{{ value }}">
@@ -29,9 +30,14 @@
 
 {% block style %}
 <style>
-	/* header,
-	footer {
+	.centered {
+		position: absolute;
+		top: 50%;
+		left: 50%;
+		transform: translate(-50%, -50%);
+	}
+	.web-footer {
 		display: none;
-	} */
+	}
 </style>
 {% endblock %}
\ No newline at end of file

From 1d9903bfdd51a42dae0034c1a0ceecde4b7fe5f2 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Tue, 21 Jul 2020 20:45:27 +0530
Subject: [PATCH 025/361] fix: get value from filter lsit if base list doesn't
 exist

---
 frappe/public/js/frappe/ui/filters/filter.js      | 4 +++-
 frappe/public/js/frappe/ui/filters/filter_list.js | 7 ++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js
index 5e41ed645e..d4fb6db436 100644
--- a/frappe/public/js/frappe/ui/filters/filter.js
+++ b/frappe/public/js/frappe/ui/filters/filter.js
@@ -249,7 +249,9 @@ frappe.ui.Filter = class {
 			let args = {};
 			if (this.filters_config[condition].depends_on) {
 				const field_name = this.filters_config[condition].depends_on;
-				const filter_value = this.base_list.get_filter_value(field_name);
+				const filter_value = this.base_list
+					? this.base_list.get_filter_value(field_name)
+					: this.filter_list.get_filter_value(field_name);
 				args[field_name] = filter_value;
 			}
 			frappe
diff --git a/frappe/public/js/frappe/ui/filters/filter_list.js b/frappe/public/js/frappe/ui/filters/filter_list.js
index ed9ddefe64..7ae48bc7d9 100644
--- a/frappe/public/js/frappe/ui/filters/filter_list.js
+++ b/frappe/public/js/frappe/ui/filters/filter_list.js
@@ -104,13 +104,18 @@ frappe.ui.FilterGroup = class {
 			filter_items: (doctype, fieldname) => {
 				return !this.filter_exists([doctype, fieldname]);
 			},
-			base_list: this.base_list
+			base_list: this.base_list,
+			filter_list: this,
 		};
 		let filter = new frappe.ui.Filter(args);
 		this.filters.push(filter);
 		return filter;
 	}
 
+	get_filter_value(fieldname) {
+		return this.filters.filter(f => f.fieldname == fieldname)[0].value;
+	}
+
 	filter_exists(filter_value) {
 		// filter_value of form: [doctype, fieldname, condition, value]
 		let exists = false;

From 3c4e1ea3a5fabb62f465a432760044b6db7529d2 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Tue, 21 Jul 2020 22:20:27 +0530
Subject: [PATCH 026/361] fix: add pycryptodome for checksum generation

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0c57da9eac..33cd292bf6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -68,4 +68,5 @@ Werkzeug==0.16.1
 Whoosh==2.7.4
 xlrd==1.2.0
 zxcvbn-python==4.4.24
-paytmchecksum==1.7.0
+pycryptodome==3.9.8
+paytmchecksum==1.7.0
\ No newline at end of file

From 1e56fcc85765681c79631116dd48bd13c1ff3368 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:40:28 +0530
Subject: [PATCH 027/361] fix: filter out standard dashboards if not in
 developer mode

---
 .../dashboard_chart/dashboard_chart.js        | 36 +++-----------
 .../dashboard_chart/dashboard_chart.py        |  8 ++-
 .../desk/doctype/number_card/number_card.js   | 36 +++-----------
 .../desk/doctype/number_card/number_card.py   |  6 +++
 .../public/js/frappe/utils/dashboard_utils.js | 49 +++++++++++++++++++
 5 files changed, 75 insertions(+), 60 deletions(-)

diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
index ee6eb3289d..65db4839a1 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
@@ -28,38 +28,16 @@ frappe.ui.form.on('Dashboard Chart', {
 		}
 
 		frm.add_custom_button('Add Chart to Dashboard', () => {
-			const d = new frappe.ui.Dialog({
-				title: __('Add to Dashboard'),
-				fields: [
-					{
-						label: __('Select Dashboard'),
-						fieldtype: 'Link',
-						fieldname: 'dashboard',
-						options: 'Dashboard',
-					}
-				],
-				primary_action: (values) => {
-					values.chart_name = frm.doc.chart_name;
-					frappe.xcall(
-						'frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard',
-						{args: values}
-					).then(()=> {
-						let dashboard_route_html =
-							`<a href = "#dashboard/${values.dashboard}">${values.dashboard}</a>`;
-						let message =
-							__(`Dashboard Chart ${values.chart_name} add to Dashboard ` + dashboard_route_html);
-
-						frappe.msgprint(message);
-					});
-
-					d.hide();
-				}
-			});
+			const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog(
+				frm.doc.name,
+				'Dashboard Chart',
+				'frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard'
+			);
 
 			if (!frm.doc.chart_name) {
 				frappe.msgprint(__('Please create chart first'));
 			} else {
-				d.show();
+				dialog.show();
 			}
 		});
 
@@ -338,7 +316,7 @@ frappe.ui.form.on('Dashboard Chart', {
 		} else if (frm.chart_filters.length) {
 			fields = frm.chart_filters.filter(f => f.fieldname);
 
-			fields.map( f => {
+			fields.map(f => {
 				if (filters[f.fieldname]) {
 					let condition = '=';
 					const filter_row =
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index 70aece3ee7..a25e0cab9e 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -128,7 +128,13 @@ def add_chart_to_dashboard(args):
 
 	dashboard = frappe.get_doc('Dashboard', args.dashboard)
 	dashboard_link = frappe.new_doc('Dashboard Chart Link')
-	dashboard_link.chart = args.chart_name
+	dashboard_link.chart = args.chart_name or args.name
+
+	if args.set_standard:
+		chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart)
+		chart.is_standard = 1
+		chart.module = dashboard.module
+		chart.save()
 
 	dashboard.append('charts', dashboard_link)
 	dashboard.save()
diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js
index 42899a4ebb..2ae163d7aa 100644
--- a/frappe/desk/doctype/number_card/number_card.js
+++ b/frappe/desk/doctype/number_card/number_card.js
@@ -32,40 +32,16 @@ frappe.ui.form.on('Number Card', {
 
 	create_add_to_dashboard_button: function(frm) {
 		frm.add_custom_button('Add Card to Dashboard', () => {
-			const d = new frappe.ui.Dialog({
-				title: __('Add to Dashboard'),
-				fields: [
-					{
-						label: __('Select Dashboard'),
-						fieldtype: 'Link',
-						fieldname: 'dashboard',
-						options: 'Dashboard',
-					}
-				],
-				primary_action: (values) => {
-					values.name = frm.doc.name;
-					frappe.xcall(
-						'frappe.desk.doctype.number_card.number_card.add_card_to_dashboard',
-						{
-							args: values
-						}
-					).then(()=> {
-						let dashboard_route_html =
-							`<a href = "#dashboard/${values.dashboard}">${values.dashboard}</a>`;
-						let message =
-							__(`Number Card ${values.name} add to Dashboard ` + dashboard_route_html);
-
-						frappe.msgprint(message);
-					});
-
-					d.hide();
-				}
-			});
+			const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog(
+				frm.doc.name,
+				'Number Card',
+				'frappe.desk.doctype.number_card.number_card.add_card_to_dashboard'
+			);
 
 			if (!frm.doc.name) {
 				frappe.msgprint(__('Please create Card first'));
 			} else {
-				d.show();
+				dialog.show();
 			}
 		});
 	},
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 5b52b60474..3eb08ec00a 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -172,5 +172,11 @@ def add_card_to_dashboard(args):
 	dashboard_link = frappe.new_doc('Number Card Link')
 	dashboard_link.card = args.name
 
+	if args.set_standard:
+		card = frappe.get_doc('Number Card', dashboard_link.card)
+		card.is_standard = 1
+		card.module = dashboard.module
+		card.save()
+
 	dashboard.append('cards', dashboard_link)
 	dashboard.save()
\ No newline at end of file
diff --git a/frappe/public/js/frappe/utils/dashboard_utils.js b/frappe/public/js/frappe/utils/dashboard_utils.js
index 8618f6dd59..4074404aa3 100644
--- a/frappe/public/js/frappe/utils/dashboard_utils.js
+++ b/frappe/public/js/frappe/utils/dashboard_utils.js
@@ -199,6 +199,55 @@ frappe.dashboard_utils = {
 		}
 
 		return filters;
+	},
+
+	get_dashboard_link_field() {
+		let field = {
+			label: __('Select Dashboard'),
+			fieldtype: 'Link',
+			fieldname: 'dashboard',
+			options: 'Dashboard',
+		}
+
+		if (!frappe.boot.developer_mode) {
+			field.get_query = () => {
+				return {
+					filters: {
+						is_standard: 0
+					}
+				};
+			};
+		}
+
+		return field;
+	},
+
+	get_add_to_dashboard_dialog(docname, doctype, method) {
+		const field = this.get_dashboard_link_field();
+
+		const dialog = new frappe.ui.Dialog({
+			title: __('Add to Dashboard'),
+			fields: [field],
+			primary_action: (values) => {
+				values.name = docname;
+				values.set_standard = frappe.boot.developer_mode;
+				frappe.xcall(
+					method,
+					{args: values}
+				).then(()=> {
+					let dashboard_route_html =
+						`<a href = "#dashboard/${values.dashboard}">${values.dashboard}</a>`;
+					let message =
+						__(`${doctype} ${values.name} added to Dashboard ` + dashboard_route_html);
+
+					frappe.msgprint(message);
+				});
+
+				dialog.hide();
+			}
+		});
+
+		return dialog;
 	}
 
 };
\ No newline at end of file

From cdd7cb2f177492e24423f3ac15bb575bbf94e22a Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:42:13 +0530
Subject: [PATCH 028/361] fix: filter out standard dashboards while creating
 from reports

---
 .../js/frappe/views/reports/query_report.js   | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 9b094149b8..777be2aaa1 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -157,6 +157,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 	add_card_to_dashboard() {
 		let field_options = frappe.report_utils.get_field_options_from_report(this.columns, this.raw_data);
+		const dashboard_field = frappe.dashboard_utils.get_dashboard_link_field();
+		const set_standard = frappe.boot.developer_mode;
+
 		const dialog = new frappe.ui.Dialog({
 			title: __('Create Card'),
 			fields: [
@@ -181,12 +184,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 					label: __('Add to Dashboard'),
 					fieldtype: 'Section Break'
 				},
-				{
-					fieldname: 'dashboard',
-					label: __('Choose Dashboard'),
-					fieldtype: 'Link',
-					options: 'Dashboard',
-				},
+				dashboard_field,
 				{
 					fieldname: 'cb_2',
 					fieldtype: 'Column Break'
@@ -202,7 +200,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				if (!values.label) {
 					values.label = `${values.report_function} of ${toTitle(values.report_field)}`;
 				}
-				this.create_number_card(values, values.dashboard, values.label);
+				this.create_number_card(values, values.dashboard, values.label, set_standard);
 				dialog.hide();
 			}
 		});
@@ -213,27 +211,26 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 	add_chart_to_dashboard() {
 		if (this.chart_fields || this.chart_options) {
+			const dashboard_field = frappe.dashboard_utils.get_dashboard_link_field();
+			const set_standard = frappe.boot.developer_mode;
+
 			const dialog = new frappe.ui.Dialog({
 				title: __('Create Chart'),
 				fields: [
-					{
-						fieldname: 'dashboard',
-						label: 'Choose Dashboard',
-						fieldtype: 'Link',
-						options: 'Dashboard',
-					},
 					{
 						fieldname: 'dashboard_chart_name',
 						label: 'Chart Name',
 						fieldtype: 'Data',
-					}
+					},
+					dashboard_field,
 				],
 				primary_action_label: __('Add'),
 				primary_action: (values) => {
 					this.create_dashboard_chart(
 						this.chart_fields || this.chart_options,
 						values.dashboard,
-						values.dashboard_chart_name
+						values.dashboard_chart_name,
+						set_standard
 					);
 					dialog.hide();
 				}
@@ -245,12 +242,13 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		}
 	}
 
-	create_number_card(values, dashboard_name, card_name) {
+	create_number_card(values, dashboard_name, card_name, set_standard) {
 		let args = {
 			'dashboard': dashboard_name || null,
 			'type': 'Report',
 			'report_name': this.report_name,
 			'filters_json': JSON.stringify(this.get_filter_values()),
+			set_standard: set_standard,
 		};
 		Object.assign(args, values);
 
@@ -263,7 +261,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		);
 	}
 
-	create_dashboard_chart(chart_args, dashboard_name, chart_name) {
+	create_dashboard_chart(chart_args, dashboard_name, chart_name, set_standard) {
 		let args = {
 			'dashboard': dashboard_name || null,
 			'chart_type': 'Report',
@@ -271,7 +269,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			'type': chart_args.chart_type || frappe.model.unscrub(chart_args.type),
 			'color': chart_args.color,
 			'filters_json': JSON.stringify(this.get_filter_values()),
-			'custom_options': {}
+			'custom_options': {},
+			'set_standard': set_standard,
 		};
 
 		for (let key in chart_args) {

From 7f1a5adb0e221452019076aa05b3dafae7d153e0 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:43:37 +0530
Subject: [PATCH 029/361] fix: disable dashboard form only if doc is standard

---
 frappe/desk/doctype/dashboard/dashboard.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/desk/doctype/dashboard/dashboard.js b/frappe/desk/doctype/dashboard/dashboard.js
index 237b549433..61300b920b 100644
--- a/frappe/desk/doctype/dashboard/dashboard.js
+++ b/frappe/desk/doctype/dashboard/dashboard.js
@@ -5,7 +5,7 @@ frappe.ui.form.on('Dashboard', {
 	refresh: function(frm) {
 		frm.add_custom_button(__("Show Dashboard"), () => frappe.set_route('dashboard', frm.doc.name));
 
-		if (!frappe.boot.developer_mode) {
+		if (!frappe.boot.developer_mode && frm.doc.is_standard) {
 			frm.disable_form();
 		}
 

From dd6a361058e425ec03d162f8e4f88a3251f5537a Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:44:21 +0530
Subject: [PATCH 030/361] fix: don't show is_standard if not in developer mode

---
 frappe/desk/doctype/dashboard/dashboard.json             | 3 ++-
 frappe/desk/doctype/dashboard_chart/dashboard_chart.json | 3 ++-
 frappe/desk/doctype/number_card/number_card.json         | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/frappe/desk/doctype/dashboard/dashboard.json b/frappe/desk/doctype/dashboard/dashboard.json
index c7128823fe..0c54f1ee7f 100644
--- a/frappe/desk/doctype/dashboard/dashboard.json
+++ b/frappe/desk/doctype/dashboard/dashboard.json
@@ -52,6 +52,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
    "label": "Is Standard"
@@ -66,7 +67,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-10 17:48:19.468813",
+ "modified": "2020-07-22 11:23:37.858568",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard",
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
index a25d0cc6c5..f0616dddd0 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
@@ -238,6 +238,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
    "label": "Is Standard"
@@ -270,7 +271,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-21 16:37:07.763482",
+ "modified": "2020-07-22 11:32:51.185499",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard Chart",
diff --git a/frappe/desk/doctype/number_card/number_card.json b/frappe/desk/doctype/number_card/number_card.json
index 333885a7f9..0d1cc279c8 100644
--- a/frappe/desk/doctype/number_card/number_card.json
+++ b/frappe/desk/doctype/number_card/number_card.json
@@ -113,6 +113,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
    "label": "Is Standard"
@@ -190,7 +191,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-18 17:08:22.882538",
+ "modified": "2020-07-22 11:26:32.039023",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Number Card",

From 1bd610f858a29a0d32681d175b418bc675a2817d Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:51:32 +0530
Subject: [PATCH 031/361] fix: separate patch to rename is_custom field

---
 frappe/patches.txt                                    |  1 +
 .../create_custom_dashboards_cards_and_charts.py      |  5 -----
 .../rename_is_custom_field_in_dashboard_chart.py      | 11 +++++++++++
 3 files changed, 12 insertions(+), 5 deletions(-)
 create mode 100644 frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py

diff --git a/frappe/patches.txt b/frappe/patches.txt
index f8c767f5a3..b207a325cb 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -295,3 +295,4 @@ frappe.patches.v13_0.update_date_filters_in_user_settings
 frappe.patches.v13_0.update_duration_options
 frappe.patches.v13_0.replace_old_data_import # 2020-06-24
 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts
+frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
diff --git a/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py b/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py
index f61941bf01..9a075a22cc 100644
--- a/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py
+++ b/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py
@@ -1,7 +1,6 @@
 import frappe
 from frappe.model.naming import append_number_if_name_exists
 from frappe.utils.dashboard import get_dashboards_with_link
-from frappe.model.utils.rename_field import rename_field
 
 def execute():
 	if not frappe.db.table_exists('Dashboard Chart')\
@@ -9,14 +8,10 @@ def execute():
 		or not frappe.db.table_exists('Dashboard'):
 		return
 
-
 	frappe.reload_doc('desk', 'doctype', 'dashboard_chart')
 	frappe.reload_doc('desk', 'doctype', 'number_card')
 	frappe.reload_doc('desk', 'doctype', 'dashboard')
 
-	if frappe.db.has_column('Dashboard Chart', 'is_custom'):
-		rename_field('Dashboard Chart', 'is_custom', 'use_report_chart')
-
 	modified_charts = get_modified_docs('Dashboard Chart')
 	modified_cards = get_modified_docs('Number Card')
 	modified_dashboards = [doc.name for doc in get_modified_docs('Dashboard')]
diff --git a/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py
new file mode 100644
index 0000000000..90860e211b
--- /dev/null
+++ b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py
@@ -0,0 +1,11 @@
+import frappe
+from frappe.model.utils.rename_field import rename_field
+
+def execute():
+	if not frappe.db.table_exists('Dashboard Chart')
+		return
+
+	frappe.reload_doc('desk', 'doctype', 'dashboard_chart')
+
+	if frappe.db.has_column('Dashboard Chart', 'is_custom'):
+		rename_field('Dashboard Chart', 'is_custom', 'use_report_chart')
\ No newline at end of file

From c25100cb47260f2e898f85cd820b952735cd97e7 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 11:59:17 +0530
Subject: [PATCH 032/361] style: fix formatting

---
 .../patches/v13_0/rename_is_custom_field_in_dashboard_chart.py  | 2 +-
 frappe/public/js/frappe/utils/dashboard_utils.js                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py
index 90860e211b..4da0f8164a 100644
--- a/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py
+++ b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py
@@ -2,7 +2,7 @@ import frappe
 from frappe.model.utils.rename_field import rename_field
 
 def execute():
-	if not frappe.db.table_exists('Dashboard Chart')
+	if not frappe.db.table_exists('Dashboard Chart'):
 		return
 
 	frappe.reload_doc('desk', 'doctype', 'dashboard_chart')
diff --git a/frappe/public/js/frappe/utils/dashboard_utils.js b/frappe/public/js/frappe/utils/dashboard_utils.js
index 4074404aa3..fca26d6ece 100644
--- a/frappe/public/js/frappe/utils/dashboard_utils.js
+++ b/frappe/public/js/frappe/utils/dashboard_utils.js
@@ -207,7 +207,7 @@ frappe.dashboard_utils = {
 			fieldtype: 'Link',
 			fieldname: 'dashboard',
 			options: 'Dashboard',
-		}
+		};
 
 		if (!frappe.boot.developer_mode) {
 			field.get_query = () => {

From 95aa4a72dcd81b98a9f318b35979964bdedbc032 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 14:53:03 +0530
Subject: [PATCH 033/361] fix: reduce code complexity

---
 frappe/public/js/frappe/ui/filters/filter.js      | 8 +++++---
 frappe/public/js/frappe/ui/filters/filter_list.js | 3 +--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js
index d4fb6db436..7c6d532f5c 100644
--- a/frappe/public/js/frappe/ui/filters/filter.js
+++ b/frappe/public/js/frappe/ui/filters/filter.js
@@ -249,9 +249,7 @@ frappe.ui.Filter = class {
 			let args = {};
 			if (this.filters_config[condition].depends_on) {
 				const field_name = this.filters_config[condition].depends_on;
-				const filter_value = this.base_list
-					? this.base_list.get_filter_value(field_name)
-					: this.filter_list.get_filter_value(field_name);
+				const filter_value = this.get_filter_value(field_name);
 				args[field_name] = filter_value;
 			}
 			frappe
@@ -267,6 +265,10 @@ frappe.ui.Filter = class {
 		}
 	}
 
+	get_filter_value(fieldname) {
+		return this.filter_list.get_filter_value(fieldname);
+	}
+
 	make_field(df, old_fieldtype) {
 		let old_text = this.field ? this.field.get_value() : null;
 		this.hide_invalid_conditions(df.fieldtype, df.original_type);
diff --git a/frappe/public/js/frappe/ui/filters/filter_list.js b/frappe/public/js/frappe/ui/filters/filter_list.js
index 7ae48bc7d9..9a35a9bb5f 100644
--- a/frappe/public/js/frappe/ui/filters/filter_list.js
+++ b/frappe/public/js/frappe/ui/filters/filter_list.js
@@ -104,8 +104,7 @@ frappe.ui.FilterGroup = class {
 			filter_items: (doctype, fieldname) => {
 				return !this.filter_exists([doctype, fieldname]);
 			},
-			base_list: this.base_list,
-			filter_list: this,
+			filter_list: this.base_list || this,
 		};
 		let filter = new frappe.ui.Filter(args);
 		this.filters.push(filter);

From 0c475792cd3f1860af674c9f0073855de4753665 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 14:55:02 +0530
Subject: [PATCH 034/361] refactor: delete unused file filters.js

---
 frappe/public/js/frappe/ui/filters/filters.js | 684 ------------------
 1 file changed, 684 deletions(-)
 delete mode 100644 frappe/public/js/frappe/ui/filters/filters.js

diff --git a/frappe/public/js/frappe/ui/filters/filters.js b/frappe/public/js/frappe/ui/filters/filters.js
deleted file mode 100644
index a775413d39..0000000000
--- a/frappe/public/js/frappe/ui/filters/filters.js
+++ /dev/null
@@ -1,684 +0,0 @@
-// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-// MIT License. See license.txt
-
-frappe.ui.FilterList = Class.extend({
-	init: function(opts) {
-		$.extend(this, opts);
-		this.filters = [];
-		this.wrapper = this.parent;
-		this.stats = [];
-		this.make();
-		this.set_events();
-	},
-	make: function() {
-		this.wrapper.find('.show_filters, .filter_area').remove();
-		this.wrapper.append(`
-			<div class="show_filters">
-				<div class="set-filters">
-					<button
-						style="margin-right: 10px;"
-						class="btn btn-default btn-xs new-filter text-muted">
-						${__("Add Filter")}</button>
-				</div>
-			</div>
-			<div class="filter_area"></div>`);
-	},
-	set_events: function() {
-		var me = this;
-		// show filters
-		this.wrapper.find('.new-filter').bind('click', function() {
-			me.add_filter();
-		});
-
-		this.wrapper.find('.clear-filters').bind('click', function() {
-			me.clear_filters();
-			$('.date-range-picker').val('')
-			me.base_list.run();
-			$(this).addClass("hide");
-		});
-	},
-
-	show_filters: function() {
-		this.wrapper.find('.show_filters').toggle();
-		if(!this.filters.length) {
-			this.add_filter(this.doctype, 'name');
-			this.filters[0].wrapper.find(".filter_field input").focus();
-		}
-	},
-
-	clear_filters: function() {
-		$.each(this.filters, function(i, f) { f.remove(true); });
-		if(this.base_list.page.fields_dict) {
-			$.each(this.base_list.page.fields_dict, (key, value) => {
-				value.set_input('');
-			});
-		}
-		this.filters = [];
-	},
-
-	add_filter: function(doctype, fieldname, condition, value, hidden) {
-		// adds a new filter, returns true if filter has been added
-
-		// allow equal to be used as like
-		let base_filter = this.base_list.page.fields_dict[fieldname];
-		if (base_filter
-			&& (base_filter.df.condition==condition
-				|| (condition==='=' && base_filter.df.condition==='like'))) {
-			// if filter exists in base_list, then exit
-			this.base_list.page.fields_dict[fieldname].set_input(value);
-
-			return true;
-		}
-
-		if(doctype && fieldname
-			&& !frappe.meta.has_field(doctype, fieldname)
-			&& !in_list(frappe.model.std_fields_list, fieldname)) {
-			frappe.msgprint({
-				message: __('Filter {0} missing', [fieldname.bold()]),
-				title: 'Invalid Filter',
-				indicator: 'red'
-			});
-			return false;
-		}
-
-		this.wrapper.find('.show_filters').toggle(true);
-		var is_new_filter = arguments.length===0;
-
-		if (is_new_filter && this.wrapper.find(".is-new-filter:visible").length) {
-			// only allow 1 new filter at a time!
-			return false;
-		}
-
-		var filter = this.push_new_filter(doctype, fieldname, condition, value);
-		if (!filter) return;
-
-		if(this.wrapper.find('.clear-filters').hasClass("hide")) {
-			this.wrapper.find('.clear-filters').removeClass("hide");
-		}
-
-		if (filter && is_new_filter) {
-			filter.wrapper.addClass("is-new-filter");
-		} else {
-			filter.freeze();
-		}
-
-		if (hidden) {
-			filter.$btn_group.addClass("hide");
-		}
-
-		return true;
-	},
-	push_new_filter: function(doctype, fieldname, condition, value) {
-		if(this.filter_exists(doctype, fieldname, condition, value)) {
-			return;
-		}
-
-		// if standard filter exists, then clear it.
-		if(this.base_list.page.fields_dict[fieldname]) {
-			this.base_list.page.fields_dict[fieldname].set_input('');
-		}
-
-		var filter = new frappe.ui.Filter({
-			flist: this,
-			_doctype: doctype,
-			fieldname: fieldname,
-			condition: condition,
-			value: value
-		});
-
-		this.filters.push(filter);
-
-		return filter;
-	},
-
-	remove: function(filter) {
-		// remove `filter` from flist
-		for (var i in this.filters) {
-			if (this.filters[i] === filter) {
-				break;
-			}
-		}
-		if (i!==undefined) {
-			// remove index
-			this.filters.splice(i, 1);
-		}
-	},
-
-	filter_exists: function(doctype, fieldname, condition, value) {
-		var flag = false;
-		for(var i in this.filters) {
-			if(this.filters[i].field) {
-				var f = this.filters[i].get_value();
-
-				if(f[0]==doctype && f[1]==fieldname && f[2]==condition && f[3]==value) {
-					flag = true;
-				} else if($.isArray(value) && frappe.utils.arrays_equal(value, f[3])) {
-					flag = true;
-				}
-			}
-		}
-		return flag;
-	},
-
-	get_filters: function() {
-		// get filter values as dict
-		var values = [];
-		$.each(this.filters, function(i, filter) {
-			if(filter.field) {
-				filter.freeze();
-				values.push(filter.get_value());
-			}
-		});
-		this.base_list.update_standard_filters(values);
-
-		return values;
-	},
-
-	// remove hidden filters
-	update_filters: function() {
-		var fl = [];
-		$.each(this.filters, function(i, f) {
-			if(f.field) fl.push(f);
-		})
-		this.filters = fl;
-		if(this.filters.length === 0) {
-			this.wrapper.find('.clear-filters').addClass("hide");
-		}
-	},
-
-	get_filter: function(fieldname) {
-		for(var i in this.filters) {
-			if(this.filters[i].field && this.filters[i].field.df.fieldname==fieldname)
-				return this.filters[i];
-		}
-	},
-
-	get_formatted_value: function(field, val){
-		var value = val;
-
-		if(field.df.fieldname==="docstatus") {
-			value = {0:"Draft", 1:"Submitted", 2:"Cancelled"}[value] || value;
-		} else if(field.df.original_type==="Check") {
-			value = {0:"No", 1:"Yes"}[cint(value)];
-		} else if (field.df.original_type === "Duration") {
-			let duration_options = {
-				hide_days: field.df.hide_days,
-				hide_seconds: field.df.hide_seconds
-			};
-			value = frappe.utils.get_formatted_duration(value, duration_options);
-		}
-
-		value = frappe.format(value, field.df, {only_value: 1});
-		return value;
-	}
-});
-
-frappe.ui.Filter = Class.extend({
-	init: function(opts) {
-		$.extend(this, opts);
-
-		this.doctype = this.flist.doctype;
-		this.make();
-		this.make_select();
-		this.set_events();
-	},
-	make: function() {
-		this.wrapper = $(frappe.render_template("edit_filter", {}))
-			.appendTo(this.flist.wrapper.find('.filter_area'));
-	},
-	make_select: function() {
-		var me = this;
-		this.fieldselect = new frappe.ui.FieldSelect({
-			parent: this.wrapper.find('.fieldname_select_area'),
-			doctype: this.doctype,
-			filter_fields: this.filter_fields,
-			select: function(doctype, fieldname) {
-				me.set_field(doctype, fieldname);
-			}
-		});
-		if(this.fieldname) {
-			this.fieldselect.set_value(this._doctype || this.doctype, this.fieldname);
-		}
-	},
-	set_events: function() {
-		var me = this;
-
-		this.wrapper.find("a.remove-filter").on("click", function() {
-			me.remove();
-		});
-
-		this.wrapper.find(".set-filter-and-run").on("click", function() {
-			me.wrapper.removeClass("is-new-filter");
-			me.flist.base_list.run();
-			me.apply();
-		});
-
-		// add help for "in" codition
-		me.wrapper.find('.condition').change(function() {
-			if(!me.field) return;
-			var condition = $(this).val();
-			if(in_list(["in", "like", "not in", "not like"], condition)) {
-				me.set_field(me.field.df.parent, me.field.df.fieldname, 'Data', condition);
-				if(!me.field.desc_area) {
-					me.field.desc_area = $('<div class="text-muted small">').appendTo(me.field.wrapper);
-				}
-				// set description
-				me.field.desc_area.html((in_list(["in", "not in"], condition)==="in"
-					? __("values separated by commas")
-					: __("use % as wildcard"))+'</div>');
-			} else {
-				//if condition selected after refresh
-				me.set_field(me.field.df.parent, me.field.df.fieldname, null, condition);
-			}
-		});
-
-		// set the field
-		if(me.fieldname) {
-			// pre-sets given (could be via tags!)
-			return this.set_values(me._doctype, me.fieldname, me.condition, me.value);
-		} else {
-			me.set_field(me.doctype, 'name');
-		}
-	},
-
-	apply: function() {
-		var f = this.get_value();
-
-		this.flist.remove(this);
-		this.flist.push_new_filter(f[0], f[1], f[2], f[3]);
-		this.remove();
-	},
-
-	remove: function(dont_run) {
-		this.wrapper.remove();
-		this.$btn_group && this.$btn_group.remove();
-		this.field = null;
-		this.flist.update_filters();
-
-		if(!dont_run) {
-			this.flist.base_list.refresh(true);
-		}
-	},
-
-	set_values: function(doctype, fieldname, condition, value) {
-		// presents given (could be via tags!)
-		this.set_field(doctype, fieldname);
-
-		// change 0,1 to Yes, No for check field type
-		if(this.field.df.original_type==='Check') {
-			if(value==0) value = 'No';
-			else if(value==1) value = 'Yes';
-		}
-
-		if(condition) {
-			this.wrapper.find('.condition').val(condition).change();
-		}
-		if(value!=null) {
-			return this.field.set_value(value);
-		}
-	},
-
-	set_field: function(doctype, fieldname, fieldtype, condition) {
-		var me = this;
-
-		// set in fieldname (again)
-		var cur = me.field ? {
-			fieldname: me.field.df.fieldname,
-			fieldtype: me.field.df.fieldtype,
-			parent: me.field.df.parent,
-		} : {};
-
-		var original_docfield = me.fieldselect.fields_by_name[doctype][fieldname];
-		if(!original_docfield) {
-			frappe.msgprint(__("Field {0} is not selectable.", [fieldname]));
-			return;
-		}
-
-		var df = copy_dict(me.fieldselect.fields_by_name[doctype][fieldname]);
-
-		// filter field shouldn't be read only or hidden
-		df.read_only = 0;
-		df.hidden = 0;
-
-		if(!condition) this.set_default_condition(df, fieldtype);
-		this.set_fieldtype(df, fieldtype);
-
-		// called when condition is changed,
-		// don't change if all is well
-		if(me.field && cur.fieldname == fieldname && df.fieldtype == cur.fieldtype &&
-			df.parent == cur.parent) {
-			return;
-		}
-
-		// clear field area and make field
-		me.fieldselect.selected_doctype = doctype;
-		me.fieldselect.selected_fieldname = fieldname;
-
-		// save old text
-		var old_text = null;
-		if(me.field) {
-			old_text = me.field.get_value();
-		}
-
-		var field_area = me.wrapper.find('.filter_field').empty().get(0);
-		var f = frappe.ui.form.make_control({
-			df: df,
-			parent: field_area,
-			only_input: true,
-		})
-		f.refresh();
-
-		me.field = f;
-		if(old_text && me.field.df.fieldtype===cur.fieldtype) {
-			me.field.set_value(old_text);
-		}
-
-		// run on enter
-		$(me.field.wrapper).find(':input').keydown(function(ev) {
-			if(ev.which==13) {
-				me.flist.base_list.run();
-			}
-		})
-	},
-
-	set_fieldtype: function(df, fieldtype) {
-		// reset
-		if(df.original_type)
-			df.fieldtype = df.original_type;
-		else
-			df.original_type = df.fieldtype;
-
-		df.description = ''; df.reqd = 0;
-		df.ignore_link_validation = true;
-
-		// given
-		if(fieldtype) {
-			df.fieldtype = fieldtype;
-			return;
-		}
-
-		// scrub
-		if(df.fieldname=="docstatus") {
-			df.fieldtype="Select",
-			df.options=[
-				{value:0, label:__("Draft")},
-				{value:1, label:__("Submitted")},
-				{value:2, label:__("Cancelled")}
-			]
-		} else if(df.fieldtype=='Check') {
-			df.fieldtype='Select';
-			df.options='No\nYes';
-		} else if(['Text','Small Text','Text Editor','Code','Tag','Comments',
-			'Dynamic Link','Read Only','Assign'].indexOf(df.fieldtype)!=-1) {
-			df.fieldtype = 'Data';
-		} else if(df.fieldtype=='Link' && ['=', '!='].indexOf(this.wrapper.find('.condition').val())==-1) {
-			df.fieldtype = 'Data';
-		}
-		if(df.fieldtype==="Data" && (df.options || "").toLowerCase()==="email") {
-			df.options = null;
-		}
-		if(this.wrapper.find('.condition').val()== "Between" && (df.fieldtype == 'Date' || df.fieldtype == 'Datetime')){
-			df.fieldtype = 'DateRange';
-		}
-	},
-
-	set_default_condition: function(df, fieldtype) {
-		if(!fieldtype) {
-			// set as "like" for data fields
-			if (df.fieldtype == 'Data') {
-				this.wrapper.find('.condition').val('like');
-			} else if (df.fieldtype == 'Date' || df.fieldtype == 'Datetime'){
-				this.wrapper.find('.condition').val('Between');
-			}else{
-				this.wrapper.find('.condition').val('=');
-			}
-		}
-	},
-
-	get_value: function() {
-		return [this.fieldselect.selected_doctype,
-			this.field.df.fieldname, this.get_condition(), this.get_selected_value()];
-	},
-
-	get_selected_value: function() {
-		var val = this.field.get_value();
-
-		if(typeof val==='string') {
-			val = strip(val);
-		}
-
-		if(this.field.df.original_type == 'Check') {
-			val = (val=='Yes' ? 1 :0);
-		}
-
-		if(this.get_condition().indexOf('like', 'not like')!==-1) {
-			// automatically append wildcards
-			if(val) {
-				if(val.slice(0,1) !== "%") {
-					val = "%" + val;
-				}
-				if(val.slice(-1) !== "%") {
-					val = val + "%";
-				}
-			}
-		} else if(in_list(["in", "not in"], this.get_condition())) {
-			if(val) {
-				val = $.map(val.split(","), function(v) { return strip(v); });
-			}
-		} if(val === '%') {
-			val = "";
-		}
-
-		return val;
-	},
-
-	get_condition: function() {
-		return this.wrapper.find('.condition').val();
-	},
-
-	freeze: function() {
-		if(this.$btn_group) {
-			// already made, just hide the condition setter
-			this.set_filter_button_text();
-			this.wrapper.toggle(false);
-			return;
-		}
-
-		var me = this;
-
-		// add a button for new filter if missing
-		this.$btn_group = $(`<div class="btn-group">
-			<button class="btn btn-default btn-xs toggle-filter"
-				title="${ __("Edit Filter") }">
-			</button>
-			<button class="btn btn-default btn-xs remove-filter"
-				title="${ __("Remove Filter") }">
-				<i class="fa fa-remove text-muted"></i>
-			</button></div>`)
-			.insertAfter(this.flist.wrapper.find(".set-filters .new-filter"));
-
-		this.set_filter_button_text();
-
-		this.$btn_group.find(".remove-filter").on("click", function() {
-			me.remove();
-		});
-
-		this.$btn_group.find(".toggle-filter").on("click", function() {
-			$(this).closest('.show_filters').find('.filter_area').show();
-			me.wrapper.toggle();
-		})
-		this.wrapper.toggle(false);
-	},
-
-	set_filter_button_text: function() {
-		var value = this.get_selected_value();
-		value = this.flist.get_formatted_value(this.field, value);
-
-		// for translations
-		// __("like"), __("not like"), __("in")
-
-		this.$btn_group.find(".toggle-filter")
-			.html(repl('%(label)s %(condition)s "%(value)s"', {
-				label: __(this.field.df.label),
-				condition: __(this.get_condition()),
-				value: __(value),
-			}));
-	}
-
-});
-
-// <select> widget with all fields of a doctype as options
-frappe.ui.FieldSelect = Class.extend({
-	// opts parent, doctype, filter_fields, with_blank, select
-	init: function(opts) {
-		var me = this;
-		$.extend(this, opts);
-		this.fields_by_name = {};
-		this.options = [];
-		this.$select = $('<input class="form-control">')
-			.appendTo(this.parent)
-			.on("click", function () { $(this).select(); });
-		this.select_input = this.$select.get(0);
-		this.awesomplete = new Awesomplete(this.select_input, {
-			minChars: 0,
-			maxItems: 99,
-			autoFirst: true,
-			list: me.options,
-			item: function(item, input) {
-				return $(repl('<li class="filter-field-select"><p>%(label)s</p></li>', item))
-				.data("item.autocomplete", item)
-				.get(0);
-			}
-		});
-		this.$select.on("awesomplete-select", function(e) {
-			var o = e.originalEvent;
-			var value = o.text.value;
-			var item = me.awesomplete.get_item(value);
-			me.selected_doctype = item.doctype;
-			me.selected_fieldname = item.fieldname;
-			if(me.select) me.select(item.doctype, item.fieldname);
-		});
-		this.$select.on("awesomplete-selectcomplete", function(e) {
-			var o = e.originalEvent;
-			var value = o.text.value;
-			var item = me.awesomplete.get_item(value);
-			me.$select.val(item.label);
-		});
-
-		if(this.filter_fields) {
-			for(var i in this.filter_fields)
-				this.add_field_option(this.filter_fields[i])
-		} else {
-			this.build_options();
-		}
-		this.set_value(this.doctype, "name");
-		window.last_filter = this;
-	},
-	get_value: function() {
-		return this.selected_doctype ? this.selected_doctype + "." + this.selected_fieldname : null;
-	},
-	val: function(value) {
-		if(value===undefined) {
-			return this.get_value();
-		} else {
-			this.set_value(value);
-		}
-	},
-	clear: function() {
-		this.selected_doctype = null;
-		this.selected_fieldname = null;
-		this.$select.val("");
-	},
-	set_value: function(doctype, fieldname) {
-		var me = this;
-		this.clear();
-		if(!doctype) return;
-
-		// old style
-		if(doctype.indexOf(".")!==-1) {
-			var parts = doctype.split(".");
-			doctype = parts[0];
-			fieldname = parts[1];
-		}
-
-		$.each(this.options, function(i, v) {
-			if(v.doctype===doctype && v.fieldname===fieldname) {
-				me.selected_doctype = doctype;
-				me.selected_fieldname = fieldname;
-				me.$select.val(v.label);
-				return false;
-			}
-		});
-	},
-	build_options: function() {
-		var me = this;
-		me.table_fields = [];
-		var std_filters = $.map(frappe.model.std_fields, function(d) {
-			var opts = {parent: me.doctype}
-			if(d.fieldname=="name") opts.options = me.doctype;
-			return $.extend(copy_dict(d), opts);
-		});
-
-		// add parenttype column
-		var doctype_obj = locals['DocType'][me.doctype];
-		if(doctype_obj && cint(doctype_obj.istable)) {
-			std_filters = std_filters.concat([{
-				fieldname: 'parent',
-				fieldtype: 'Data',
-				label: 'Parent',
-				parent: me.doctype,
-			}]);
-		}
-
-		// blank
-		if(this.with_blank) {
-			this.options.push({
-				label:"",
-				value:"",
-			})
-		}
-
-		// main table
-		var main_table_fields = std_filters.concat(frappe.meta.docfield_list[me.doctype]);
-		$.each(frappe.utils.sort(main_table_fields, "label", "string"), function(i, df) {
-			// show fields where user has read access and if report hide flag is not set
-			if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide)
-				me.add_field_option(df);
-		});
-
-		// child tables
-		$.each(me.table_fields, function(i, table_df) {
-			if(table_df.options) {
-				var child_table_fields = [].concat(frappe.meta.docfield_list[table_df.options]);
-				$.each(frappe.utils.sort(child_table_fields, "label", "string"), function(i, df) {
-					// show fields where user has read access and if report hide flag is not set
-					if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide)
-						me.add_field_option(df);
-				});
-			}
-		});
-	},
-
-	add_field_option: function(df) {
-		var me = this;
-		if(me.doctype && df.parent==me.doctype) {
-			var label = __(df.label);
-			var table = me.doctype;
-			if(frappe.model.table_fields.includes(df.fieldtype)) me.table_fields.push(df);
-		} else {
-			var label = __(df.label) + ' (' + __(df.parent) + ')';
-			var table = df.parent;
-		}
-		if(frappe.model.no_value_type.indexOf(df.fieldtype) == -1 &&
-			!(me.fields_by_name[df.parent] && me.fields_by_name[df.parent][df.fieldname])) {
-			this.options.push({
-				label: label,
-				value: table + "." + df.fieldname,
-				fieldname: df.fieldname,
-				doctype: df.parent
-			});
-			if(!me.fields_by_name[df.parent]) me.fields_by_name[df.parent] = {};
-			me.fields_by_name[df.parent][df.fieldname] = df;
-		}
-	},
-})

From c64df2787f8328e6d7e46c4cb722ec27820eb722 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 22 Jul 2020 16:51:17 +0530
Subject: [PATCH 035/361] fix: use get_filter_value in filter_list

---
 frappe/public/js/frappe/ui/filters/filter.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js
index 7c6d532f5c..bfc0b1449d 100644
--- a/frappe/public/js/frappe/ui/filters/filter.js
+++ b/frappe/public/js/frappe/ui/filters/filter.js
@@ -249,7 +249,7 @@ frappe.ui.Filter = class {
 			let args = {};
 			if (this.filters_config[condition].depends_on) {
 				const field_name = this.filters_config[condition].depends_on;
-				const filter_value = this.get_filter_value(field_name);
+				const filter_value = this.filter_list.get_filter_value(fieldname);
 				args[field_name] = filter_value;
 			}
 			frappe
@@ -265,10 +265,6 @@ frappe.ui.Filter = class {
 		}
 	}
 
-	get_filter_value(fieldname) {
-		return this.filter_list.get_filter_value(fieldname);
-	}
-
 	make_field(df, old_fieldtype) {
 		let old_text = this.field ? this.field.get_value() : null;
 		this.hide_invalid_conditions(df.fieldtype, df.original_type);

From a2b2c9c926b1fd2be8192499dc8a21b250019eaa Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:54:48 +0530
Subject: [PATCH 036/361] fix: Validate select values in Column

---
 frappe/core/doctype/data_import/importer.py | 29 ++++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 910e42af1a..729f4e8956 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -615,7 +615,7 @@ class Row:
 	def validate_value(self, value, col):
 		df = col.df
 		if df.fieldtype == "Select":
-			select_options = [d for d in (df.options or '').split('\n') if d]
+			select_options = get_select_options(df)
 			if select_options and value not in select_options:
 				options_string = ", ".join([frappe.bold(d) for d in select_options])
 				msg = _("Value must be one of {0}").format(options_string)
@@ -971,12 +971,23 @@ class Column:
 			# guess date format
 			self.date_format = self.guess_date_format_for_column()
 			if not self.date_format:
-				self.date_format = '%Y-%m-%d'
-				self.warnings.append({
-					'col': self.column_number,
-					'message': _("Date format could not determined from the values in this column. Defaulting to yyyy-mm-dd."),
-					'type': 'info'
-				})
+		elif self.df.fieldtype == "Select":
+			options = get_select_options(self.df)
+			if options:
+				values = list(set([cstr(v) for v in self.column_values[1:] if v]))
+				invalid = list(set(values) - set(options))
+				if invalid:
+					valid_values = ", ".join([frappe.bold(o) for o in options])
+					invalid_values = ", ".join([frappe.bold(i) for i in invalid])
+					self.warnings.append(
+						{
+							"col": self.column_number,
+							"message": (
+								"The following values are invalid: {0}. Values must be"
+								" one of {1}".format(invalid_values, valid_values)
+							),
+						}
+					)
 
 	def as_dict(self):
 		d = frappe._dict()
@@ -1170,3 +1181,7 @@ def df_as_json(df):
 		'parent': df.parent,
 		'default': df.default
 	}
+
+
+def get_select_options(df):
+	return [d for d in (df.options or "").split("\n") if d]

From 5dcb910ae9e040f76b1e7e18354e31c2dd7ce729 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:57:01 +0530
Subject: [PATCH 037/361] fix: Set default date format for parsed date values

---
 frappe/core/doctype/data_import/importer.py | 32 ++++++++++++++++-----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 729f4e8956..47113d35cd 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -7,7 +7,7 @@ import io
 import frappe
 import timeit
 import json
-from datetime import datetime
+from datetime import datetime, date
 from frappe import _
 from frappe.utils import cint, flt, update_progress_bar, cstr
 from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets
@@ -668,7 +668,7 @@ class Row:
 
 	def parse_value(self, value, col):
 		df = col.df
-		if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]:
+		if isinstance(value, (datetime, date)) and df.fieldtype in ["Date", "Datetime"]:
 			return value
 
 		value = cstr(value)
@@ -689,7 +689,7 @@ class Row:
 		return value
 
 	def get_date(self, value, column):
-		if isinstance(value, datetime):
+		if isinstance(value, (datetime, date)):
 			return value
 
 		date_format = column.date_format
@@ -918,13 +918,20 @@ class Column:
 		self.skip_import = skip_import
 
 	def guess_date_format_for_column(self):
-		""" Guesses date format for a column by parsing all the values in the column,
+		"""Guesses date format for a column by parsing all the values in the column,
 		getting the date format and then returning the one which has the maximum frequency
 		"""
 
-		date_formats = [
-			frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str)
-		]
+		def guess_date_format(d):
+			if isinstance(d, (datetime, date)):
+				if self.df.fieldtype == "Date":
+					return "%Y-%m-%d"
+				if self.df.fieldtype == "Datetime":
+					return "%Y-%m-%d %H:%M:%S"
+			if isinstance(d, str):
+				return frappe.utils.guess_date_format(d)
+
+		date_formats = [guess_date_format(d) for d in self.column_values]
 		date_formats = [d for d in date_formats if d]
 		if not date_formats:
 			return
@@ -971,6 +978,17 @@ class Column:
 			# guess date format
 			self.date_format = self.guess_date_format_for_column()
 			if not self.date_format:
+				self.date_format = "%Y-%m-%d"
+				self.warnings.append(
+					{
+						"col": self.column_number,
+						"message": _(
+							"Date format could not be determined from the values in"
+							" this column. Defaulting to yyyy-mm-dd."
+						),
+						"type": "info",
+					}
+				)
 		elif self.df.fieldtype == "Select":
 			options = get_select_options(self.df)
 			if options:

From 015ab803a36355cd02df0e640dba0897515d8ebf Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:57:13 +0530
Subject: [PATCH 038/361] style: formatting

---
 frappe/core/doctype/data_import/importer.py | 63 ++++++++++-----------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 47113d35cd..6bc8b0586f 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -233,7 +233,7 @@ class Importer:
 			return updated_doc
 		else:
 			# throw if no changes
-			frappe.throw('No changes to update')
+			frappe.throw("No changes to update")
 
 	def get_eta(self, current, total, processing_time):
 		self.last_eta = getattr(self, "last_eta", 0)
@@ -322,7 +322,7 @@ class ImportFile:
 		if isinstance(file, frappe.string_types):
 			if frappe.db.exists("File", {"file_url": file}):
 				self.file_doc = frappe.get_doc("File", {"file_url": file})
-			elif 'docs.google.com/spreadsheets' in file:
+			elif "docs.google.com/spreadsheets" in file:
 				self.google_sheets_url = file
 			elif os.path.exists(file):
 				self.file_path = file
@@ -348,7 +348,7 @@ class ImportFile:
 
 		elif self.google_sheets_url:
 			content = get_csv_content_from_google_sheets(self.google_sheets_url)
-			extension = 'csv'
+			extension = "csv"
 
 		if not content:
 			frappe.throw(_("Invalid or corrupted content for import"))
@@ -620,11 +620,7 @@ class Row:
 				options_string = ", ".join([frappe.bold(d) for d in select_options])
 				msg = _("Value must be one of {0}").format(options_string)
 				self.warnings.append(
-					{
-						"row": self.row_number,
-						"field": df_as_json(df),
-						"message": msg,
-					}
+					{"row": self.row_number, "field": df_as_json(df), "message": msg,}
 				)
 				return
 
@@ -635,11 +631,7 @@ class Row:
 					frappe.bold(value), frappe.bold(df.options)
 				)
 				self.warnings.append(
-					{
-						"row": self.row_number,
-						"field": df_as_json(df),
-						"message": msg,
-					}
+					{"row": self.row_number, "field": df_as_json(df), "message": msg,}
 				)
 				return
 		elif df.fieldtype in ["Date", "Datetime"]:
@@ -786,9 +778,7 @@ class Header(Row):
 		for j, header in enumerate(row):
 			column_values = [get_item_at_index(r, j) for r in raw_data]
 			map_to_field = column_to_field_map.get(str(j))
-			column = Column(
-				j, header, self.doctype, column_values, map_to_field, self.seen
-			)
+			column = Column(j, header, self.doctype, column_values, map_to_field, self.seen)
 			self.seen.append(header)
 			self.columns.append(column)
 
@@ -962,18 +952,26 @@ class Column:
 		if not self.df:
 			return
 
-		if self.df.fieldtype == 'Link':
+		if self.df.fieldtype == "Link":
 			# find all values that dont exist
 			values = list(set([cstr(v) for v in self.column_values[1:] if v]))
-			exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})]
+			exists = [
+				d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})
+			]
 			not_exists = list(set(values) - set(exists))
 			if not_exists:
-				missing_values = ', '.join(not_exists)
-				self.warnings.append({
-					'col': self.column_number,
-					'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values),
-					'type': 'warning'
-				})
+				missing_values = ", ".join(not_exists)
+				self.warnings.append(
+					{
+						"col": self.column_number,
+						"message": (
+							"The following values do not exist for {}: {}".format(
+								self.df.options, missing_values
+							)
+						),
+						"type": "warning",
+					}
+				)
 		elif self.df.fieldtype in ("Date", "Time", "Datetime"):
 			# guess date format
 			self.date_format = self.guess_date_format_for_column()
@@ -1016,7 +1014,7 @@ class Column:
 		d.map_to_field = self.map_to_field
 		d.date_format = self.date_format
 		d.df = self.df
-		if hasattr(self.df, 'is_child_table_field'):
+		if hasattr(self.df, "is_child_table_field"):
 			d.is_child_table_field = self.df.is_child_table_field
 			d.child_table_df = self.df.child_table_df
 		d.skip_import = self.skip_import
@@ -1096,7 +1094,7 @@ def build_fields_dict_for_column_matching(parent_doctype):
 		# other fields
 		fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields
 		for df in fields:
-			label = (df.label or '').strip()
+			label = (df.label or "").strip()
 			fieldtype = df.fieldtype or "Data"
 			parent = df.parent or parent_doctype
 			if fieldtype not in no_value_fields:
@@ -1190,14 +1188,15 @@ def get_user_format(date_format):
 		.replace("%d", "dd")
 	)
 
+
 def df_as_json(df):
 	return {
-		'fieldname': df.fieldname,
-		'fieldtype': df.fieldtype,
-		'label': df.label,
-		'options': df.options,
-		'parent': df.parent,
-		'default': df.default
+		"fieldname": df.fieldname,
+		"fieldtype": df.fieldtype,
+		"label": df.label,
+		"options": df.options,
+		"parent": df.parent,
+		"default": df.default,
 	}
 
 

From e34e48bb2f503c20899b4f4e6f56ca74ffbbb1ee Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:57:57 +0530
Subject: [PATCH 039/361] fix: Show Column Name in warnings header

---
 frappe/core/doctype/data_import/data_import.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js
index 6a922618cb..0e827a42d8 100644
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@@ -317,6 +317,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_warnings(frm, preview_data) {
+		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || '[]');
 		warnings = warnings.concat(preview_data.warnings || []);
 
@@ -367,11 +368,13 @@ frappe.ui.form.on('Data Import', {
 			.map(warning => {
 				let header = '';
 				if (warning.col) {
-					header = __('Column {0}', [warning.col]);
+					let column_number = `<span class="text-uppercase">${__('Column {0}', [warning.col])}</span>`;
+					let column_header = columns[warning.col].header_title;
+					header = `${column_number} (${column_header})`;
 				}
 				return `
 					<div class="warning" data-col="${warning.col}">
-						<h5 class="text-uppercase">${header}</h5>
+						<h5>${header}</h5>
 						<div class="body">${warning.message}</div>
 					</div>
 				`;

From de07437015dd3a21cb1a4db1bc0fdf68230a35a3 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:58:16 +0530
Subject: [PATCH 040/361] fix: Show Pending imports as Not Started

---
 frappe/core/doctype/data_import/data_import_list.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js
index 1dee4319f9..0eb05aa354 100644
--- a/frappe/core/doctype/data_import/data_import_list.js
+++ b/frappe/core/doctype/data_import/data_import_list.js
@@ -17,6 +17,7 @@ frappe.listview_settings['Data Import'] = {
 	get_indicator: function(doc) {
 		var colors = {
 			'Pending': 'orange',
+			'Not Started': 'orange',
 			'Partial Success': 'orange',
 			'Success': 'green',
 			'In Progress': 'orange',
@@ -26,6 +27,9 @@ frappe.listview_settings['Data Import'] = {
 		if (imports_in_progress.includes(doc.name)) {
 			status = 'In Progress';
 		}
+		if (status == 'Pending') {
+			status = 'Not Started';
+		}
 		return [__(status), colors[status], 'status,=,' + doc.status];
 	},
 	formatters: {

From b0786fd5e5ade05555f16d4e0a1b50a6bbe611d0 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 17:58:36 +0530
Subject: [PATCH 041/361] fix: Human friendly time format

---
 frappe/public/js/frappe/data_import/import_preview.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js
index 4edcb87aeb..6c17cb4351 100644
--- a/frappe/public/js/frappe/data_import/import_preview.js
+++ b/frappe/public/js/frappe/data_import/import_preview.js
@@ -98,6 +98,9 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					.replace('%y', 'yy')
 					.replace('%m', 'mm')
 					.replace('%d', 'dd')
+					.replace('%H', 'HH')
+					.replace('%M', 'mm')
+					.replace('%S', 'ss')
 				: null;
 
 			let column_title = `<span class="indicator green">
@@ -354,4 +357,4 @@ function get_fields_as_options(doctype, column_map) {
 			});
 		})
 	);
-}
\ No newline at end of file
+}

From 4b7e36f90da10a8c49fdbb827ea922e1f548e1e5 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 19:30:17 +0530
Subject: [PATCH 042/361] fix: Handle child table row additions

---
 frappe/core/doctype/data_import/importer.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 6bc8b0586f..34e53488e5 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -602,12 +602,20 @@ class Row:
 
 		is_table = frappe.get_meta(doctype).istable
 		is_update = self.import_type == UPDATE
-		if is_table and is_update and doc.get("name") in INVALID_VALUES:
-			# for table rows being inserted in update
-			# create a new doc with defaults set
-			new_doc = frappe.new_doc(doctype, as_dict=True)
-			new_doc.update(doc)
-			doc = new_doc
+		if is_table and is_update:
+			# check if the row already exists
+			# if yes, fetch the original doc so that it is not updated
+			# if no, create a new doc
+			id_field = get_id_field(doctype)
+			id_value = doc.get(id_field.fieldname)
+			if id_value and frappe.db.exists(doctype, id_value):
+				doc = frappe.get_doc(doctype, id_value)
+			else:
+				# for table rows being inserted in update
+				# create a new doc with defaults set
+				new_doc = frappe.new_doc(doctype, as_dict=True)
+				new_doc.update(doc)
+				doc = new_doc
 
 		self.check_mandatory_fields(doctype, doc, table_df)
 		return doc

From dd038a0cc201f0b5061864b8960892a419b4c0d0 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 19:30:41 +0530
Subject: [PATCH 043/361] fix: Show autoname column instead of ID in export

---
 .../js/frappe/data_import/data_exporter.js    | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js
index f6af338235..8651f2083d 100644
--- a/frappe/public/js/frappe/data_import/data_exporter.js
+++ b/frappe/public/js/frappe/data_import/data_exporter.js
@@ -274,23 +274,29 @@ frappe.data_import.DataExporter = class DataExporter {
 			? this.column_map[child_fieldname]
 			: this.column_map[doctype];
 
-		let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname)
-			|| (df.reqd && this.exporting_for == 'Insert New Records');
+		let is_field_mandatory = df => {
+			if (df.reqd && this.exporting_for == 'Insert New Records') {
+				return true;
+			}
+			if (autoname_field && df.fieldname == autoname_field.fieldname) {
+				return true;
+			}
+			if (df.fieldname === 'name') {
+				return true;
+			}
+			return false;
+		}
 
 		return fields
 			.filter(df => {
-				if (autoname_field && df.fieldname === autoname_field.fieldname) {
+				if (autoname_field && df.fieldname === 'name') {
 					return false;
 				}
 				return true;
 			})
 			.map(df => {
-				let label = __(df.label);
-				if (autoname_field && df.fieldname === 'name') {
-					label = label + ` (${__(autoname_field.label)})`;
-				}
 				return {
-					label,
+					label: __(df.label),
 					value: df.fieldname,
 					danger: is_field_mandatory(df),
 					checked: false,

From 5558b20834fa0800da42a6dbab71b1e308c8e863 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 22 Jul 2020 19:52:56 +0530
Subject: [PATCH 044/361] style: missing semicolon

---
 frappe/public/js/frappe/data_import/data_exporter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js
index 8651f2083d..dee4839b34 100644
--- a/frappe/public/js/frappe/data_import/data_exporter.js
+++ b/frappe/public/js/frappe/data_import/data_exporter.js
@@ -285,7 +285,7 @@ frappe.data_import.DataExporter = class DataExporter {
 				return true;
 			}
 			return false;
-		}
+		};
 
 		return fields
 			.filter(df => {

From 6187de8e3da34cc4f0f74198f1148170c3d9d83d Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Thu, 23 Jul 2020 11:12:13 +0530
Subject: [PATCH 045/361] fix: make is_standard read only if not in developer
 mode

---
 frappe/desk/doctype/dashboard/dashboard.json             | 6 +++---
 frappe/desk/doctype/dashboard_chart/dashboard_chart.json | 6 +++---
 frappe/desk/doctype/number_card/number_card.json         | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/frappe/desk/doctype/dashboard/dashboard.json b/frappe/desk/doctype/dashboard/dashboard.json
index 0c54f1ee7f..95b0846452 100644
--- a/frappe/desk/doctype/dashboard/dashboard.json
+++ b/frappe/desk/doctype/dashboard/dashboard.json
@@ -52,10 +52,10 @@
   },
   {
    "default": "0",
-   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "read_only_depends_on": "eval: !frappe.boot.developer_mode"
   },
   {
    "depends_on": "eval: doc.is_standard",
@@ -67,7 +67,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-22 11:23:37.858568",
+ "modified": "2020-07-23 11:05:41.890459",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard",
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
index f0616dddd0..d4bba53068 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json
@@ -238,10 +238,10 @@
   },
   {
    "default": "0",
-   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "read_only_depends_on": "eval: !frappe.boot.developer_mode"
   },
   {
    "depends_on": "eval: doc.is_standard",
@@ -271,7 +271,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-22 11:32:51.185499",
+ "modified": "2020-07-23 11:10:33.509497",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard Chart",
diff --git a/frappe/desk/doctype/number_card/number_card.json b/frappe/desk/doctype/number_card/number_card.json
index 0d1cc279c8..d3e9598eb7 100644
--- a/frappe/desk/doctype/number_card/number_card.json
+++ b/frappe/desk/doctype/number_card/number_card.json
@@ -113,10 +113,10 @@
   },
   {
    "default": "0",
-   "depends_on": "eval: frappe.boot.developer_mode",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "read_only_depends_on": "eval: !frappe.boot.developer_mode"
   },
   {
    "depends_on": "eval: doc.is_standard",
@@ -191,7 +191,7 @@
   }
  ],
  "links": [],
- "modified": "2020-07-22 11:26:32.039023",
+ "modified": "2020-07-23 11:11:03.391719",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Number Card",

From 33e0b785764f873c75b20e43a1e940c6bc25e715 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 23 Jul 2020 15:06:46 +0530
Subject: [PATCH 046/361] fix: validate recipients before sending newsletter

---
 frappe/email/doctype/newsletter/newsletter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py
index a82b52a663..48688afdb6 100755
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@@ -107,6 +107,9 @@ class Newsletter(WebsiteGenerator):
 		if self.get("__islocal"):
 			throw(_("Please save the Newsletter before sending"))
 
+		if not self.recipients:
+			frappe.throw(_("Newsletter should have at least one recipient"))
+
 	def get_context(self, context):
 		newsletters = get_newsletter_list("Newsletter", None, None, 0)
 		if newsletters:

From 24adc8c3625f146ecae895ac57bdd8ff081d92b8 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Thu, 23 Jul 2020 17:34:18 +0530
Subject: [PATCH 047/361] fix: Use find instead of filter

---
 frappe/public/js/frappe/ui/filters/filter_list.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter_list.js b/frappe/public/js/frappe/ui/filters/filter_list.js
index 9a35a9bb5f..ee04fb6077 100644
--- a/frappe/public/js/frappe/ui/filters/filter_list.js
+++ b/frappe/public/js/frappe/ui/filters/filter_list.js
@@ -112,7 +112,8 @@ frappe.ui.FilterGroup = class {
 	}
 
 	get_filter_value(fieldname) {
-		return this.filters.filter(f => f.fieldname == fieldname)[0].value;
+		let filter_obj = this.filters.find(f => f.fieldname == filename) || {};
+		return filter_obj.value;
 	}
 
 	filter_exists(filter_value) {

From 5f5facfb0a0c6d7725c309708eadb90af50c7278 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Thu, 23 Jul 2020 17:40:51 +0530
Subject: [PATCH 048/361] fix: set frm.filters to empty array for number card

---
 frappe/desk/doctype/dashboard_chart/dashboard_chart.js | 2 +-
 frappe/desk/doctype/number_card/number_card.js         | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
index 65db4839a1..8d89cc2f31 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
@@ -291,7 +291,7 @@ frappe.ui.form.on('Dashboard Chart', {
 			set_filters && frm.set_value('filters_json', JSON.stringify(filters));
 		}
 
-		let fields;
+		let fields = [];
 		if (is_document_type) {
 			fields = [
 				{
diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js
index 2ae163d7aa..63b41b956e 100644
--- a/frappe/desk/doctype/number_card/number_card.js
+++ b/frappe/desk/doctype/number_card/number_card.js
@@ -116,6 +116,7 @@ frappe.ui.form.on('Number Card', {
 	},
 
 	report_name: function(frm) {
+		frm.filters = [];
 		frm.set_value('filters_json', '{}');
 		frm.set_value('dynamic_filters_json', '{}');
 		frm.set_df_property('report_field', 'options', []);
@@ -247,7 +248,7 @@ frappe.ui.form.on('Number Card', {
 			set_filters && frm.set_value('filters_json', JSON.stringify(filters));
 		}
 
-		let fields;
+		let fields = [];
 		if (is_document_type) {
 			fields = [
 				{

From 771feca0730a563cd3278fc4de11b10060389b27 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Thu, 23 Jul 2020 17:41:55 +0530
Subject: [PATCH 049/361] feat: remove restricted button in dashboard

---
 frappe/core/page/dashboard/dashboard.js                   | 7 -------
 frappe/public/js/frappe/views/dashboard/dashboard_view.js | 4 ----
 2 files changed, 11 deletions(-)

diff --git a/frappe/core/page/dashboard/dashboard.js b/frappe/core/page/dashboard/dashboard.js
index cee230265f..7e45163a7e 100644
--- a/frappe/core/page/dashboard/dashboard.js
+++ b/frappe/core/page/dashboard/dashboard.js
@@ -26,13 +26,6 @@ class Dashboard {
 		</div>`).appendTo(this.wrapper.find(".page-content").empty());
 		this.container = this.wrapper.find(".dashboard-graph");
 		this.page = wrapper.page;
-
-		this.page.set_title_sub(
-			$(`<button class="restricted-button">
-				<span class="octicon octicon-lock"></span>
-				<span>${__('Restricted')}</span>
-			</button>`)
-		);
 	}
 
 	show() {
diff --git a/frappe/public/js/frappe/views/dashboard/dashboard_view.js b/frappe/public/js/frappe/views/dashboard/dashboard_view.js
index 83f45da5be..6f6279fd08 100644
--- a/frappe/public/js/frappe/views/dashboard/dashboard_view.js
+++ b/frappe/public/js/frappe/views/dashboard/dashboard_view.js
@@ -44,10 +44,6 @@ frappe.views.DashboardView = class DashboardView extends frappe.views.ListView {
 				<div class="text-muted">
 					<span class="uppercase header-title"> ${dashboard_name} </span>
 				</div>
-				<button class="restricted-button">
-					<span class="octicon octicon-lock"></span>
-					<span>${__('Restricted')}</span>
-				</button>
 				<div class="text-muted customize-dashboard" data-action="customize">${__('Customize')}</div>
 				<div class="small text-muted customize-options small-bounce">
 					<span class="reset-customization customize-option" data-action="reset_dashboard_customization">${__('Reset')}</span>

From b95d28d5f126785e2198eae1b5eba556599dd814 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Thu, 23 Jul 2020 17:43:01 +0530
Subject: [PATCH 050/361] fix: remove unused style

---
 frappe/core/page/dashboard/dashboard.css | 5 -----
 frappe/public/less/dashboard_view.less   | 7 -------
 2 files changed, 12 deletions(-)
 delete mode 100644 frappe/core/page/dashboard/dashboard.css

diff --git a/frappe/core/page/dashboard/dashboard.css b/frappe/core/page/dashboard/dashboard.css
deleted file mode 100644
index b319cc1ed2..0000000000
--- a/frappe/core/page/dashboard/dashboard.css
+++ /dev/null
@@ -1,5 +0,0 @@
-.restricted-button {
-	cursor: default;
-	position: relative;
-	right: -5px;
-}
\ No newline at end of file
diff --git a/frappe/public/less/dashboard_view.less b/frappe/public/less/dashboard_view.less
index ab78fa5b2a..17fa52cc0f 100644
--- a/frappe/public/less/dashboard_view.less
+++ b/frappe/public/less/dashboard_view.less
@@ -33,13 +33,6 @@
 				line-height: 1.5em;
 				vertical-align: text-bottom;
 			}
-
-			.restricted-button {
-				cursor: default;
-				position: relative;
-				right: 5px;
-				top: -3px;
-			}
 		}
 
 		.customize-dashboard {

From 160a84cee4177cf4fa828fd098ebdae89d307040 Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Thu, 23 Jul 2020 17:55:23 +0530
Subject: [PATCH 051/361] fix: use correct variable

---
 frappe/public/js/frappe/ui/filters/filter_list.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter_list.js b/frappe/public/js/frappe/ui/filters/filter_list.js
index ee04fb6077..6c577aa0bc 100644
--- a/frappe/public/js/frappe/ui/filters/filter_list.js
+++ b/frappe/public/js/frappe/ui/filters/filter_list.js
@@ -112,7 +112,7 @@ frappe.ui.FilterGroup = class {
 	}
 
 	get_filter_value(fieldname) {
-		let filter_obj = this.filters.find(f => f.fieldname == filename) || {};
+		let filter_obj = this.filters.find(f => f.fieldname == fieldname) || {};
 		return filter_obj.value;
 	}
 

From b0b53e03f6a602c5cddcf2c3de5db580dba197af Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 23 Jul 2020 19:01:18 +0530
Subject: [PATCH 052/361] feat: Add Bulk restore API

---
 .../deleted_document/deleted_document.py      | 34 +++++++++++++++++--
 frappe/exceptions.py                          |  1 +
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index bc2962ab3f..a1786e0a3e 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 import frappe, json
+from frappe.desk.doctype.bulk_update.bulk_update import show_progress
 from frappe.model.document import Document
 from frappe import _
 
@@ -11,9 +12,14 @@ class DeletedDocument(Document):
 	pass
 
 @frappe.whitelist()
-def restore(name):
+def restore(name, alert=True):
 	deleted = frappe.get_doc('Deleted Document', name)
+
+	if deleted.restored:
+		frappe.throw(_("Document {0} Already Restored".format(name)), exc=frappe.DocumentAlreadyRestored)
+
 	doc = frappe.get_doc(json.loads(deleted.data))
+
 	try:
 		doc.insert()
 	except frappe.DocstatusTransitionError:
@@ -27,4 +33,28 @@ def restore(name):
 	deleted.restored = 1
 	deleted.db_update()
 
-	frappe.msgprint(_('Document Restored'))
+	if alert:
+		frappe.msgprint(_('Document Restored'))
+
+
+@frappe.whitelist()
+def bulk_restore(docnames):
+	docnames = frappe.parse_json(docnames)
+	doctype = "Deleted Document"
+	restored = []
+	failed = []
+
+	for i, d in enumerate(docnames):
+		try:
+			restore(d, alert=False)
+			message = _('Restoring {0}').format(doctype)
+			frappe.db.commit()
+			show_progress(docnames, message, i+1, d)
+			restored.append(d)
+		except frappe.DocumentAlreadyRestored:
+			failed.append(d)
+		except Exception as e:
+			failed.append(d)
+			frappe.db.rollback()
+
+	return restored
\ No newline at end of file
diff --git a/frappe/exceptions.py b/frappe/exceptions.py
index 8ebda9c7b8..88428b875c 100644
--- a/frappe/exceptions.py
+++ b/frappe/exceptions.py
@@ -104,6 +104,7 @@ class IncompatibleApp(ValidationError): pass
 class InvalidDates(ValidationError): pass
 class DataTooLongException(ValidationError): pass
 class FileAlreadyAttachedException(Exception): pass
+class DocumentAlreadyRestored(Exception): pass
 # OAuth exceptions
 class InvalidAuthorizationHeader(CSRFTokenError): pass
 class InvalidAuthorizationPrefix(CSRFTokenError): pass

From 70e966d5b691b8b9d3889fb21e65d828d35549af Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 23 Jul 2020 19:01:51 +0530
Subject: [PATCH 053/361] style: formatting + dropped unused variables

---
 .../deleted_document/deleted_document.py      | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index a1786e0a3e..636e3c4f37 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -3,14 +3,17 @@
 # For license information, please see license.txt
 
 from __future__ import unicode_literals
-import frappe, json
+import frappe
+import json
 from frappe.desk.doctype.bulk_update.bulk_update import show_progress
 from frappe.model.document import Document
 from frappe import _
 
+
 class DeletedDocument(Document):
 	pass
 
+
 @frappe.whitelist()
 def restore(name, alert=True):
 	deleted = frappe.get_doc('Deleted Document', name)
@@ -40,21 +43,20 @@ def restore(name, alert=True):
 @frappe.whitelist()
 def bulk_restore(docnames):
 	docnames = frappe.parse_json(docnames)
-	doctype = "Deleted Document"
+	message = _('Restoring Deleted Document')
 	restored = []
-	failed = []
 
 	for i, d in enumerate(docnames):
 		try:
+			show_progress(docnames, message, i + 1, d)
 			restore(d, alert=False)
-			message = _('Restoring {0}').format(doctype)
 			frappe.db.commit()
-			show_progress(docnames, message, i+1, d)
 			restored.append(d)
+
 		except frappe.DocumentAlreadyRestored:
-			failed.append(d)
-		except Exception as e:
-			failed.append(d)
+			pass
+
+		except Exception:
 			frappe.db.rollback()
 
-	return restored
\ No newline at end of file
+	return restored

From 9dd0304773c10fb363351cf1e0f07634082ac624 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 23 Jul 2020 19:02:25 +0530
Subject: [PATCH 054/361] feat: Bulk Restore action under Deleted Document

---
 .../deleted_document/deleted_document_list.js | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 frappe/core/doctype/deleted_document/deleted_document_list.js

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
new file mode 100644
index 0000000000..eb5a9e087d
--- /dev/null
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -0,0 +1,33 @@
+frappe.listview_settings["Deleted Document"] = {
+	onload: function (doclist) {
+		const action = () => {
+			const selected_docs = doclist.get_checked_items();
+			if (selected_docs.length > 0) {
+				let docnames = selected_docs.map((doc) => {
+					return doc.name;
+				});
+				frappe.call({
+					method:
+						"frappe.core.doctype.deleted_document.deleted_document.bulk_restore",
+					args: { docnames: docnames },
+					callback: function (r) {
+						if (r.message) {
+							let num = r.message.length;
+							frappe.msgprint(
+								__("{0} Document{1} {2} Restored", [
+									num,
+									num == 1 ? "" : "s",
+									num == 1 ? "was" : "were",
+								])
+							);
+							if (num > 0) {
+								doclist.refresh();
+							}
+						}
+					},
+				});
+			}
+		};
+		doclist.page.add_actions_menu_item(__("Restore"), action, false);
+	},
+};

From 0fdb7953bb8b86b7fabe3f34cbf9987bf9d107c9 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 23 Jul 2020 19:12:08 +0530
Subject: [PATCH 055/361] fix: translation syntax and style fixes

---
 frappe/core/doctype/deleted_document/deleted_document.py     | 2 +-
 .../core/doctype/deleted_document/deleted_document_list.js   | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index 636e3c4f37..598bd00cf6 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -19,7 +19,7 @@ def restore(name, alert=True):
 	deleted = frappe.get_doc('Deleted Document', name)
 
 	if deleted.restored:
-		frappe.throw(_("Document {0} Already Restored".format(name)), exc=frappe.DocumentAlreadyRestored)
+		frappe.throw(_("Document {0} Already Restored").format(name), exc=frappe.DocumentAlreadyRestored)
 
 	doc = frappe.get_doc(json.loads(deleted.data))
 
diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index eb5a9e087d..12ed62f6de 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -7,9 +7,8 @@ frappe.listview_settings["Deleted Document"] = {
 					return doc.name;
 				});
 				frappe.call({
-					method:
-						"frappe.core.doctype.deleted_document.deleted_document.bulk_restore",
-					args: { docnames: docnames },
+					method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore",
+					args: { "docnames": docnames },
 					callback: function (r) {
 						if (r.message) {
 							let num = r.message.length;

From 441630acc48bc58b71bd0fe100b73559e5180150 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 23 Jul 2020 23:43:35 +0530
Subject: [PATCH 056/361] fix: check file backup config before backing up files

---
 .../doctype/google_drive/google_drive.py        | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py
index c110694dff..58f28f882a 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.py
+++ b/frappe/integrations/doctype/google_drive/google_drive.py
@@ -189,14 +189,17 @@ def upload_system_backup_to_google_drive():
 	if frappe.flags.create_new_backup:
 		set_progress(1, "Backing up Data.")
 		backup = new_backup()
-		fileurl_backup = backup.backup_path_db
-		fileurl_site_config = backup.site_config_backup_path
-		fileurl_public_files = backup.backup_path_files
-		fileurl_private_files = backup.backup_path_private_files
-	else:
-		fileurl_backup, fileurl_site_config, fileurl_public_files, fileurl_private_files = get_latest_backup_file(with_files=True)
+		file_urls = []
+		file_urls.append(backup.backup_path_db)
+		file_urls.append(backup.site_config_backup_path)
 
-	for fileurl in [fileurl_backup, fileurl_site_config, fileurl_public_files, fileurl_private_files]:
+		if account.file_backup:
+			file_urls.append(backup.backup_path_files)
+			file_urls.append(backup.backup_path_private_files)
+	else:
+		file_urls = get_latest_backup_file(with_files=account.file_backup)
+
+	for fileurl in file_urls:
 		if not fileurl:
 			continue
 

From 47b0fb3de4882787a910938feff605f880cc44b3 Mon Sep 17 00:00:00 2001
From: Anupam K <anupamvns0099@gmail.com>
Date: Fri, 24 Jul 2020 03:31:58 +0530
Subject: [PATCH 057/361] fix: print format custom button

---
 .../doctype/print_format/print_format.js      | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js
index 252c706e51..bb0ba27e38 100644
--- a/frappe/printing/doctype/print_format/print_format.js
+++ b/frappe/printing/doctype/print_format/print_format.js
@@ -35,13 +35,20 @@ frappe.ui.form.on("Print Format", {
 			else if (frm.doc.custom_format && !frm.doc.raw_printing) {
 				frm.set_df_property("html", "reqd", 1);
 			}
-			frm.add_custom_button(__("Make Default"), function () {
-				frappe.call({
-					method: "frappe.printing.doctype.print_format.print_format.make_default",
-					args: {
-						name: frm.doc.name
-					}
-				})
+			frappe.db.get_value('DocType', frm.doc.doc_type, ['default_print_format', 'custom'], (r) => {
+				if (r.default_print_format != frm.doc.name && r.custom) {
+					frm.add_custom_button(__("Set as Default"), function () {
+						frappe.call({
+							method: "frappe.printing.doctype.print_format.print_format.make_default",
+							args: {
+								name: frm.doc.name
+							},
+							callback: function() {
+								frm.refresh();
+							}
+						});
+					});
+				}
 			});
 		}
 	},

From ac1f82af9aeb457a202bb8a2907f3aed1df3e7aa Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Fri, 24 Jul 2020 11:41:10 +0530
Subject: [PATCH 058/361] fix: translation syntax

Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>
---
 .../deleted_document/deleted_document_list.js      | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index 12ed62f6de..4443dc883d 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -12,13 +12,13 @@ frappe.listview_settings["Deleted Document"] = {
 					callback: function (r) {
 						if (r.message) {
 							let num = r.message.length;
-							frappe.msgprint(
-								__("{0} Document{1} {2} Restored", [
-									num,
-									num == 1 ? "" : "s",
-									num == 1 ? "was" : "were",
-								])
-							);
+							let message;
+							if (num === 1) {
+								message = "{0} Document was Restored"
+							} else {
+								message = "{0} Documents were Restored"
+							}
+							frappe.msgprint(__(message, [num])
 							if (num > 0) {
 								doclist.refresh();
 							}

From cfa286c5b03a01d9a6898f4bc90058c1470a08d1 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 24 Jul 2020 11:58:50 +0530
Subject: [PATCH 059/361] style: formatting, syntax and spacing

---
 .../core/doctype/deleted_document/deleted_document_list.js  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index 4443dc883d..a49d8125e3 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -14,11 +14,11 @@ frappe.listview_settings["Deleted Document"] = {
 							let num = r.message.length;
 							let message;
 							if (num === 1) {
-								message = "{0} Document was Restored"
+								message = "{0} Document was Restored";
 							} else {
-								message = "{0} Documents were Restored"
+								message = "{0} Documents were Restored";
 							}
-							frappe.msgprint(__(message, [num])
+							frappe.msgprint(__(message, [num]));
 							if (num > 0) {
 								doclist.refresh();
 							}

From 85f8c712dcfba4f9c643e9d2f1dc4efd85aff010 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 24 Jul 2020 11:59:29 +0530
Subject: [PATCH 060/361] fix: Show files restore summary

---
 .../deleted_document/deleted_document.py      | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index 598bd00cf6..e637f4f000 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -45,6 +45,8 @@ def bulk_restore(docnames):
 	docnames = frappe.parse_json(docnames)
 	message = _('Restoring Deleted Document')
 	restored = []
+	invalid = []
+	failed = []
 
 	for i, d in enumerate(docnames):
 		try:
@@ -54,9 +56,36 @@ def bulk_restore(docnames):
 			restored.append(d)
 
 		except frappe.DocumentAlreadyRestored:
-			pass
+			frappe.message_log.pop()
+			invalid.append(d)
 
 		except Exception:
+			failed.append(d)
 			frappe.db.rollback()
+			frappe.message_log.pop()
+
+	if failed or invalid:
+		tail = "</ul>"
+
+		restored_data = ""
+		restored_head = _("Documents restored successfully") + "<br><ul>"
+		for docname in restored:
+			restored_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
+		restored_body = restored_head + restored_data + tail
+
+		invalid_data = ""
+		invalid_head = _("Documents that were already Restored") + "<br><ul>"
+		for docname in invalid:
+			invalid_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
+		invalid_body = invalid_head + invalid_data + tail
+
+		failed_data = ""
+		failed_head = _("Documents that Failed to Restore") + "<br><ul>"
+		for docname in failed:
+			failed_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
+		failed_body = failed_head + failed_data + tail
+
+		summary = (restored_body if restored else "") + (invalid_body if invalid else "") + (failed_body if failed else "")
+		frappe.msgprint(summary, title="Document Restoration Summary", indicator="orange", is_minimizable=True)
 
 	return restored

From 9d24389c096f1dd2c4edfdfd588f9192bf94e437 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 24 Jul 2020 12:21:02 +0530
Subject: [PATCH 061/361] style: commonify summary generation

---
 .../deleted_document/deleted_document.py      | 35 +++++++------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index e637f4f000..d8f39ca139 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -44,9 +44,7 @@ def restore(name, alert=True):
 def bulk_restore(docnames):
 	docnames = frappe.parse_json(docnames)
 	message = _('Restoring Deleted Document')
-	restored = []
-	invalid = []
-	failed = []
+	restored, invalid, failed = [], [], []
 
 	for i, d in enumerate(docnames):
 		try:
@@ -60,32 +58,25 @@ def bulk_restore(docnames):
 			invalid.append(d)
 
 		except Exception:
+			frappe.message_log.pop()
 			failed.append(d)
 			frappe.db.rollback()
-			frappe.message_log.pop()
 
 	if failed or invalid:
-		tail = "</ul>"
+		def body(docnames):
+			href = "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>"
+			return "<br><ul>" + "".join([href.format(docname) for docname in docnames])
 
-		restored_data = ""
-		restored_head = _("Documents restored successfully") + "<br><ul>"
-		for docname in restored:
-			restored_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
-		restored_body = restored_head + restored_data + tail
+		def message(title, docnames):
+			if docnames:
+				return _(title) + body(docnames) + "</ul>"
+			return ""
 
-		invalid_data = ""
-		invalid_head = _("Documents that were already Restored") + "<br><ul>"
-		for docname in invalid:
-			invalid_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
-		invalid_body = invalid_head + invalid_data + tail
+		restored_summary = message("Documents restored successfully", restored)
+		invalid_summary = message("Documents that were already Restored", invalid)
+		failed_summary = message("Documents that Failed to Restore", failed)
 
-		failed_data = ""
-		failed_head = _("Documents that Failed to Restore") + "<br><ul>"
-		for docname in failed:
-			failed_data += "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>".format(docname)
-		failed_body = failed_head + failed_data + tail
-
-		summary = (restored_body if restored else "") + (invalid_body if invalid else "") + (failed_body if failed else "")
+		summary = restored_summary + invalid_summary + failed_summary
 		frappe.msgprint(summary, title="Document Restoration Summary", indicator="orange", is_minimizable=True)
 
 	return restored

From 35f3b5b4051b9d0054c540e93e04606937048977 Mon Sep 17 00:00:00 2001
From: Wolfram Schmidt <wolfram.schmidt@tueit.de>
Date: Fri, 24 Jul 2020 14:11:24 +0200
Subject: [PATCH 062/361] fix: Allowing to rename DocType Tag (#11100)

---
 frappe/desk/doctype/tag/tag.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/desk/doctype/tag/tag.json b/frappe/desk/doctype/tag/tag.json
index 895516594e..ad9838d10f 100644
--- a/frappe/desk/doctype/tag/tag.json
+++ b/frappe/desk/doctype/tag/tag.json
@@ -1,4 +1,5 @@
 {
+ "allow_rename": 1,
  "autoname": "Prompt",
  "creation": "2016-05-25 09:43:44.767581",
  "doctype": "DocType",
@@ -46,4 +47,4 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC"
-}
\ No newline at end of file
+}

From bc56a596715c47ed73cb2ad76db8e090ac8536fd Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 24 Jul 2020 20:19:35 +0530
Subject: [PATCH 063/361] fix(paytm-integration): simplify code and remove
 additional loop

---
 .../doctype/paytm_settings/paytm_settings.py  | 20 +++++++------------
 .../pages/integrations/paytm_checkout.py      |  2 --
 2 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index bfa9b6b3eb..a166d245f5 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -25,7 +25,7 @@ class PaytmSettings(Document):
 
 	def validate_transaction_currency(self, currency):
 		if currency not in self.supported_currencies:
-			frappe.throw(_("Please select another payment method. Stripe does not support transactions in currency '{0}'").format(currency))
+			frappe.throw(_("Please select another payment method. Paytm does not support transactions in currency '{0}'").format(currency))
 
 	def get_payment_url(self, **kwargs):
 		'''Return payment url with several params'''
@@ -87,29 +87,23 @@ def get_paytm_params(payment_details, order_id, paytm_config):
 def verify_transaction(**kwargs):
 	'''Verify checksum for received data in the callback and then verify the transaction'''
 	paytm_config = get_paytm_config()
-	received_data = frappe._dict(kwargs)
+	paytm_params = frappe._dict(kwargs)
 	is_valid_checksum = False
 
-	paytm_params = {}
-	for key, value in received_data.items(): 
-		if key == 'CHECKSUMHASH':
-			paytm_checksum = value
-		elif key == 'cmd':
-			continue
-		else:
-			paytm_params[key] = value
+	paytm_params.pop('cmd', None)
+	paytm_checksum = paytm_params.pop('CHECKSUMHASH', None)
 
 	if paytm_params and paytm_config and paytm_checksum:
 		# Verify checksum
 		is_valid_checksum = verifySignature(paytm_params, paytm_config.merchant_key, paytm_checksum)
 
-	if is_valid_checksum and received_data['RESPCODE'] == '01':
-		verify_transaction_status(paytm_config, received_data['ORDERID'])
+	if is_valid_checksum and paytm_params.get('RESPCODE') == '01':
+		verify_transaction_status(paytm_config, paytm_params['ORDERID'])
 	else:
 		frappe.respond_as_web_page("Payment Failed",
 			"Transaction failed to complete. In case of any deductions, deducted amount will get refunded to your account.",
 			http_status_code=401, indicator_color='red')
-		frappe.log_error("Order unsuccessful. Failed Response:"+cstr(received_data), 'Paytm Payment Failed')
+		frappe.log_error("Order unsuccessful. Failed Response:"+cstr(paytm_params), 'Paytm Payment Failed')
 
 def verify_transaction_status(paytm_config, order_id):
 	'''Verify transaction completion after checksum has been verified'''
diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py
index 3018a9d65b..bc385b5784 100644
--- a/frappe/templates/pages/integrations/paytm_checkout.py
+++ b/frappe/templates/pages/integrations/paytm_checkout.py
@@ -6,8 +6,6 @@ from frappe import _
 import json
 from frappe.integrations.doctype.paytm_settings.paytm_settings import get_paytm_params, get_paytm_config
 
-no_cache = 1
-
 def get_context(context):
 	context.no_cache = 1
 	paytm_config = get_paytm_config()

From ab8a63d6fe0fe892ac578edb7b4deeae29fcc4cf Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 24 Jul 2020 22:59:16 +0530
Subject: [PATCH 064/361] fix: minor changes

---
 frappe/integrations/doctype/paytm_settings/paytm_settings.py | 3 +--
 frappe/www/message.html                                      | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
index a166d245f5..616c3837d4 100644
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
@@ -84,10 +84,9 @@ def get_paytm_params(payment_details, order_id, paytm_config):
 	return paytm_params
 
 @frappe.whitelist(allow_guest=True)
-def verify_transaction(**kwargs):
+def verify_transaction(**paytm_params):
 	'''Verify checksum for received data in the callback and then verify the transaction'''
 	paytm_config = get_paytm_config()
-	paytm_params = frappe._dict(kwargs)
 	is_valid_checksum = False
 
 	paytm_params.pop('cmd', None)
diff --git a/frappe/www/message.html b/frappe/www/message.html
index 87d311d9cc..2919592901 100644
--- a/frappe/www/message.html
+++ b/frappe/www/message.html
@@ -24,7 +24,7 @@ html, body {
 		<span class='indicator {{ indicator_color or "blue" }}'>
 			{{ title or _("Message") }}</span>
 	</h5>
-	<div class="page-card-body ellipsis">
+	<div class="page-card-body">
 	{% block message_body %}
 		<p>{{ message or "" }}</p>
 		{% if primary_action %}

From 8844c2b690d2e6a1011963b1a63fd24c40445b22 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 15:37:01 +0530
Subject: [PATCH 065/361] fix: route to new doc 1 explicitly

---
 frappe/public/js/frappe/widgets/onboarding_widget.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js
index 9b4c9973e2..8c1d2cbb5b 100644
--- a/frappe/public/js/frappe/widgets/onboarding_widget.js
+++ b/frappe/public/js/frappe/widgets/onboarding_widget.js
@@ -136,7 +136,7 @@ export default class OnboardingWidget extends Widget {
 		if (step.is_single) {
 			route = `Form/${step.reference_document}`;
 		} else {
-			route = `Form/${step.reference_document}/${__('New')} ${__(step.reference_document)}`;
+			route = `Form/${step.reference_document}/${__('New')} ${__(step.reference_document)} 1`;
 		}
 
 		let current_route = frappe.get_route();
@@ -262,7 +262,7 @@ export default class OnboardingWidget extends Widget {
 			frappe.route_hooks.after_save = callback;
 		}
 
-		frappe.set_route(`Form/${step.reference_document}/${__('New')} ${__(step.reference_document)}`);
+		frappe.set_route(`Form/${step.reference_document}/${__('New')} ${__(step.reference_document)} 1`);
 	}
 
 	show_quick_entry(step) {

From af8771d5dccd8c882f74d5cca84e3275a6ae84a2 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Mon, 27 Jul 2020 15:51:44 +0530
Subject: [PATCH 066/361] fix: throw message for invalid date filter

---
 frappe/utils/data.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index d1b409d1fc..f36fa656b0 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -94,7 +94,10 @@ def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, se
 		as_string = True
 		if " " in date:
 			as_datetime = True
-		date = parser.parse(date)
+		try:
+			date = parser.parse(date)
+		except:
+			frappe.throw(frappe._("Please select a valid date filter"), title=frappe._("Invalid Date"))
 
 	date = date + relativedelta(years=years, months=months, weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)
 

From 28605191b1f1359b4b71ae9fd9435d47d26e9ad1 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Mon, 27 Jul 2020 16:59:49 +0530
Subject: [PATCH 067/361] fix(Number Card): return value in promise

---
 frappe/public/js/frappe/widgets/number_card_widget.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/number_card_widget.js b/frappe/public/js/frappe/widgets/number_card_widget.js
index 220394919c..6b38412ebd 100644
--- a/frappe/public/js/frappe/widgets/number_card_widget.js
+++ b/frappe/public/js/frappe/widgets/number_card_widget.js
@@ -164,7 +164,7 @@ export default class NumberCardWidget extends Widget {
 
 	get_number() {
 		return frappe.xcall(this.settings.method, this.settings.args).then(res => {
-			this.settings.get_number(res);
+			return this.settings.get_number(res);
 		});
 	}
 

From ad1152f8e8367f8d7c46d0e4e9a14ee9e2d0df9b Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 27 Jul 2020 17:13:18 +0530
Subject: [PATCH 068/361] feat: override site name via _site

added filter parameter for logger
---
 frappe/__init__.py     |  4 ++--
 frappe/utils/logger.py | 16 +++++++++-------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 8c24980189..504ac96eb9 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1559,10 +1559,10 @@ def get_doctype_app(doctype):
 
 loggers = {}
 log_level = None
-def logger(module=None, with_more_info=False, _site=None):
+def logger(module=None, with_more_info=False, _site=None, filter=None):
 	'''Returns a python logger that uses StreamHandler'''
 	from frappe.utils.logger import get_logger
-	return get_logger(module=module, with_more_info=with_more_info, _site=_site)
+	return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter)
 
 def log_error(message=None, title=_("Error")):
 	'''Log error to Error Log'''
diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 9254c0aedd..56fcf9857a 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -17,22 +17,20 @@ default_log_level = logging.DEBUG
 site = getattr(frappe.local, 'site', None)
 
 
-def get_logger(module, with_more_info=False, _site=None):
+def get_logger(module, with_more_info=False, _site=None, filter=None):
 	"""Application Logger for your given module
 
 	Args:
 		module (str): Name of your logger and consequently your log file.
 		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
-		_site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None.
+		_site (str, optional): Overrides the global site variable set in this scope. Defaults to None.
+		filter (function, optional): Add a filter function for your logger
 
 	Returns:
 		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
 	"""
 	global site
 
-	def allow_site():
-		return (_site and _site == site) or bool(site)
-
 	if module in frappe.loggers:
 		return frappe.loggers[module]
 
@@ -42,6 +40,7 @@ def get_logger(module, with_more_info=False, _site=None):
 
 	logfile = module + '.log'
 	site = getattr(frappe.local, 'site', None)
+	equivalent_site = _site or site
 	LOG_FILENAME = os.path.join('..', 'logs', logfile)
 
 	logger = logging.getLogger(module)
@@ -52,8 +51,8 @@ def get_logger(module, with_more_info=False, _site=None):
 	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
 	logger.addHandler(handler)
 
-	if allow_site():
-		SITELOG_FILENAME = os.path.join(site, 'logs', logfile)
+	if equivalent_site:
+		SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile)
 		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
 		site_handler.setFormatter(formatter)
 		logger.addHandler(site_handler)
@@ -61,6 +60,9 @@ def get_logger(module, with_more_info=False, _site=None):
 	if with_more_info:
 		handler.addFilter(SiteContextFilter())
 
+	if filter:
+		logger.addFilter(filter)
+
 	handler.setFormatter(formatter)
 
 	frappe.loggers[module] = logger

From 1345c2feb311bf7abb06b3f1969c4a2a43dc9b94 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 27 Jul 2020 17:31:12 +0530
Subject: [PATCH 069/361] feat: Allow_site flag in frappe.logger

---
 frappe/__init__.py     |  4 ++--
 frappe/app.py          |  2 +-
 frappe/utils/logger.py | 22 +++++++++++++---------
 3 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 504ac96eb9..7ea95ec3db 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1559,10 +1559,10 @@ def get_doctype_app(doctype):
 
 loggers = {}
 log_level = None
-def logger(module=None, with_more_info=False, _site=None, filter=None):
+def logger(module=None, with_more_info=False, allow_site=True, filter=None):
 	'''Returns a python logger that uses StreamHandler'''
 	from frappe.utils.logger import get_logger
-	return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter)
+	return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter)
 
 def log_error(message=None, title=_("Error")):
 	'''Log error to Error Log'''
diff --git a/frappe/app.py b/frappe/app.py
index 83a9b8c149..100b10e9ef 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -100,7 +100,7 @@ def application(request):
 		frappe.recorder.dump()
 
 		_site = get_site_name(request.host)
-		frappe.logger("frappe.web", _site=_site).info({
+		frappe.logger("frappe.web", allow_site=_site).info({
 			"site": _site,
 			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
 			"base_url": getattr(request, "base_url", "NOTFOUND"),
diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 56fcf9857a..e7058ce42d 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -14,22 +14,20 @@ import frappe
 
 
 default_log_level = logging.DEBUG
-site = getattr(frappe.local, 'site', None)
 
 
-def get_logger(module, with_more_info=False, _site=None, filter=None):
+def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	"""Application Logger for your given module
 
 	Args:
 		module (str): Name of your logger and consequently your log file.
 		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
-		_site (str, optional): Overrides the global site variable set in this scope. Defaults to None.
+		allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True.
 		filter (function, optional): Add a filter function for your logger
 
 	Returns:
 		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
 	"""
-	global site
 
 	if module in frappe.loggers:
 		return frappe.loggers[module]
@@ -39,8 +37,14 @@ def get_logger(module, with_more_info=False, _site=None, filter=None):
 		with_more_info = True
 
 	logfile = module + '.log'
-	site = getattr(frappe.local, 'site', None)
-	equivalent_site = _site or site
+
+	if allow_site == True:
+		site = getattr(frappe.local, 'site', None)
+	elif allow_site:
+		site = allow_site
+	else:
+		site = False
+
 	LOG_FILENAME = os.path.join('..', 'logs', logfile)
 
 	logger = logging.getLogger(module)
@@ -51,8 +55,8 @@ def get_logger(module, with_more_info=False, _site=None, filter=None):
 	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
 	logger.addHandler(handler)
 
-	if equivalent_site:
-		SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile)
+	if site:
+		SITELOG_FILENAME = os.path.join(site, 'logs', logfile)
 		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
 		site_handler.setFormatter(formatter)
 		logger.addHandler(site_handler)
@@ -73,7 +77,7 @@ class SiteContextFilter(logging.Filter):
 	"""This is a filter which injects request information (if available) into the log."""
 	def filter(self, record):
 		if "Form Dict" not in text_type(record.msg):
-			record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, getattr(frappe.local, 'form_dict', None))
+			record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None))
 			return True
 
 def set_log_level(level):

From 589a9b1d0f58e00fac8a6f4750157288363dbe02 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Mon, 27 Jul 2020 18:12:29 +0530
Subject: [PATCH 070/361] fix: OAUTH redirect fixes

---
 frappe/utils/oauth.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/frappe/utils/oauth.py b/frappe/utils/oauth.py
index 969623c369..b06227e289 100644
--- a/frappe/utils/oauth.py
+++ b/frappe/utils/oauth.py
@@ -213,6 +213,7 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No
 		redirect_post_login(
 			desk_user=frappe.local.response.get('message') == 'Logged In',
 			redirect_to=redirect_to,
+			provider=provider
 		)
 
 def update_oauth_user(user, data, provider):
@@ -300,12 +301,13 @@ def get_last_name(data):
 def get_email(data):
 	return data.get("email") or data.get("upn") or data.get("unique_name")
 
-def redirect_post_login(desk_user, redirect_to=None):
+def redirect_post_login(desk_user, redirect_to=None, provider=None):
 	# redirect!
 	frappe.local.response["type"] = "redirect"
 
 	if not redirect_to:
 		# the #desktop is added to prevent a facebook redirect bug
-		redirect_to = "/desk#desktop" if desk_user else "/me"
+		desk_uri = "/desk#desktop" if provider == 'facebook' else '/desk'
+		redirect_to = desk_uri if desk_user else "/me"
 
-	frappe.local.response["location"] = redirect_to
+	frappe.local.response["location"] = frappe.utils.get_url(redirect_to)

From dc46450f89a2d95163eda8002f047ba3e6b04630 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Mon, 27 Jul 2020 18:14:02 +0530
Subject: [PATCH 071/361] fix: push fieldtype in invalid conditions map

---
 frappe/public/js/frappe/ui/filters/filter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js
index bfc0b1449d..4dedfb32fe 100644
--- a/frappe/public/js/frappe/ui/filters/filter.js
+++ b/frappe/public/js/frappe/ui/filters/filter.js
@@ -57,7 +57,7 @@ frappe.ui.Filter = class {
 				this.conditions.push([key, __(`{0}`, [filter.label])]);
 				for (let fieldtype of Object.keys(this.invalid_condition_map)) {
 					if (!filter.valid_for_fieldtypes.includes(fieldtype)) {
-						this.invalid_condition_map[fieldtype].push(filter.label);
+						this.invalid_condition_map[fieldtype].push(key);
 					}
 				}
 			}

From 59ab9249cc3a9e7abeb5f6e599133f705ea6d8e6 Mon Sep 17 00:00:00 2001
From: Anupam K <anupamvns0099@gmail.com>
Date: Mon, 27 Jul 2020 19:40:11 +0530
Subject: [PATCH 072/361] fix: print format custom button

---
 frappe/printing/doctype/print_format/print_format.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js
index bb0ba27e38..e6599b2496 100644
--- a/frappe/printing/doctype/print_format/print_format.js
+++ b/frappe/printing/doctype/print_format/print_format.js
@@ -35,8 +35,8 @@ frappe.ui.form.on("Print Format", {
 			else if (frm.doc.custom_format && !frm.doc.raw_printing) {
 				frm.set_df_property("html", "reqd", 1);
 			}
-			frappe.db.get_value('DocType', frm.doc.doc_type, ['default_print_format', 'custom'], (r) => {
-				if (r.default_print_format != frm.doc.name && r.custom) {
+			frappe.db.get_value('DocType', frm.doc.doc_type, 'default_print_format', (r) => {
+				if (r.default_print_format != frm.doc.name) {
 					frm.add_custom_button(__("Set as Default"), function () {
 						frappe.call({
 							method: "frappe.printing.doctype.print_format.print_format.make_default",

From 10a5900ed3d792653623d97da4a3128737f03f63 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Mon, 27 Jul 2020 19:47:46 +0530
Subject: [PATCH 073/361] fix: Attach files to the document if they are set
 programmatically

---
 frappe/core/doctype/file/file.py | 37 ++++++++++++++++++++++++++++++++
 frappe/hooks.py                  |  3 ++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py
index 1748c60020..316915e43a 100755
--- a/frappe/core/doctype/file/file.py
+++ b/frappe/core/doctype/file/file.py
@@ -922,3 +922,40 @@ def update_existing_file_docs(doc):
 		content_hash=doc.content_hash,
 		file_name=doc.name
 	))
+
+def attach_files_to_document(doc, event):
+	""" Runs on on_update hook of all documents.
+	Goes through every Attach and Attach Image field and attaches
+	the file url to the document if it is not already attached.
+	"""
+
+	attach_fields = doc.meta.get(
+		"fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}
+	)
+
+	for df in attach_fields:
+		# this method runs in on_update hook of all documents
+		# we dont want the update to fail if file cannot be attached for some reason
+		try:
+			value = doc.get(df.fieldname)
+			if not value.startswith(("/files", "/private/files")):
+				return
+
+			if frappe.db.exists("File", {
+				"file_url": value,
+				"attached_to_name": doc.name,
+				"attached_to_doctype": doc.doctype,
+				"attached_to_field": df.fieldname,
+			}):
+				return
+
+			frappe.get_doc(
+				doctype="File",
+				file_url=value,
+				attached_to_name=doc.name,
+				attached_to_doctype=doc.doctype,
+				attached_to_field=df.fieldname,
+				folder="Home/Attachments",
+			).insert()
+		except Exception:
+			frappe.log_error(title=_("Error Attaching File"))
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 1f209f00a2..0a3d5d011c 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -132,7 +132,8 @@ doc_events = {
 			"frappe.core.doctype.activity_log.feed.update_feed",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.apply",
-			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone"
+			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone",
+			"frappe.core.doctype.file.file.attach_files_to_document"
 		],
 		"after_rename": "frappe.desk.notifications.clear_doctype_notifications",
 		"on_cancel": [

From 786625a92a94536b749f0b02acc9fc4cf56516c7 Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Mon, 27 Jul 2020 21:34:56 +0530
Subject: [PATCH 074/361] fix: regex for youtube video help link

---
 frappe/public/js/frappe/utils/help.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/utils/help.js b/frappe/public/js/frappe/utils/help.js
index 336d1ef526..f88849d212 100644
--- a/frappe/public/js/frappe/utils/help.js
+++ b/frappe/public/js/frappe/utils/help.js
@@ -17,7 +17,7 @@ frappe.help.show = function(doctype) {
 
 frappe.help.show_video = function(youtube_id, title) {
 	if (frappe.utils.is_url(youtube_id)) {
-		const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?/s]{11})';
+		const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?\\s]{11})';
 		youtube_id = youtube_id.match(expression)[1];
 	}
 

From c8817e52210f526c8b71a4989bc65ad05e2e4e79 Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Mon, 27 Jul 2020 21:39:00 +0530
Subject: [PATCH 075/361] fix: check for empty report filters while creating
 report type dashboard chart

---
 frappe/public/js/frappe/widgets/chart_widget.js | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/chart_widget.js b/frappe/public/js/frappe/widgets/chart_widget.js
index 4aa0b23a63..5d2c89b739 100644
--- a/frappe/public/js/frappe/widgets/chart_widget.js
+++ b/frappe/public/js/frappe/widgets/chart_widget.js
@@ -656,14 +656,15 @@ export default class ChartWidget extends Widget {
 	}
 
 	update_default_date_filters(report_filters, chart_filters) {
-		report_filters.map(f => {
-			if (['Date', 'DateRange'].includes(f.fieldtype) && f.default) {
-				if (f.reqd || chart_filters[f.fieldname]) {
-					chart_filters[f.fieldname] = f.default;
+		if (report_filters) {
+			report_filters.map(f => {
+				if (['Date', 'DateRange'].includes(f.fieldtype) && f.default) {
+					if (f.reqd || chart_filters[f.fieldname]) {
+						chart_filters[f.fieldname] = f.default;
+					}
 				}
-			}
-		});
-
+			});
+		}
 		return chart_filters;
 	}
 

From cb6208b19be27f72e05c017ef395588755a1aba1 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Mon, 27 Jul 2020 22:49:42 +0530
Subject: [PATCH 076/361] fix(field-group): strip html to check for null values

---
 frappe/public/js/frappe/ui/field_group.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js
index 4d827354d8..ec7234c9bb 100644
--- a/frappe/public/js/frappe/ui/field_group.js
+++ b/frappe/public/js/frappe/ui/field_group.js
@@ -86,7 +86,7 @@ frappe.ui.FieldGroup = frappe.ui.form.Layout.extend({
 			var f = this.fields_dict[key];
 			if(f.get_value) {
 				var v = f.get_value();
-				if(f.df.reqd && is_null(v))
+				if(f.df.reqd && is_null(strip_html(cstr(v))))
 					errors.push(__(f.df.label));
 
 				if(!is_null(v)) ret[f.df.fieldname] = v;

From a11496f05ca1270abf51574489dd19f0b6ffbb6f Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Mon, 27 Jul 2020 23:35:19 +0530
Subject: [PATCH 077/361] fix: translation syntax

---
 frappe/public/js/frappe/widgets/chart_widget.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/chart_widget.js b/frappe/public/js/frappe/widgets/chart_widget.js
index 5d2c89b739..ccec5b3ef4 100644
--- a/frappe/public/js/frappe/widgets/chart_widget.js
+++ b/frappe/public/js/frappe/widgets/chart_widget.js
@@ -615,9 +615,7 @@ export default class ChartWidget extends Widget {
 	}
 
 	update_last_synced() {
-		let last_synced_text = __("Last synced {0}", [
-			comment_when(this.chart_doc.last_synced_on)
-		]);
+		let last_synced_text = __("Last synced {0}", [comment_when(this.chart_doc.last_synced_on)]);
 		this.footer.html(last_synced_text);
 	}
 

From 7383a55c9adb9af62590ba18738609784b4ea8b3 Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Tue, 28 Jul 2020 11:29:22 +0530
Subject: [PATCH 078/361] fix: remove useless escape character

---
 frappe/public/js/frappe/utils/help.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/utils/help.js b/frappe/public/js/frappe/utils/help.js
index f88849d212..ec6f7c8158 100644
--- a/frappe/public/js/frappe/utils/help.js
+++ b/frappe/public/js/frappe/utils/help.js
@@ -17,7 +17,7 @@ frappe.help.show = function(doctype) {
 
 frappe.help.show_video = function(youtube_id, title) {
 	if (frappe.utils.is_url(youtube_id)) {
-		const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?\\s]{11})';
+		const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^"&?\\s]{11})';
 		youtube_id = youtube_id.match(expression)[1];
 	}
 

From d9c8b6b66281946ee504c240c84a8819e60bc903 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Tue, 28 Jul 2020 13:09:24 +0530
Subject: [PATCH 079/361] fix: Don't render meta tags if no value

---
 frappe/templates/includes/meta_block.html | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/templates/includes/meta_block.html b/frappe/templates/includes/meta_block.html
index 3d867aaef3..1aef8a160e 100644
--- a/frappe/templates/includes/meta_block.html
+++ b/frappe/templates/includes/meta_block.html
@@ -1,5 +1,8 @@
 {%- if metatags -%}
 {%- for name in metatags %}
-<meta {% if name.startswith('og:') %}property="{{ name }}"{% else %}name="{{ name }}"{% endif %} content="{{ metatags[name] | striptags | escape }}">
+{%- set content = metatags.get(name, '') -%}
+{%- if content -%}
+<meta {% if name.startswith('og:') %}property="{{ name }}"{% else %}name="{{ name }}"{% endif %} content="{{ content | striptags | escape }}">
+{%- endif -%}
 {%- endfor -%}
 {%- endif -%}

From 8ce340046ee27fccb1232e80d1aa3d3d5f8d4128 Mon Sep 17 00:00:00 2001
From: Sahil Khan <sahilkhan28297@gmail.com>
Date: Tue, 28 Jul 2020 19:33:44 +0530
Subject: [PATCH 080/361] fix: do not sanitize code field

Co-authored-by: Chinmay D. Pai <chinmaydpai@gmail.com>
---
 frappe/model/base_document.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index 7d56736cdc..09d303bec7 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -703,8 +703,8 @@ class BaseDocument(object):
 			sanitized_value = value
 
 			if df and (df.get("ignore_xss_filter")
-				or (df.get("fieldtype")=="Code" and df.get("options")!="Email")
-				or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode")
+				or (df.get("fieldtype") in ("Data", "Small Text", "Text") and df.get("options")=="Email")
+				or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code")
 
 				# cancelled and submit but not update after submit should be ignored
 				or self.docstatus==2

From ac2493b391a4213f3412390b88b157a2fcf2bbd7 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Wed, 29 Jul 2020 11:00:59 +0530
Subject: [PATCH 081/361] style: use 1tbs braces for if-else

---
 frappe/public/js/frappe/views/reports/query_report.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 609e1feb46..0701a34b76 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -869,8 +869,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 					if (column.colIndex === index && !value) {
 						value = "Total";
 						column.fieldtype = "Data"; // avoid type issues for value if Date column
-					}
-					else if (in_list(["Currency", "Float"], column.fieldtype)) {
+					} else if (in_list(["Currency", "Float"], column.fieldtype)) {
 						// proxy for currency and float
 						data = this.data[0];
 					}

From 1903e169dfa464704f2360bd7e1ccb8c03962424 Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Wed, 29 Jul 2020 11:16:24 +0530
Subject: [PATCH 082/361] fix: Update translations

ref: https://github.com/frappe/frappe/pull/11094#discussion_r462050750
---
 .../doctype/deleted_document/deleted_document_list.js  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index a49d8125e3..ac21d0ff42 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -13,12 +13,14 @@ frappe.listview_settings["Deleted Document"] = {
 						if (r.message) {
 							let num = r.message.length;
 							let message;
-							if (num === 1) {
-								message = "{0} Document was Restored";
+							if (num === 0) {
+								message = __("No Documents were Restored");
+							} else if (num === 1) {
+								message = __("Document was Restored");
 							} else {
-								message = "{0} Documents were Restored";
+								message = __("{0} Documents were Restored", [num]);
 							}
-							frappe.msgprint(__(message, [num]));
+							frappe.msgprint(message);
 							if (num > 0) {
 								doclist.refresh();
 							}

From ae3ee5b2eb5c64fa00f4461ad1a11841ced65c54 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 29 Jul 2020 13:33:04 +0530
Subject: [PATCH 083/361] fix: save frappe loggers site wise

---
 frappe/utils/logger.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index e7058ce42d..c38e0e6997 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -29,8 +29,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
 	"""
 
+	if allow_site == True:
+		site = getattr(frappe.local, 'site', None)
+	elif allow_site:
+		site = allow_site
+	else:
+		site = False
+
 	if module in frappe.loggers:
-		return frappe.loggers[module]
+		try:
+			return frappe.loggers[module][site or "all"]
+		except:
+			pass
 
 	if not module:
 		module = "frappe"
@@ -38,12 +48,6 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 
 	logfile = module + '.log'
 
-	if allow_site == True:
-		site = getattr(frappe.local, 'site', None)
-	elif allow_site:
-		site = allow_site
-	else:
-		site = False
 
 	LOG_FILENAME = os.path.join('..', 'logs', logfile)
 
@@ -69,7 +73,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 
 	handler.setFormatter(formatter)
 
-	frappe.loggers[module] = logger
+	try:
+		frappe.loggers[module][site or "all"] = logger
+	except KeyError:
+		frappe.loggers[module] = {}
+		frappe.loggers[module][site or "all"] = logger
 
 	return logger
 

From 727a2b13bf3050c8dd4e2bed9bf6f1911864dcd5 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 29 Jul 2020 14:28:45 +0530
Subject: [PATCH 084/361] fix: fetch latest backups

updated behaviour of fetch latest backups. doesnt take new backups
and accepts no args
---
 frappe/utils/backups.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 3b905de6bd..7eb00ceccd 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -208,26 +208,22 @@ def get_backup():
 
 
 @frappe.whitelist()
-def fetch_latest_backups(with_files=True, recent=3):
-	"""Takes backup on-demand if doesnt exist satisfying the `recent` parameter
+def fetch_latest_backups():
+	"""Fetches paths of the latest backup taken in the last 30 days
 	Only for: System Managers
 
-	Args:
-		with_files (bool, optional): If set, files will backuped up. Defaults to True.
-		recent (int, optional): Won't take a new backup if backup exists within this paramter. Defaults to 3 hours
-
 	Returns:
 		dict: relative Backup Paths
 	"""
 	frappe.only_for("System Manager")
 	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name, frappe.conf.db_password, db_host=frappe.db.host, db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
-	odb.get_backup(older_than=recent, ignore_files=not with_files)
+	database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
 
 	return {
-		"database": odb.backup_path_db,
-		"public": odb.backup_path_files,
-		"private": odb.backup_path_private_files,
-		"config": odb.site_config_backup_path
+		"database": database,
+		"public": public,
+		"private": private,
+		"config": config
 	}
 
 

From d3e4a97b6fc96f2003c1fad3c14a99b105ade5ed Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Wed, 29 Jul 2020 17:17:53 +0530
Subject: [PATCH 085/361] test: Make email account is email account test
 (#11139)

---
 frappe/core/doctype/server_script/test_server_script.py  | 7 ++++---
 frappe/email/doctype/email_account/test_email_account.py | 5 ++++-
 frappe/model/workflow.py                                 | 2 +-
 frappe/tests/test_naming.py                              | 1 +
 frappe/workflow/doctype/workflow/test_workflow.py        | 3 ++-
 5 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py
index 3e6b7a3a98..5c12858e8a 100644
--- a/frappe/core/doctype/server_script/test_server_script.py
+++ b/frappe/core/doctype/server_script/test_server_script.py
@@ -52,9 +52,10 @@ class TestServerScript(unittest.TestCase):
 
 		frappe.db.commit()
 
-	# @classmethod
-	# def tearDownClass(cls):
-	# 	frappe.db.sql('truncate `tabServer Script`')
+	@classmethod
+	def tearDownClass(cls):
+		frappe.db.commit()
+		frappe.db.sql('truncate `tabServer Script`')
 
 	def setUp(self):
 		frappe.cache().delete_value('server_script_map')
diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py
index 29b54d7f8b..f87ee32bb1 100644
--- a/frappe/email/doctype/email_account/test_email_account.py
+++ b/frappe/email/doctype/email_account/test_email_account.py
@@ -5,7 +5,10 @@ from __future__ import unicode_literals
 import frappe, os
 import unittest, email
 
-test_records = frappe.get_test_records('Email Account')
+from frappe.test_runner import make_test_records
+
+make_test_records("User")
+make_test_records("Email Account")
 
 from frappe.core.doctype.communication.email import make
 from frappe.desk.form.load import get_attachments
diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py
index ea563dfc13..32919b3333 100644
--- a/frappe/model/workflow.py
+++ b/frappe/model/workflow.py
@@ -307,4 +307,4 @@ def set_workflow_state_on_action(doc, workflow_name, action):
 	for state in workflow.states:
 		if state.doc_status == docstatus:
 			doc.set(workflow_state_field, state.state)
-			return
\ No newline at end of file
+			return
diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py
index 90ab6a6a94..b47fb809ca 100644
--- a/frappe/tests/test_naming.py
+++ b/frappe/tests/test_naming.py
@@ -88,6 +88,7 @@ class TestNaming(unittest.TestCase):
 		series = 'TEST-'
 		key = 'TEST-'
 		name = 'TEST-00003'
+		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
 		count = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 9999df40cb..2719bc7cf0 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -10,6 +10,7 @@ from frappe.model.workflow import apply_workflow, WorkflowTransitionError, Workf
 class TestWorkflow(unittest.TestCase):
 	def setUp(self):
 		frappe.db.sql('DELETE FROM `tabToDo`')
+		frappe.db.sql("DELETE FROM `tabHas Role` WHERE `role`='Test Approver'")
 		if not getattr(self, 'workflow', None):
 			self.workflow = create_todo_workflow()
 		frappe.set_user('Administrator')
@@ -146,4 +147,4 @@ def create_todo_workflow():
 		return workflow
 
 def create_new_todo():
-	return frappe.get_doc(dict(doctype='ToDo', description='workflow ' + random_string(10))).insert()
\ No newline at end of file
+	return frappe.get_doc(dict(doctype='ToDo', description='workflow ' + random_string(10))).insert()

From 3df3065c4e966690e8c0c88683581377ab0350fd Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Fri, 20 Mar 2020 20:58:43 +0530
Subject: [PATCH 086/361] fix: disable generate report button till report is
 generated

---
 .../js/frappe/views/reports/query_report.js   | 49 ++++++++++++-------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 53486ea3d6..f926fdb868 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -66,6 +66,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 	setup_events() {
 		frappe.realtime.on("report_generated", (data) => {
+			this.toggle_primary_button_disabled(false);
 			if(data.report_name) {
 				this.prepared_report_action = "Rebuild";
 				// If generated report and currently active Prepared Report has same fiters
@@ -660,29 +661,37 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		// 1. First time with given filters, no data.
 		// 2. Showing data from specific report
 		// 3. Showing data from an old report without specific report name
-		if(this.prepared_report_action == "New") {
-			this.page.set_primary_action(
-				__("Generate New Report"),
-				() => {
-					this.generate_background_report();
-				}
-			);
-		} else if(this.prepared_report_action == "Edit") {
-			this.page.set_primary_action(
-				__("Edit"),
-				() => {
-					frappe.set_route(frappe.get_route());
-				}
-			);
-		} else if(this.prepared_report_action == "Rebuild"){
-			this.page.set_primary_action(
-				__("Rebuild"),
-				this.generate_background_report.bind(this)
+		this.primary_action_map = {
+			"New": {
+				label: __("Generate New Report"),
+				action: () => this.generate_background_report()
+			},
+			"Edit": {
+				label: __("Edit"),
+				action: () => frappe.set_route(frappe.get_route())
+			},
+			"Rebuild": {
+				label:	__("Rebuild"),
+				action: () => this.generate_background_report()
+			}
+		}
+		let primary_action = this.primary_action_map[this.prepared_report_action];
+
+		if (!this.primary_button || this.primary_button.text() !== primary_action.label) {
+			this.primary_button = this.page.set_primary_action(
+				primary_action.label,
+				primary_action.action
 			);
 		}
 	}
 
+	toggle_primary_button_disabled(disable) {
+		this.primary_button.prop('disabled', disable);
+	}
+
 	generate_background_report() {
+		this.toggle_primary_button_disabled(true);
+
 		let mandatory = this.filters.filter(f => f.df.reqd);
 		let missing_mandatory = mandatory.filter(f => !f.get_value());
 		if (!missing_mandatory.length){
@@ -1619,7 +1628,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 
 		if (flag && this.prepared_report) {
 			this.prepared_report_action = "New";
-			this.add_prepared_report_buttons();
+			if (!this.primary_button.is(':visible')) {
+				this.add_prepared_report_buttons();
+			}
 		}
 	}
 

From f3b8b9bf5047feab14864e0b01032269b5af1bd0 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Fri, 5 Jun 2020 13:33:18 +0530
Subject: [PATCH 087/361] fix: don't return response

---
 .../public/js/frappe/views/reports/query_report.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index f926fdb868..1cf9faf96a 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -664,15 +664,21 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		this.primary_action_map = {
 			"New": {
 				label: __("Generate New Report"),
-				action: () => this.generate_background_report()
+				click: () => {
+					this.generate_background_report()
+				},
 			},
 			"Edit": {
 				label: __("Edit"),
-				action: () => frappe.set_route(frappe.get_route())
+				click: () => {
+					frappe.set_route(frappe.get_route())
+				}
 			},
 			"Rebuild": {
 				label:	__("Rebuild"),
-				action: () => this.generate_background_report()
+				click: () => {
+					this.generate_background_report()
+				}
 			}
 		}
 		let primary_action = this.primary_action_map[this.prepared_report_action];
@@ -680,7 +686,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		if (!this.primary_button || this.primary_button.text() !== primary_action.label) {
 			this.primary_button = this.page.set_primary_action(
 				primary_action.label,
-				primary_action.action
+				primary_action.click
 			);
 		}
 	}

From 212ee269d6b1f2309c985afd8fa4d4b8ab25adaf Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 17:31:07 +0530
Subject: [PATCH 088/361] feat(Prepared Report): system setting to
 automatically delete prepared reports

---
 .../system_settings/system_settings.js        |  8 ++++++
 .../system_settings/system_settings.json      | 26 +++++++++++++++----
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js
index aa60ea7e95..d2fe3cba69 100644
--- a/frappe/core/doctype/system_settings/system_settings.js
+++ b/frappe/core/doctype/system_settings/system_settings.js
@@ -27,3 +27,11 @@ frappe.ui.form.on("System Settings", "enable_two_factor_auth", function(frm) {
 		frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
 	}
 });
+
+frappe.ui.form.on("System Settings", "enable_prepared_report_auto_deletion", function(frm) {
+	if(frm.doc.enable_prepared_report_auto_deletion){
+		if (!frm.doc.prepared_report_expiry_period) {
+			frm.set_value('prepared_report_expiry_period', 30);
+		}
+	}
+});
diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index b2cb67dbc9..ffc353a6ea 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -59,7 +59,9 @@
   "column_break_18",
   "disable_standard_email_footer",
   "hide_footer_in_auto_email_reports",
-  "attach_view_link",
+  "prepared_report_section",
+  "enable_prepared_report_auto_deletion",
+  "prepared_report_expiry_period",
   "chat",
   "enable_chat",
   "use_socketio_to_upload_file"
@@ -425,16 +427,30 @@
    "label": "Enable Onboarding"
   },
   {
-   "default": "1",
-   "fieldname": "attach_view_link",
+   "default": "30",
+   "depends_on": "enable_prepared_report_auto_deletion",
+   "description": "System will automatically delete Prepared Reports after these many days since creation",
+   "fieldname": "prepared_report_expiry_period",
+   "fieldtype": "Int",
+   "label": "Prepared Report Expiry Period (Days)"
+  },
+  {
+   "default": "0",
+   "fieldname": "enable_prepared_report_auto_deletion",
    "fieldtype": "Check",
-   "label": "Send document Web View link in email"
+   "label": "Enable Auto-deletion of Prepared Reports"
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "prepared_report_section",
+   "fieldtype": "Section Break",
+   "label": "Prepared Report"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2020-07-02 16:13:00.166382",
+ "modified": "2020-07-26 15:57:34.197718",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

From a19d045f885818408b75ee13bc187a15ce13c9fd Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 17:33:50 +0530
Subject: [PATCH 089/361] feat: add scheduler event to delete old prepared
 reports

---
 .../prepared_report/prepared_report.py        | 34 +++++++++++++++++++
 frappe/hooks.py                               |  3 +-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index 4e87bb92dd..2c02d99dad 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -69,6 +69,40 @@ def run_background(prepared_report):
 		user=frappe.session.user
 	)
 
+@frappe.whitelist()
+def get_reports_in_queued_state(report_name, filters):
+	reports = frappe.get_all('Prepared Report',
+		filters = {
+			'report_name': report_name,
+			'filters': json.dumps(json.loads(filters)),
+			'status': 'Queued'
+		})
+	return reports
+
+def delete_expired_prepared_reports():
+	system_settings = frappe.get_single('System Settings')
+	enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion
+	if enable_auto_deletion:
+		expiry_period = system_settings.prepared_report_expiry_period
+		prepared_reports_to_delete = frappe.get_all('Prepared Report',
+			filters = {
+				'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
+			})
+
+		args = {
+			'reports': prepared_reports_to_delete,
+			'limit': 50
+		}
+
+		enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
+
+@frappe.whitelist()
+def delete_prepared_reports(reports, limit=None):
+	reports = frappe.parse_json(reports)
+	for index, doc in enumerate(reports):
+		if limit and index == limit:
+			return
+		frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
 
 def create_json_gz_file(data, dt, dn):
 	# Storing data in CSV file causes information loss
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 1f209f00a2..132d54150f 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -212,7 +212,8 @@ scheduler_events = {
 		"frappe.integrations.doctype.google_contacts.google_contacts.sync",
 		"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
 		"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
-		"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails"
+		"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
+		"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports"
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",

From c5e988dcf1d60492d26c905739b0410a31e23be1 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 17:34:15 +0530
Subject: [PATCH 090/361] feat: show warning dialog if reports are already
 queued

---
 frappe/public/js/frappe/ui/messages.js        | 12 ++--
 .../js/frappe/views/reports/query_report.js   | 72 +++++++++++++++++--
 2 files changed, 74 insertions(+), 10 deletions(-)

diff --git a/frappe/public/js/frappe/ui/messages.js b/frappe/public/js/frappe/ui/messages.js
index eb5f4f09a2..c37cc41650 100644
--- a/frappe/public/js/frappe/ui/messages.js
+++ b/frappe/public/js/frappe/ui/messages.js
@@ -53,7 +53,7 @@ frappe.confirm = function(message, ifyes, ifno) {
 	return d;
 }
 
-frappe.warn = function(title, message_html, proceed_action, primary_label) {
+frappe.warn = function(title, message_html, proceed_action, primary_label, is_minimizable) {
 	const d = new frappe.ui.Dialog({
 		title: title,
 		indicator: 'red',
@@ -70,11 +70,15 @@ frappe.warn = function(title, message_html, proceed_action, primary_label) {
 			d.hide();
 		},
 		secondary_action_label: __("Cancel"),
+		minimizable: is_minimizable
 	});
 
-	d.buttons.find('.btn-primary').removeClass('btn-primary').addClass('btn-danger');
-	const modal_footer = $(`<div class="modal-footer"></div>`).insertAfter($(d.modal_body));
-	modal_footer.html(d.buttons);
+	d.footer = $(`<div class="modal-footer"></div>`).insertAfter($(d.modal_body));
+
+	d.get_close_btn().appendTo(d.footer);
+	d.get_primary_btn().appendTo(d.footer);
+
+	d.footer.find('.btn-primary').removeClass('btn-primary').addClass('btn-danger');
 
 	d.show();
 	return d;
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 1cf9faf96a..30e2021045 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -665,19 +665,19 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			"New": {
 				label: __("Generate New Report"),
 				click: () => {
-					this.generate_background_report()
+					this.show_warning_or_generate_report();
 				},
 			},
 			"Edit": {
 				label: __("Edit"),
 				click: () => {
-					frappe.set_route(frappe.get_route())
+					frappe.set_route(frappe.get_route());
 				}
 			},
 			"Rebuild": {
 				label:	__("Rebuild"),
 				click: () => {
-					this.generate_background_report()
+					this.show_warning_or_generate_report();
 				}
 			}
 		}
@@ -695,9 +695,69 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		this.primary_button.prop('disabled', disable);
 	}
 
+	show_warning_or_generate_report() {
+		frappe.xcall(
+			'frappe.core.doctype.prepared_report.prepared_report.get_reports_in_queued_state',
+			{
+				filters: this.get_filter_values(),
+				report_name: this.report_name,
+			}
+		).then(reports => {
+			this.queued_prepared_reports = reports;
+
+			if (reports.length) {
+				const message = this.get_queued_prepared_reports_warning_message(reports);
+				this.prepared_report_dialog = frappe.warn(
+					__('Reports already in Queue'),
+					message,
+					() => this.generate_background_report(),
+					__('Proceed Anyway'),
+					true
+				);
+
+				this.prepared_report_dialog.footer.prepend(`
+					<button type="button" class="btn btn-sm btn-default pull-left" data-action="delete_old_queued_reports">
+						${__('Delete and Generate New')}
+					</button>`);
+
+				frappe.utils.bind_actions_with_object(this.prepared_report_dialog.wrapper, this);
+			} else {
+				this.generate_background_report();
+			}
+		});
+	}
+
+	get_queued_prepared_reports_warning_message(reports) {
+		const route = `#List/Prepared Report/List?status=Queued&report_name=${this.report_name}`;
+		const no_of_reports_html = reports.length == 1
+			? `${__('There is already ')}<a class="underline" href="${route}">${__('1 Report')}</a>`
+			: `${__('There are already ')}<a class="underline" href="${route}">${__(`${reports.length} Reports`)}</a>`;
+
+		let warning_message = `
+			<p>
+				${__(`Are you sure you want to generate a new report?
+					${no_of_reports_html} with the same filters already in the queue:`)}
+			</p>`;
+
+		let get_item_html = item => `<a class="underline" href="#Form/Prepared Report/${item.name}">${item.name}</a>`;
+
+		warning_message += reports.map(get_item_html).join(', ');
+
+		return warning_message;
+	}
+
+	delete_old_queued_reports() {
+		this.prepared_report_dialog.hide();
+		frappe.xcall(
+			'frappe.core.doctype.prepared_report.prepared_report.delete_prepared_reports',
+			{
+				reports: this.queued_prepared_reports,
+			}
+		).then(() => this.generate_background_report());
+	}
+
 	generate_background_report() {
 		this.toggle_primary_button_disabled(true);
-
 		let mandatory = this.filters.filter(f => f.df.reqd);
 		let missing_mandatory = mandatory.filter(f => !f.get_value());
 		if (!missing_mandatory.length){
@@ -715,8 +775,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				// Rememeber the name of Prepared Report doc
 				this.prepared_report_doc_name = data.name;
 				let alert_message = `Report initiated. You can track its status
-					<a class='text-info' href='#Form/Prepared Report/${data.name}'>here</a>`;
-				frappe.show_alert({message: alert_message, indicator: 'orange'});
+					<a class="bold" href='#Form/Prepared Report/${data.name}'>here</a>`;
+				frappe.show_alert({message: alert_message, indicator: 'orange'}, 10);
 				this.toggle_nothing_to_show(true);
 			});
 		}

From 7bfc8a9b874e56bf5fc0b5864e3e0e9b53bf0f6d Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 17:49:26 +0530
Subject: [PATCH 091/361] style: fix formatting

---
 frappe/core/doctype/system_settings/system_settings.js | 2 +-
 frappe/public/js/frappe/form/toolbar.js                | 4 ++--
 frappe/public/js/frappe/views/reports/query_report.js  | 3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js
index d2fe3cba69..eebc984a2f 100644
--- a/frappe/core/doctype/system_settings/system_settings.js
+++ b/frappe/core/doctype/system_settings/system_settings.js
@@ -29,7 +29,7 @@ frappe.ui.form.on("System Settings", "enable_two_factor_auth", function(frm) {
 });
 
 frappe.ui.form.on("System Settings", "enable_prepared_report_auto_deletion", function(frm) {
-	if(frm.doc.enable_prepared_report_auto_deletion){
+	if (frm.doc.enable_prepared_report_auto_deletion) {
 		if (!frm.doc.prepared_report_expiry_period) {
 			frm.set_value('prepared_report_expiry_period', 30);
 		}
diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js
index 01e8fdbb98..be69775b26 100644
--- a/frappe/public/js/frappe/form/toolbar.js
+++ b/frappe/public/js/frappe/form/toolbar.js
@@ -185,8 +185,8 @@ frappe.ui.form.Toolbar = Class.extend({
 		return this.page.add_dropdown(label);
 	},
 	set_indicator: function() {
-		if(this.frm.save_disabled)
-			return;
+		// if(this.frm.save_disabled)
+		// 	return;
 
 		var indicator = frappe.get_indicator(this.frm.doc);
 		if(indicator) {
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 30e2021045..9f2378e109 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -680,7 +680,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 					this.show_warning_or_generate_report();
 				}
 			}
-		}
+		};
+
 		let primary_action = this.primary_action_map[this.prepared_report_action];
 
 		if (!this.primary_button || this.primary_button.text() !== primary_action.label) {

From 6ca654c93c3e0c3ac9c0d9fe3497dfe24026a61a Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 18:28:47 +0530
Subject: [PATCH 092/361] fix: revert removed field attach_view_link

---
 frappe/core/doctype/system_settings/system_settings.json | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index ffc353a6ea..45ab84e62b 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -59,6 +59,7 @@
   "column_break_18",
   "disable_standard_email_footer",
   "hide_footer_in_auto_email_reports",
+  "attach_view_link",
   "prepared_report_section",
   "enable_prepared_report_auto_deletion",
   "prepared_report_expiry_period",
@@ -426,6 +427,12 @@
    "fieldtype": "Check",
    "label": "Enable Onboarding"
   },
+  {
+   "default": "1",
+   "fieldname": "attach_view_link",
+   "fieldtype": "Check",
+   "label": "Send document Web View link in email"
+  },
   {
    "default": "30",
    "depends_on": "enable_prepared_report_auto_deletion",
@@ -450,7 +457,7 @@
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2020-07-26 15:57:34.197718",
+ "modified": "2020-07-26 17:58:34.197718",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

From 02a9a3220b81400cf97e4949ec0fd179ffeb33d3 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 29 Jul 2020 19:27:23 +0530
Subject: [PATCH 093/361] fix: fix message

---
 frappe/public/js/frappe/views/reports/query_report.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 9f2378e109..83926c18be 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -731,8 +731,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 	get_queued_prepared_reports_warning_message(reports) {
 		const route = `#List/Prepared Report/List?status=Queued&report_name=${this.report_name}`;
 		const no_of_reports_html = reports.length == 1
-			? `${__('There is already ')}<a class="underline" href="${route}">${__('1 Report')}</a>`
-			: `${__('There are already ')}<a class="underline" href="${route}">${__(`${reports.length} Reports`)}</a>`;
+			? `${__('There is ')}<a class="underline" href="${route}">${__('1 Report')}</a>`
+			: `${__('There are ')}<a class="underline" href="${route}">${__(`${reports.length} Reports`)}</a>`;
 
 		let warning_message = `
 			<p>

From 219fbc8e88d8505391fe8259b0fbeda86844f42d Mon Sep 17 00:00:00 2001
From: Matteo D <md@lab51.org>
Date: Wed, 29 Jul 2020 17:28:46 +0200
Subject: [PATCH 094/361] fix: Added AWS S3 missing regions (#11109)

---
 .gitignore                                                | 5 ++++-
 .../doctype/s3_backup_settings/s3_backup_settings.json    | 8 ++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7df673c1f1..900ae1c7b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -188,4 +188,7 @@ typings/
 
 # cypress
 cypress/screenshots
-cypress/videos
\ No newline at end of file
+cypress/videos
+
+# JetBrains IDEs
+.idea/
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
index 830afbae53..123bb21e88 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
@@ -74,11 +74,11 @@
   },
   {
    "default": "us-east-1",
-   "description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.",
+   "description": "See https://docs.aws.amazon.com/general/latest/gr/s3.html for details.",
    "fieldname": "region",
    "fieldtype": "Select",
    "label": "Region",
-   "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1"
+   "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\naf-south-1\nap-east-1\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-south-1\neu-north-1\nme-south-1\nsa-east-1"
   },
   {
    "fieldname": "endpoint_url",
@@ -151,7 +151,7 @@
  "hide_toolbar": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-04-13 20:57:24.432183",
+ "modified": "2020-07-27 17:27:21.400000",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "S3 Backup Settings",
@@ -172,4 +172,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
\ No newline at end of file
+}

From 8d18fb43230c80b40c00732850b5631c4346d54a Mon Sep 17 00:00:00 2001
From: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
Date: Wed, 29 Jul 2020 17:48:39 +0200
Subject: [PATCH 095/361] fix(oauth provider): parse cookies correctly (#11066)

---
 frappe/oauth.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/frappe/oauth.py b/frappe/oauth.py
index 4dc50366be..122c806072 100644
--- a/frappe/oauth.py
+++ b/frappe/oauth.py
@@ -4,6 +4,7 @@ import pytz
 
 from frappe import _
 from frappe.auth import LoginManager
+from http import cookies
 from oauthlib.oauth2.rfc6749.tokens import BearerToken
 from oauthlib.oauth2.rfc6749.grant_types import AuthorizationCodeGrant, ImplicitGrant, ResourceOwnerPasswordCredentialsGrant, ClientCredentialsGrant,  RefreshTokenGrant
 from oauthlib.oauth2 import RequestValidator
@@ -130,15 +131,12 @@ class OAuthWebRequestValidator(RequestValidator):
 		oac.scopes = get_url_delimiter().join(request.scopes)
 		oac.redirect_uri_bound_to_authorization_code = request.redirect_uri
 		oac.client = client_id
-		oac.user = unquote(cookie_dict['user_id'])
+		oac.user = unquote(cookie_dict['user_id'].value)
 		oac.authorization_code = code['code']
 		oac.save(ignore_permissions=True)
 		frappe.db.commit()
 
 	def authenticate_client(self, request, *args, **kwargs):
-
-		cookie_dict = get_cookie_dict_from_headers(request)
-
 		#Get ClientID in URL
 		if request.client_id:
 			oc = frappe.get_doc("OAuth Client", request.client_id)
@@ -155,7 +153,9 @@ class OAuthWebRequestValidator(RequestValidator):
 		except Exception as e:
 			print("Failed body authentication: Application %s does not exist".format(cid=request.client_id))
 
-		return frappe.session.user == unquote(cookie_dict.get('user_id', "Guest"))
+		cookie_dict = get_cookie_dict_from_headers(request)
+		user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest"
+		return frappe.session.user == user_id
 
 	def authenticate_client_id(self, client_id, request, *args, **kwargs):
 		cli_id = frappe.db.get_value('OAuth Client', client_id, 'name')
@@ -400,13 +400,10 @@ class OAuthWebRequestValidator(RequestValidator):
 		return True
 
 def get_cookie_dict_from_headers(r):
+	cookie = cookies.BaseCookie()
 	if r.headers.get('Cookie'):
-		cookie = r.headers.get('Cookie')
-		cookie = cookie.split("; ")
-		cookie_dict = {k:v for k,v in (x.split('=') for x in cookie)}
-		return cookie_dict
-	else:
-		return {}
+		cookie.load(r.headers.get('Cookie'))
+	return cookie
 
 def calculate_at_hash(access_token, hash_alg):
 	"""Helper method for calculating an access token

From 9cef385005e68221cc2d4b628f9080b81a434da2 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 30 Jul 2020 14:37:22 +0530
Subject: [PATCH 096/361] refactor: refer to frappe.boot.single_types instead
 of db call

---
 .../public/js/frappe/widgets/widget_dialog.js  | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index 054159116f..1089d7e629 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -174,18 +174,12 @@ class ShortcutDialog extends WidgetDialog {
 				onchange: () => {
 					if (this.dialog.get_value("type") == "DocType") {
 						let doctype = this.dialog.get_value("link_to");
-
-						doctype &&
-							frappe.db
-								.get_value("DocType", doctype, "issingle")
-								.then((res) => {
-									if (res.message && res.message.issingle) {
-										this.hide_filters();
-									} else {
-										this.setup_filter(doctype);
-										this.show_filters();
-									}
-								});
+						if (frappe.boot.single_types.includes("Stock Settings")) {
+							this.hide_filters();
+						} else {
+							this.setup_filter(doctype);
+							this.show_filters();
+						}
 					} else {
 						this.hide_filters();
 					}

From 2ad15ee9f75dca2391e45b83f5298b1e7e90e163 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Thu, 30 Jul 2020 14:57:01 +0530
Subject: [PATCH 097/361] fix: use doctype variable

---
 frappe/public/js/frappe/widgets/widget_dialog.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index 1089d7e629..e3b4cc1d8c 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -174,7 +174,7 @@ class ShortcutDialog extends WidgetDialog {
 				onchange: () => {
 					if (this.dialog.get_value("type") == "DocType") {
 						let doctype = this.dialog.get_value("link_to");
-						if (frappe.boot.single_types.includes("Stock Settings")) {
+						if (frappe.boot.single_types.includes(doctype)) {
 							this.hide_filters();
 						} else {
 							this.setup_filter(doctype);

From 7427fb9cd3b9589d8366f330b968a9f5e8c6f310 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Thu, 30 Jul 2020 15:56:38 +0530
Subject: [PATCH 098/361] fix: Dont validate skipped columns

---
 frappe/core/doctype/data_import/importer.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 910e42af1a..14626eb5e3 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -955,6 +955,9 @@ class Column:
 		if not self.df:
 			return
 
+		if self.skip_import:
+			return
+
 		if self.df.fieldtype == 'Link':
 			# find all values that dont exist
 			values = list(set([cstr(v) for v in self.column_values[1:] if v]))

From 84a15407a733a57ce5d1b13c2c1fefd80f12b198 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Thu, 30 Jul 2020 16:06:10 +0530
Subject: [PATCH 099/361] fix: Respect hostname config in sitemap.xml

---
 frappe/www/sitemap.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/www/sitemap.py b/frappe/www/sitemap.py
index 8b93270ab5..f8f03c45f8 100644
--- a/frappe/www/sitemap.py
+++ b/frappe/www/sitemap.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 import frappe
 from frappe.model.document import get_controller
-from frappe.utils import get_request_site_address, get_datetime, nowdate
+from frappe.utils import get_datetime, nowdate, get_url
 from frappe.website.router import get_pages, get_all_page_context_from_doctypes
 from six import iteritems
 from six.moves.urllib.parse import quote, urljoin
@@ -25,13 +25,13 @@ def get_context(context):
 	for route, page in iteritems(get_pages()):
 		if page.sitemap:
 			links.append({
-				"loc": urljoin(host, quote(page.name.encode("utf-8"))),
+				"loc": get_url(quote(page.name.encode("utf-8"))),
 				"lastmod": nowdate()
 			})
 
 	for route, data in iteritems(get_public_pages_from_doctypes()):
 		links.append({
-			"loc": urljoin(host, quote((route or "").encode("utf-8"))),
+			"loc": get_url(quote((route or "").encode("utf-8"))),
 			"lastmod": get_datetime(data.get("modified")).strftime("%Y-%m-%d")
 		})
 

From 7bf69c62c1cc942d394d40484502c51b96e98b9f Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 30 Jul 2020 17:47:01 +0530
Subject: [PATCH 100/361] fix: suffix logger name with site

---
 frappe/utils/logger.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index c38e0e6997..1fbd54dd38 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -36,9 +36,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	else:
 		site = False
 
-	if module in frappe.loggers:
+	LOGGER_NAME = "{}-{}".format(module, site or "all")
+
+	if LOGGER_NAME in frappe.loggers:
 		try:
-			return frappe.loggers[module][site or "all"]
+			return frappe.loggers[LOGGER_NAME]
 		except:
 			pass
 
@@ -51,7 +53,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 
 	LOG_FILENAME = os.path.join('..', 'logs', logfile)
 
-	logger = logging.getLogger(module)
+	logger = logging.getLogger(LOGGER_NAME)
 	logger.setLevel(frappe.log_level or default_log_level)
 	logger.propagate = False
 
@@ -73,11 +75,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 
 	handler.setFormatter(formatter)
 
-	try:
-		frappe.loggers[module][site or "all"] = logger
-	except KeyError:
-		frappe.loggers[module] = {}
-		frappe.loggers[module][site or "all"] = logger
+	frappe.loggers[LOGGER_NAME] = logger
 
 	return logger
 

From 090b81f2f268447581eac496bb245af3a17224c5 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 30 Jul 2020 17:59:01 +0530
Subject: [PATCH 101/361] style: black + restructure

---
 frappe/utils/logger.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 1fbd54dd38..79ea93e3bd 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -30,7 +30,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	"""
 
 	if allow_site == True:
-		site = getattr(frappe.local, 'site', None)
+		site = getattr(frappe.local, "site", None)
 	elif allow_site:
 		site = allow_site
 	else:
@@ -48,21 +48,21 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 		module = "frappe"
 		with_more_info = True
 
-	logfile = module + '.log'
+	logfile = module + ".log"
 
-
-	LOG_FILENAME = os.path.join('..', 'logs', logfile)
+	LOG_FILENAME = os.path.join("..", "logs", logfile)
 
 	logger = logging.getLogger(LOGGER_NAME)
 	logger.setLevel(frappe.log_level or default_log_level)
 	logger.propagate = False
 
-	formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s')
+	formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
 	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
+	handler.setFormatter(formatter)
 	logger.addHandler(handler)
 
 	if site:
-		SITELOG_FILENAME = os.path.join(site, 'logs', logfile)
+		SITELOG_FILENAME = os.path.join(site, "logs", logfile)
 		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
 		site_handler.setFormatter(formatter)
 		logger.addHandler(site_handler)
@@ -73,20 +73,23 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	if filter:
 		logger.addFilter(filter)
 
-	handler.setFormatter(formatter)
-
 	frappe.loggers[LOGGER_NAME] = logger
 
 	return logger
 
+
 class SiteContextFilter(logging.Filter):
 	"""This is a filter which injects request information (if available) into the log."""
+
 	def filter(self, record):
 		if "Form Dict" not in text_type(record.msg):
-			record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None))
+			site = getattr(frappe.local, "site", None)
+			form_dict = getattr(frappe.local, "form_dict", None)
+			record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, form_dict)
 			return True
 
+
 def set_log_level(level):
-	'''Use this method to set log level to something other than the default DEBUG'''
-	frappe.log_level = getattr(logging, (level or '').upper(), None) or default_log_level
+	"""Use this method to set log level to something other than the default DEBUG"""
+	frappe.log_level = getattr(logging, (level or "").upper(), None) or default_log_level
 	frappe.loggers = {}

From fac7f359ae0b49064003e7424989579d580b7535 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 30 Jul 2020 18:49:59 +0530
Subject: [PATCH 102/361] fix: original log format + EAFP

---
 frappe/utils/logger.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 79ea93e3bd..46269fe909 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -38,26 +38,24 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 
 	LOGGER_NAME = "{}-{}".format(module, site or "all")
 
-	if LOGGER_NAME in frappe.loggers:
-		try:
-			return frappe.loggers[LOGGER_NAME]
-		except:
-			pass
+	try:
+		return frappe.loggers[LOGGER_NAME]
+	except KeyError:
+		pass
 
 	if not module:
 		module = "frappe"
 		with_more_info = True
 
 	logfile = module + ".log"
-
-	LOG_FILENAME = os.path.join("..", "logs", logfile)
+	log_filename = os.path.join("..", "logs", logfile)
 
 	logger = logging.getLogger(LOGGER_NAME)
 	logger.setLevel(frappe.log_level or default_log_level)
 	logger.propagate = False
 
-	formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
-	handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20)
+	formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module))
+	handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20)
 	handler.setFormatter(formatter)
 	logger.addHandler(handler)
 

From faa9a5466eacc4d07e19286309f36aeb8468f7ec Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 30 Jul 2020 19:04:04 +0530
Subject: [PATCH 103/361] fix: Slider

---
 frappe/utils/logger.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 46269fe909..6d24eef9d3 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -29,7 +29,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
 	"""
 
-	if allow_site == True:
+	if allow_site is True:
 		site = getattr(frappe.local, "site", None)
 	elif allow_site:
 		site = allow_site

From 444e8225a46188137028f99cb12efcab195559b5 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 30 Jul 2020 19:50:36 +0530
Subject: [PATCH 104/361] fix: setup filters only if doctype is available

---
 frappe/public/js/frappe/widgets/widget_dialog.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index e3b4cc1d8c..2ca26d59e1 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -174,9 +174,9 @@ class ShortcutDialog extends WidgetDialog {
 				onchange: () => {
 					if (this.dialog.get_value("type") == "DocType") {
 						let doctype = this.dialog.get_value("link_to");
-						if (frappe.boot.single_types.includes(doctype)) {
+						if (doctype && frappe.boot.single_types.includes(doctype)) {
 							this.hide_filters();
-						} else {
+						} else if (doctype) {
 							this.setup_filter(doctype);
 							this.show_filters();
 						}

From 0c16533cd21eb5ee3b95297fcab32cb7e7dd70f6 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Fri, 31 Jul 2020 12:32:17 +0530
Subject: [PATCH 105/361] fix(field-group): consider mandatory separately for
 text editor

---
 frappe/public/js/frappe/ui/field_group.js | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js
index ec7234c9bb..d432e553f1 100644
--- a/frappe/public/js/frappe/ui/field_group.js
+++ b/frappe/public/js/frappe/ui/field_group.js
@@ -82,17 +82,22 @@ frappe.ui.FieldGroup = frappe.ui.form.Layout.extend({
 	get_values: function(ignore_errors) {
 		var ret = {};
 		var errors = [];
-		for(var key in this.fields_dict) {
+		for (var key in this.fields_dict) {
 			var f = this.fields_dict[key];
-			if(f.get_value) {
+			if (f.get_value) {
 				var v = f.get_value();
-				if(f.df.reqd && is_null(strip_html(cstr(v))))
+				if (f.df.reqd && is_null(v))
 					errors.push(__(f.df.label));
 
-				if(!is_null(v)) ret[f.df.fieldname] = v;
+				if (f.df.reqd
+					&& f.df.fieldtype === 'Text Editor'
+					&& is_null(strip_html(cstr(v))))
+					errors.push(__(f.df.label));
+
+				if (!is_null(v)) ret[f.df.fieldname] = v;
 			}
 		}
-		if(errors.length && !ignore_errors) {
+		if (errors.length && !ignore_errors) {
 			frappe.msgprint({
 				title: __('Missing Values Required'),
 				message: __('Following fields have missing values:') +

From 94116ae93c416958619e98aa47ad979481ec9dd1 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 31 Jul 2020 16:07:51 +0530
Subject: [PATCH 106/361] feat(logger): allow max_size and file_count params

---
 frappe/__init__.py     |  4 ++--
 frappe/utils/logger.py | 20 +++++++++++---------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 7ea95ec3db..45f40f7783 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1559,10 +1559,10 @@ def get_doctype_app(doctype):
 
 loggers = {}
 log_level = None
-def logger(module=None, with_more_info=False, allow_site=True, filter=None):
+def logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20):
 	'''Returns a python logger that uses StreamHandler'''
 	from frappe.utils.logger import get_logger
-	return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter)
+	return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter, max_size=max_size, file_count=file_count)
 
 def log_error(message=None, title=_("Error")):
 	'''Log error to Error Log'''
diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 6d24eef9d3..b3295cccdf 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -16,14 +16,16 @@ import frappe
 default_log_level = logging.DEBUG
 
 
-def get_logger(module, with_more_info=False, allow_site=True, filter=None):
+def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20):
 	"""Application Logger for your given module
 
 	Args:
 		module (str): Name of your logger and consequently your log file.
 		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
 		allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True.
-		filter (function, optional): Add a filter function for your logger
+		filter (function, optional): Add a filter function for your logger. Defaults to None.
+		max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000.
+		file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20.
 
 	Returns:
 		<class 'logging.Logger'>: Returns a Python logger object with Site and Bench level logging capabilities.
@@ -36,10 +38,10 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	else:
 		site = False
 
-	LOGGER_NAME = "{}-{}".format(module, site or "all")
+	logger_name = "{0}-{1}".format(module, site or "all")
 
 	try:
-		return frappe.loggers[LOGGER_NAME]
+		return frappe.loggers[logger_name]
 	except KeyError:
 		pass
 
@@ -50,18 +52,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	logfile = module + ".log"
 	log_filename = os.path.join("..", "logs", logfile)
 
-	logger = logging.getLogger(LOGGER_NAME)
+	logger = logging.getLogger(logger_name)
 	logger.setLevel(frappe.log_level or default_log_level)
 	logger.propagate = False
 
 	formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module))
-	handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20)
+	handler = RotatingFileHandler(log_filename, maxBytes=max_size, backupCount=file_count)
 	handler.setFormatter(formatter)
 	logger.addHandler(handler)
 
 	if site:
-		SITELOG_FILENAME = os.path.join(site, "logs", logfile)
-		site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20)
+		sitelog_filename = os.path.join(site, "logs", logfile)
+		site_handler = RotatingFileHandler(sitelog_filename, maxBytes=max_size, backupCount=file_count)
 		site_handler.setFormatter(formatter)
 		logger.addHandler(site_handler)
 
@@ -71,7 +73,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None):
 	if filter:
 		logger.addFilter(filter)
 
-	frappe.loggers[LOGGER_NAME] = logger
+	frappe.loggers[logger_name] = logger
 
 	return logger
 

From f04ce5dd7a7957d46ef12d2d5e2216bb93e753e1 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 31 Jul 2020 16:11:39 +0530
Subject: [PATCH 107/361] chore: Typo

---
 frappe/utils/logger.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index b3295cccdf..451211b74b 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -24,7 +24,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_s
 		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
 		allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True.
 		filter (function, optional): Add a filter function for your logger. Defaults to None.
-		max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000.
+		max_size (int, optional): Max file size of each log file in bytes. Defaults to 100_000.
 		file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20.
 
 	Returns:

From a87531e1d0a808f69ce9d12a7e2fc9a9bb962641 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 31 Jul 2020 17:26:37 +0530
Subject: [PATCH 108/361] fix(get_logger): allow None for module

---
 frappe/utils/logger.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 451211b74b..398cdd0bfc 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -16,11 +16,11 @@ import frappe
 default_log_level = logging.DEBUG
 
 
-def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20):
+def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20):
 	"""Application Logger for your given module
 
 	Args:
-		module (str): Name of your logger and consequently your log file.
+		module (str, optional): Name of your logger and consequently your log file. Defaults to None.
 		with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False.
 		allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True.
 		filter (function, optional): Add a filter function for your logger. Defaults to None.

From c423eb7288623fb770216d712e923ee96c22e88f Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Fri, 31 Jul 2020 18:05:23 +0530
Subject: [PATCH 109/361] fix: text in MultiSelectList doesn't get truncated

---
 frappe/public/js/frappe/form/controls/multiselect_list.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/controls/multiselect_list.js b/frappe/public/js/frappe/form/controls/multiselect_list.js
index cd86bdd767..2a7ee5cb10 100644
--- a/frappe/public/js/frappe/form/controls/multiselect_list.js
+++ b/frappe/public/js/frappe/form/controls/multiselect_list.js
@@ -3,7 +3,7 @@ frappe.ui.form.ControlMultiSelectList = frappe.ui.form.ControlData.extend({
 		let template  = `
 			<div class="multiselect-list dropdown">
 				<div class="form-control cursor-pointer dropdown-toggle input-sm" data-toggle="dropdown" tabindex=0>
-					<span class="status-text ellipsis"></span>
+					<div class="status-text ellipsis"></div>
 				</div>
 				<ul class="dropdown-menu">
 					<li class="dropdown-input-wrapper">

From e304281c2aef714de8a4c623cc8c164d95961d65 Mon Sep 17 00:00:00 2001
From: Glen Whitney <glen@studioinfinity.org>
Date: Fri, 31 Jul 2020 15:51:24 +0000
Subject: [PATCH 110/361] feat(Desktop): additional options for Desk Shortcuts
 to DocTypes

  Since we generally have new contractors for most projects, the first step
  in entering almost all invoices is the creation of a new supplier. Hence,
  it was critical for us to have a direct shortcut to the New Supplier form
  on the Buying desk page. The existing options for Desk Shortcuts do not
  cover this case. Hence, this pull request adds an additional option
  selection for DocType Desk Shortcuts: the view. The shortcut may be
  specified to take you to the List, Tree, Report Builder, or Dashboard view
  of the specified DocType, or to the Form for entering a New instance of
  that DocType. If the view option is not specified, the behavior reverts
  to the current behavior for DocType shortcuts: they route to the Tree
  view for those DocTypes specified in the bootinfo to have treeviews, and
  to the List view otherwise.

  This is an improved version of PR #11019, redone according to the feedback
  from the Frappe maintainer who commented there.

  This commit involves adding the doc_view field to a Desk Shortcut and to
  the ShortcutDialog used for entering Desk Shortcuts. The field is made
  dependent so that it only appears when the type of the Desk Shortcut is
  "DocType." Also, the utility function generate_route has to be extended
  to take into account the doc_view field.
---
 .../doctype/desk_shortcut/desk_shortcut.json  | 12 +++++-
 .../js/frappe/widgets/shortcut_widget.js      |  4 +-
 frappe/public/js/frappe/widgets/utils.js      | 38 +++++++++++++++----
 .../public/js/frappe/widgets/widget_dialog.js | 10 +++++
 4 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/frappe/desk/doctype/desk_shortcut/desk_shortcut.json b/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
index f3fd546a77..faf0b55d4b 100644
--- a/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
+++ b/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
@@ -7,6 +7,7 @@
  "field_order": [
   "type",
   "link_to",
+  "doc_view",
   "column_break_4",
   "label",
   "icon",
@@ -34,6 +35,15 @@
    "options": "type",
    "reqd": 1
   },
+  {
+   "depends_on": "eval:doc.type == \"DocType\"",
+   "description": "Which view of the associated DocType should this shortcut take you to?",
+   "fieldname": "doc_view",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "DocType View",
+   "options": "\nList\nReport Builder\nDashboard\nTree\nNew"
+  },
   {
    "fieldname": "stats_filter",
    "fieldtype": "Code",
@@ -88,7 +98,7 @@
  ],
  "istable": 1,
  "links": [],
- "modified": "2020-05-14 16:02:15.420993",
+ "modified": "2020-07-30 12:35:48.095761",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Desk Shortcut",
diff --git a/frappe/public/js/frappe/widgets/shortcut_widget.js b/frappe/public/js/frappe/widgets/shortcut_widget.js
index 1de7e8bf75..a8eead8a3d 100644
--- a/frappe/public/js/frappe/widgets/shortcut_widget.js
+++ b/frappe/public/js/frappe/widgets/shortcut_widget.js
@@ -13,6 +13,7 @@ export default class ShortcutWidget extends Widget {
 			label: this.label,
 			format: this.format,
 			link_to: this.link_to,
+			doc_view: this.doc_view,
 			color: this.color,
 			restrict_to_domain: this.restrict_to_domain,
 			stats_filter: this.stats_filter,
@@ -30,6 +31,7 @@ export default class ShortcutWidget extends Widget {
 				type: this.type,
 				is_query_report: this.is_query_report,
 				doctype: this.ref_doctype,
+				doc_view: this.doc_view
 			});
 
 			let filters = this.get_doctype_filter();
@@ -96,4 +98,4 @@ export default class ShortcutWidget extends Widget {
 
 		buttons.appendTo(this.action_area);
 	}
-}
\ No newline at end of file
+}
diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index 22b3167977..00a2cb5df6 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -8,15 +8,39 @@ function generate_route(item) {
 		if (item.link) {
 			route = strip(item.link, "#");
 		} else if (type === "doctype") {
-			if (frappe.model.is_tree(item.doctype)) {
-				route = "Tree/" + item.doctype;
-			} else if (frappe.model.is_single(item.doctype)) {
+			if (frappe.model.is_single(item.doctype)) {
 				route = "Form/" + item.doctype;
 			} else {
-				if (item.filters) {
-					frappe.route_options = item.filters;
+				if (!item.doc_view) {
+					if (frappe.model.is_tree(item.doctype)) {
+						item.doc_view = "Tree";
+					} else {
+						item.doc_view = "List";
+					}
+				}
+				switch (item.doc_view) {
+				case "List":
+					if (item.filters) {
+						frappe.route_options = item.filters;
+					}
+					route = "List/" + item.doctype;
+					break;
+				case "Tree":
+					route = "Tree/" + item.doctype;
+					break;
+				case "Report Builder":
+					route = "List/" + item.doctype + "/Report";
+					break;
+				case "Dashboard":
+				        route = "List/" + item.doctype + "/Dashboard";
+					break;
+				case "New":
+					route = "Form/" + item.doctype + "/New " + item.doctype;
+					break;
+				default:
+					frappe.throw({message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View")});
+					route = "";
 				}
-				route = "List/" + item.doctype;
 			}
 		} else if (type === "report" && item.is_query_report) {
 			route = "query-report/" + item.name;
@@ -157,4 +181,4 @@ function get_number_system(country) {
 	return number_system_map[country];
 }
 
-export { generate_route, generate_grid, build_summary_item, shorten_number };
\ No newline at end of file
+export { generate_route, generate_grid, build_summary_item, shorten_number };
diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index 2ca26d59e1..1728cbe863 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -185,6 +185,16 @@ class ShortcutDialog extends WidgetDialog {
 					}
 				},
 			},
+			{
+				fieldtype: "Select",
+				fieldname: "doc_view",
+				label: "DocType View",
+				options: "\nList\nReport Builder\nDashboard\nTree\nNew",
+				description: "Which view of the associated DocType should this shortcut take you to?",
+				depends_on: (state) => {
+					return state.type == "DocType";
+				}
+			},
 			{
 				fieldtype: "Section Break",
 				fieldname: "filters_section_break",

From eab96ab904d9f39739111f8c486d0cecf4240638 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Sat, 1 Aug 2020 13:42:33 +0530
Subject: [PATCH 111/361] Revert "fix(image-with-blur): Set width, height in
 CSS"

---
 frappe/templates/includes/image_with_blur.html | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/frappe/templates/includes/image_with_blur.html b/frappe/templates/includes/image_with_blur.html
index dbddb9d8ef..20b0380b88 100644
--- a/frappe/templates/includes/image_with_blur.html
+++ b/frappe/templates/includes/image_with_blur.html
@@ -1,9 +1,7 @@
 {%- set res = frappe.utils.get_thumbnail_base64_for_image(src) if src else false -%}
 {%- if res and res['base64'].startswith('data:') -%}
 <img src="{{ res['base64'] }}" class="image-with-blur {{ resolve_class(class) }}"
-	data-src="{{ src or '' }}" alt="{{ alt or '' }}"
-	width="{{ res['width'] }}" height="{{ res['height'] }}"
-	style="width: {{ res['width'] }}px; height: {{ res['height'] }}px;" />
+	alt="{{ alt or '' }}" width="{{ res['width'] }}" height="{{ res['height'] }}" data-src="{{ src or '' }}" />
 {%- else -%}
 <img src="{{ src or '' }}" class="{{ resolve_class(class) }}" alt="{{ alt or '' }}" />
 {%- endif -%}

From 0bc2e6e9c808126feee66b6da7b592fe451ab7ae Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Mon, 3 Aug 2020 10:44:51 +0530
Subject: [PATCH 112/361] chore: Run test in verbose mode

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 63895675ea..3ae844bc3a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,12 +31,12 @@ matrix:
   - name: "Python 3.7 MariaDB"
     python: 3.7
     env: DB=mariadb TYPE=server
-    script: bench --site test_site run-tests --coverage
+    script: bench --site test_site --versbose run-tests --coverage
 
   - name: "Python 3.7 PostgreSQL"
     python: 3.7
     env: DB=postgres TYPE=server
-    script: bench --site test_site run-tests --coverage
+    script: bench --site test_site --versbose run-tests --coverage
 
   - name: "Cypress"
     python: 3.7

From 817e90d17111b4618f4e3c44e623222e62255f51 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Mon, 3 Aug 2020 10:54:20 +0530
Subject: [PATCH 113/361] fix: Typo

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3ae844bc3a..eb35e0cca2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,12 +31,12 @@ matrix:
   - name: "Python 3.7 MariaDB"
     python: 3.7
     env: DB=mariadb TYPE=server
-    script: bench --site test_site --versbose run-tests --coverage
+    script: bench --site test_site --verbose run-tests --coverage
 
   - name: "Python 3.7 PostgreSQL"
     python: 3.7
     env: DB=postgres TYPE=server
-    script: bench --site test_site --versbose run-tests --coverage
+    script: bench --site test_site --verbose run-tests --coverage
 
   - name: "Cypress"
     python: 3.7

From f0729ff6a8a58239e68c5c0810becc3ec27b056e Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Mon, 3 Aug 2020 12:00:44 +0530
Subject: [PATCH 114/361] test: Reset workflow doc

---
 frappe/workflow/doctype/workflow/test_workflow.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 2719bc7cf0..b8c40d4d3b 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -78,7 +78,7 @@ class TestWorkflow(unittest.TestCase):
 		frappe.set_user('test2@example.com')
 
 		doc = self.test_default_condition()
-		workflow_actions = frappe.get_all('Workflow Action', fields=['status'])
+		workflow_actions = frappe.get_all('Workflow Action', fields=['status', 'reference_name'])
 		self.assertEqual(len(workflow_actions), 1)
 
 		# test if status of workflow actions are updated on approval
@@ -102,6 +102,9 @@ class TestWorkflow(unittest.TestCase):
 		todo.reload()
 		self.assertEqual(todo.docstatus, 1)
 
+		self.workflow.states[1].doc_status = 0
+		self.workflow.save()
+
 	def test_if_workflow_set_on_action(self):
 		self.workflow.states[1].doc_status = 1
 		self.workflow.save()
@@ -111,6 +114,9 @@ class TestWorkflow(unittest.TestCase):
 		self.assertEqual(todo.docstatus, 1)
 		self.assertEqual(todo.workflow_state, 'Approved')
 
+		self.workflow.states[1].doc_status = 0
+		self.workflow.save()
+
 def create_todo_workflow():
 	if frappe.db.exists('Workflow', 'Test ToDo'):
 		return frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True)

From ccf7f28d98f70c9cf016d63aabdde9578c2ff5bc Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Mon, 3 Aug 2020 14:38:46 +0530
Subject: [PATCH 115/361] fix: get url only if redirect_to is not specified

---
 frappe/utils/oauth.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/utils/oauth.py b/frappe/utils/oauth.py
index b06227e289..7e4f7ede6c 100644
--- a/frappe/utils/oauth.py
+++ b/frappe/utils/oauth.py
@@ -309,5 +309,6 @@ def redirect_post_login(desk_user, redirect_to=None, provider=None):
 		# the #desktop is added to prevent a facebook redirect bug
 		desk_uri = "/desk#desktop" if provider == 'facebook' else '/desk'
 		redirect_to = desk_uri if desk_user else "/me"
+		redirect_to = frappe.utils.get_url(redirect_to)
 
-	frappe.local.response["location"] = frappe.utils.get_url(redirect_to)
+	frappe.local.response["location"] = redirect_to

From 5af06743158c20ade9bc57de8f0f593d858052c7 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Mon, 3 Aug 2020 14:40:07 +0530
Subject: [PATCH 116/361] fix: Redirect to workspace instead of desktop

---
 frappe/utils/oauth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/utils/oauth.py b/frappe/utils/oauth.py
index 7e4f7ede6c..c4dfd3dc11 100644
--- a/frappe/utils/oauth.py
+++ b/frappe/utils/oauth.py
@@ -307,7 +307,7 @@ def redirect_post_login(desk_user, redirect_to=None, provider=None):
 
 	if not redirect_to:
 		# the #desktop is added to prevent a facebook redirect bug
-		desk_uri = "/desk#desktop" if provider == 'facebook' else '/desk'
+		desk_uri = "/desk#workspace" if provider == 'facebook' else '/desk'
 		redirect_to = desk_uri if desk_user else "/me"
 		redirect_to = frappe.utils.get_url(redirect_to)
 

From 9445c115ecdf9ffbf5386adaf279935648a96754 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Mon, 3 Aug 2020 15:51:30 +0530
Subject: [PATCH 117/361] fix: check if is_standard is "Yes" (#11180)

---
 frappe/core/doctype/report/report.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js
index b4c30f5bbd..c410e9aa1a 100644
--- a/frappe/core/doctype/report/report.js
+++ b/frappe/core/doctype/report/report.js
@@ -1,6 +1,6 @@
 frappe.ui.form.on('Report', {
 	refresh: function(frm) {
-		if (frm.doc.is_standard && !frappe.boot.developer_mode) {
+		if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) {
 			// make the document read-only
 			frm.set_read_only();
 		}

From 4b93f75eba40625bba5b83cca2470aa179e3f082 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 3 Aug 2020 17:23:32 +0530
Subject: [PATCH 118/361] fix: Log site only if exists

---
 frappe/app.py          | 5 ++---
 frappe/utils/logger.py | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/app.py b/frappe/app.py
index 100b10e9ef..450e3b5f17 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -99,9 +99,8 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		_site = get_site_name(request.host)
-		frappe.logger("frappe.web", allow_site=_site).info({
-			"site": _site,
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
 			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
 			"base_url": getattr(request, "base_url", "NOTFOUND"),
 			"full_path": getattr(request, "full_path", "NOTFOUND"),
diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py
index 398cdd0bfc..364ffa776d 100755
--- a/frappe/utils/logger.py
+++ b/frappe/utils/logger.py
@@ -11,6 +11,7 @@ from six import text_type
 
 # imports - module imports
 import frappe
+from frappe.utils import get_sites
 
 
 default_log_level = logging.DEBUG
@@ -33,7 +34,7 @@ def get_logger(module=None, with_more_info=False, allow_site=True, filter=None,
 
 	if allow_site is True:
 		site = getattr(frappe.local, "site", None)
-	elif allow_site:
+	elif allow_site in get_sites():
 		site = allow_site
 	else:
 		site = False

From 00a8edf910b4c328aeb69f674db31439e206fd54 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Mon, 3 Aug 2020 23:53:01 +0530
Subject: [PATCH 119/361] test: Make User records before executing test

---
 frappe/workflow/doctype/workflow/test_workflow.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index b8c40d4d3b..fc12832e57 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -5,7 +5,10 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 from frappe.utils import random_string
-from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions
+from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow
+from frappe.test_runner import make_test_records
+
+make_test_records("User")
 
 class TestWorkflow(unittest.TestCase):
 	def setUp(self):

From e555970485467b69cddcbc7fce990f94c722f1cc Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Mon, 3 Aug 2020 23:53:44 +0530
Subject: [PATCH 120/361] chore: Comment out possibly problematic test

---
 .../event_producer/test_event_producer.py     | 80 +++++++++----------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 0d414f5d63..cda4ca8bf6 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase):
 
 		reset_configuration(producer_url)
 
-	def test_inner_mapping(self):
-		producer = get_remote_site()
-		event_producer = frappe.get_doc('Event Producer', producer_url)
-		event_producer.producer_doctypes = []
-		inner_mapping = [
-			{
-				'local_fieldname':'role_name',
-				'remote_fieldname':'title'
-			}
-		]
-		inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
-		mapping = [
-			{
-				'local_fieldname':'description',
-				'remote_fieldname':'content',
-			},
-			{
-				'local_fieldname': 'role',
-				'remote_fieldname': 'title',
-				'mapping_type': 'Document',
-				'mapping': inner_map,
-				'remote_value_filters': json.dumps({'title': 'title'})
-			}
-		]
-		event_producer.append('producer_doctypes', {
-			'ref_doctype': 'ToDo',
-			'use_same_name': 1,
-			'has_mapping': 1,
-			'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
-		})
-		event_producer.save()
+	# def test_inner_mapping(self):
+	# 	producer = get_remote_site()
+	# 	event_producer = frappe.get_doc('Event Producer', producer_url)
+	# 	event_producer.producer_doctypes = []
+	# 	inner_mapping = [
+	# 		{
+	# 			'local_fieldname':'role_name',
+	# 			'remote_fieldname':'title'
+	# 		}
+	# 	]
+	# 	inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
+	# 	mapping = [
+	# 		{
+	# 			'local_fieldname':'description',
+	# 			'remote_fieldname':'content',
+	# 		},
+	# 		{
+	# 			'local_fieldname': 'role',
+	# 			'remote_fieldname': 'title',
+	# 			'mapping_type': 'Document',
+	# 			'mapping': inner_map,
+	# 			'remote_value_filters': json.dumps({'title': 'title'})
+	# 		}
+	# 	]
+	# 	event_producer.append('producer_doctypes', {
+	# 		'ref_doctype': 'ToDo',
+	# 		'use_same_name': 1,
+	# 		'has_mapping': 1,
+	# 		'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
+	# 	})
+	# 	event_producer.save()
 
-		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
-		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
-		producer_note = producer.insert(producer_note)
-		self.pull_producer_data()
+	# 	producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
+	# 	delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
+	# 	producer_note = producer.insert(producer_note)
+	# 	self.pull_producer_data()
 
-		#check dependency inserted
-		self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title}))
-		#check doc inserted
-		self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content}))
+	# 	#check dependency inserted
+	# 	self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title}))
+	# 	#check doc inserted
+	# 	self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content}))
 
-		reset_configuration(producer_url)
+	# 	reset_configuration(producer_url)
 
 
 def insert_into_producer(producer, description):

From 88b68c6cf51061b4b1a37b93b5426279fc1dfe9d Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 4 Aug 2020 00:11:48 +0530
Subject: [PATCH 121/361] chore: Remove unwanted changes

---
 .travis.yml                                       | 4 ++--
 frappe/workflow/doctype/workflow/test_workflow.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index eb35e0cca2..63895675ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,12 +31,12 @@ matrix:
   - name: "Python 3.7 MariaDB"
     python: 3.7
     env: DB=mariadb TYPE=server
-    script: bench --site test_site --verbose run-tests --coverage
+    script: bench --site test_site run-tests --coverage
 
   - name: "Python 3.7 PostgreSQL"
     python: 3.7
     env: DB=postgres TYPE=server
-    script: bench --site test_site --verbose run-tests --coverage
+    script: bench --site test_site run-tests --coverage
 
   - name: "Cypress"
     python: 3.7
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index fc12832e57..253f95dab7 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 from frappe.utils import random_string
-from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow
+from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions
 from frappe.test_runner import make_test_records
 
 make_test_records("User")

From 89dfd82134d02d1a88444d02f858be820371129b Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 4 Aug 2020 00:36:42 +0530
Subject: [PATCH 122/361] chore: Add debug statement - Uncomment innocent test

---
 .../event_producer/test_event_producer.py     | 80 +++++++++----------
 .../doctype/workflow/test_workflow.py         |  3 +
 2 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index cda4ca8bf6..0d414f5d63 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase):
 
 		reset_configuration(producer_url)
 
-	# def test_inner_mapping(self):
-	# 	producer = get_remote_site()
-	# 	event_producer = frappe.get_doc('Event Producer', producer_url)
-	# 	event_producer.producer_doctypes = []
-	# 	inner_mapping = [
-	# 		{
-	# 			'local_fieldname':'role_name',
-	# 			'remote_fieldname':'title'
-	# 		}
-	# 	]
-	# 	inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
-	# 	mapping = [
-	# 		{
-	# 			'local_fieldname':'description',
-	# 			'remote_fieldname':'content',
-	# 		},
-	# 		{
-	# 			'local_fieldname': 'role',
-	# 			'remote_fieldname': 'title',
-	# 			'mapping_type': 'Document',
-	# 			'mapping': inner_map,
-	# 			'remote_value_filters': json.dumps({'title': 'title'})
-	# 		}
-	# 	]
-	# 	event_producer.append('producer_doctypes', {
-	# 		'ref_doctype': 'ToDo',
-	# 		'use_same_name': 1,
-	# 		'has_mapping': 1,
-	# 		'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
-	# 	})
-	# 	event_producer.save()
+	def test_inner_mapping(self):
+		producer = get_remote_site()
+		event_producer = frappe.get_doc('Event Producer', producer_url)
+		event_producer.producer_doctypes = []
+		inner_mapping = [
+			{
+				'local_fieldname':'role_name',
+				'remote_fieldname':'title'
+			}
+		]
+		inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
+		mapping = [
+			{
+				'local_fieldname':'description',
+				'remote_fieldname':'content',
+			},
+			{
+				'local_fieldname': 'role',
+				'remote_fieldname': 'title',
+				'mapping_type': 'Document',
+				'mapping': inner_map,
+				'remote_value_filters': json.dumps({'title': 'title'})
+			}
+		]
+		event_producer.append('producer_doctypes', {
+			'ref_doctype': 'ToDo',
+			'use_same_name': 1,
+			'has_mapping': 1,
+			'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
+		})
+		event_producer.save()
 
-	# 	producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
-	# 	delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
-	# 	producer_note = producer.insert(producer_note)
-	# 	self.pull_producer_data()
+		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
+		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
+		producer_note = producer.insert(producer_note)
+		self.pull_producer_data()
 
-	# 	#check dependency inserted
-	# 	self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title}))
-	# 	#check doc inserted
-	# 	self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content}))
+		#check dependency inserted
+		self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title}))
+		#check doc inserted
+		self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content}))
 
-	# 	reset_configuration(producer_url)
+		reset_configuration(producer_url)
 
 
 def insert_into_producer(producer, description):
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 253f95dab7..f94193fab9 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -30,6 +30,9 @@ class TestWorkflow(unittest.TestCase):
 	def test_approve(self, doc=None):
 		'''test simple workflow'''
 		todo = doc or self.test_default_condition()
+		print(frappe.db.sql("select name from `tabRole`"))
+		print('------------')
+		print(frappe.get_roles(), frappe.session.user)
 
 		apply_workflow(todo, 'Approve')
 		# default condition is set

From 6e9169a0aadc6b05af83432293297220e633631d Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 4 Aug 2020 10:12:41 +0530
Subject: [PATCH 123/361] test: Remove debug statements & clear-cache

---
 frappe/workflow/doctype/workflow/test_workflow.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index f94193fab9..fb42e6fd6e 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -30,9 +30,6 @@ class TestWorkflow(unittest.TestCase):
 	def test_approve(self, doc=None):
 		'''test simple workflow'''
 		todo = doc or self.test_default_condition()
-		print(frappe.db.sql("select name from `tabRole`"))
-		print('------------')
-		print(frappe.get_roles(), frappe.session.user)
 
 		apply_workflow(todo, 'Approve')
 		# default condition is set
@@ -129,6 +126,7 @@ def create_todo_workflow():
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)
+		frappe.cache().hdel('roles', 'Administrator')
 		workflow = frappe.new_doc('Workflow')
 		workflow.workflow_name = 'Test ToDo'
 		workflow.document_type = 'ToDo'

From 744a9413fdc7ffee3ac26883add9a950c3f0ab19 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 4 Aug 2020 10:36:16 +0530
Subject: [PATCH 124/361] fix: Use common function to get_roles

---
 frappe/__init__.py                                | 6 +-----
 frappe/workflow/doctype/workflow/test_workflow.py | 1 -
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index d644d2a473..f91317584d 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -437,11 +437,7 @@ def get_roles(username=None):
 	if not local.session:
 		return ["Guest"]
 
-	if username:
-		import frappe.permissions
-		return frappe.permissions.get_roles(username)
-	else:
-		return get_user().get_roles()
+	return frappe.permissions.get_roles(username or local.session.user)
 
 def get_request_header(key, default=None):
 	"""Return HTTP request header.
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index fb42e6fd6e..253f95dab7 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -126,7 +126,6 @@ def create_todo_workflow():
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)
-		frappe.cache().hdel('roles', 'Administrator')
 		workflow = frappe.new_doc('Workflow')
 		workflow.workflow_name = 'Test ToDo'
 		workflow.document_type = 'ToDo'

From 3c3b1037eaf9b7e92a52dd7c391d18f969d06102 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Tue, 4 Aug 2020 10:44:45 +0530
Subject: [PATCH 125/361] fix: Import frappe.permissions

---
 frappe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index f91317584d..31ce00e440 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -436,7 +436,7 @@ def get_roles(username=None):
 	"""Returns roles of current user."""
 	if not local.session:
 		return ["Guest"]
-
+	import frappe.permissions
 	return frappe.permissions.get_roles(username or local.session.user)
 
 def get_request_header(key, default=None):

From 86a9ceb29b93a0df1db233bb461adbcd0b005739 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Tue, 4 Aug 2020 11:16:18 +0530
Subject: [PATCH 126/361] test: Clear role cache

---
 frappe/workflow/doctype/workflow/test_workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 253f95dab7..d7f7939d3d 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -126,6 +126,7 @@ def create_todo_workflow():
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)
+		frappe.cache().hdel('roles', frappe.session.user)
 		workflow = frappe.new_doc('Workflow')
 		workflow.workflow_name = 'Test ToDo'
 		workflow.document_type = 'ToDo'

From 645331799af44feb4629ccc50f214278579be200 Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Tue, 4 Aug 2020 12:17:33 +0530
Subject: [PATCH 127/361] fix: whitelist functions called through search widget
 (#11153)

---
 frappe/core/doctype/user_permission/user_permission.py        | 1 +
 .../permitted_documents_for_user.py                           | 1 +
 frappe/desk/doctype/dashboard_chart/dashboard_chart.py        | 1 +
 frappe/desk/doctype/number_card/number_card.py                | 4 ++++
 frappe/email/__init__.py                                      | 1 +
 5 files changed, 8 insertions(+)

diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py
index 48d4fcb5d4..325c41622d 100644
--- a/frappe/core/doctype/user_permission/user_permission.py
+++ b/frappe/core/doctype/user_permission/user_permission.py
@@ -119,6 +119,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None):
 
 	return has_same_user_permission
 
+@frappe.whitelist()
 def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters):
 	linked_doctypes_map = get_linked_doctypes(doctype, True)
 
diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
index 8b2d1e01fa..5ca21b811e 100644
--- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
+++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
@@ -41,6 +41,7 @@ def get_columns_and_fields(doctype):
 
 	return columns, fields
 
+@frappe.whitelist()
 def query_doctypes(doctype, txt, searchfield, start, page_len, filters):
 	user = filters.get("user")
 	user_perms = frappe.utils.user.UserPermissions(user)
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index a25e0cab9e..7547efa234 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -344,6 +344,7 @@ def get_year_ending(date):
 	# last day of this month
 	return add_to_date(date, days=-1)
 
+@frappe.whitelist()
 def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters):
 	or_filters = {'owner': frappe.session.user, 'is_public': 1}
 	return frappe.db.get_list('Dashboard Chart',
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 3eb08ec00a..f44893d950 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -124,11 +124,15 @@ def create_number_card(args):
 	doc.insert(ignore_permissions=True)
 	return doc
 
+@frappe.whitelist()
 def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters):
 	meta = frappe.get_meta(doctype)
 	searchfields = meta.get_search_fields()
 	search_conditions = []
 
+	if not frappe.db.exists('DocType', doctype):
+		return
+
 	if txt:
 		for field in searchfields:
 			search_conditions.append('`tab{doctype}`.`{field}` like %(txt)s'.format(field=field, doctype=doctype, txt=txt))
diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py
index d58b35040e..0b874a03bb 100644
--- a/frappe/email/__init__.py
+++ b/frappe/email/__init__.py
@@ -57,6 +57,7 @@ def relink(name, reference_doctype=None, reference_name=None):
 			communication_type = "Communication" and
 			name = %s""", (reference_doctype, reference_name, name))
 
+@frappe.whitelist()
 def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters):
 	user_perms = frappe.utils.user.UserPermissions(frappe.session.user)
 	user_perms.build_permissions()

From 9b267027b5ce2c0674a4dd1cb476d5b80b873632 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Tue, 4 Aug 2020 13:25:49 +0530
Subject: [PATCH 128/361] feat: don't load user translation in
 make_dict_from_messages (#11120)

---
 frappe/translate.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/frappe/translate.py b/frappe/translate.py
index 2fc2d3f328..ee1f8fa5ff 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -119,7 +119,7 @@ def get_dict(fortype, name=None):
 			messages += frappe.db.sql("select 'Role:', name from tabRole")
 			messages += frappe.db.sql("select 'Module:', name from `tabModule Def`")
 
-		message_dict = make_dict_from_messages(messages)
+		message_dict = make_dict_from_messages(messages, load_user_translation=False)
 		message_dict.update(get_dict_from_hooks(fortype, name))
 		# remove untranslated
 		message_dict = {k:v for k, v in iteritems(message_dict) if k!=v}
@@ -127,6 +127,7 @@ def get_dict(fortype, name=None):
 		cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True)
 
 	translation_map = translation_assets[asset_key]
+
 	if fortype == "boot":
 		translation_map.update(get_user_translations(frappe.local.lang))
 
@@ -144,14 +145,17 @@ def get_dict_from_hooks(fortype, name):
 
 	return translated_dict
 
-def make_dict_from_messages(messages, full_dict=None):
+def make_dict_from_messages(messages, full_dict=None, load_user_translation=True):
 	"""Returns translated messages as a dict in Language specified in `frappe.local.lang`
 
 	:param messages: List of untranslated messages
 	"""
 	out = {}
 	if full_dict==None:
-		full_dict = get_full_dict(frappe.local.lang)
+		if load_user_translation:
+			full_dict = get_full_dict(frappe.local.lang)
+		else:
+			full_dict = load_lang(frappe.local.lang)
 
 	for m in messages:
 		if m[1] in full_dict:
@@ -189,11 +193,9 @@ def get_full_dict(lang):
 	try:
 		# get user specific transaltion data
 		user_translations = get_user_translations(lang)
-	except Exception:
-		user_translations = None
-
-	if user_translations:
 		frappe.local.lang_full_dict.update(user_translations)
+	except Exception:
+		pass
 
 	return frappe.local.lang_full_dict
 

From d5622f541021e9ecfa682bf30977baeb3e48f89b Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Tue, 4 Aug 2020 17:38:48 +0530
Subject: [PATCH 129/361] fix: Permission query in Dashboard Chart (#10981)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 .../dashboard_chart/dashboard_chart.py        | 25 ++++++++++++++-----
 .../desk/doctype/number_card/number_card.py   | 14 +++++++----
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index 7547efa234..88c9623945 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -28,15 +28,28 @@ def get_permission_query_conditions(user):
 	if "System Manager" in roles:
 		return None
 
-	allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()]
-	allowed_reports = ['"%s"' % key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()]
+	doctype_condition = False
+	report_condition = False
+
+	allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
+	allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()]
+
+	if allowed_doctypes:
+		doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format(
+			allowed_doctypes=','.join(allowed_doctypes))
+	if allowed_reports:
+		report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format(
+			allowed_reports=','.join(allowed_reports))
 
 	return '''
-			`tabDashboard Chart`.`document_type` in ({allowed_doctypes})
-			or `tabDashboard Chart`.`report_name` in ({allowed_reports})
+			(`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average')
+			and {doctype_condition})
+			or
+			(`tabDashboard Chart`.`chart_type` = 'Report'
+			and {report_condition})
 		'''.format(
-			allowed_doctypes=','.join(allowed_doctypes),
-			allowed_reports=','.join(allowed_reports)
+			doctype_condition=doctype_condition,
+			report_condition=report_condition
 		)
 
 
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index f44893d950..68ed79e64b 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -32,13 +32,17 @@ def get_permission_query_conditions(user=None):
 	if "System Manager" in roles:
 		return None
 
-	allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()]
+	doctype_condition = False
+
+	allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
+
+	if allowed_doctypes:
+		doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format(
+			allowed_doctypes=','.join(allowed_doctypes))
 
 	return '''
-			`tabNumber Card`.`document_type` in ({allowed_doctypes})
-		'''.format(
-			allowed_doctypes=','.join(allowed_doctypes)
-		)
+			{doctype_condition}
+		'''.format(doctype_condition=doctype_condition)
 
 def has_permission(doc, ptype, user):
 	roles = frappe.get_roles(user)

From dd157ba82255727e0095925eaf8d64fbfe89b275 Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Tue, 4 Aug 2020 17:39:35 +0530
Subject: [PATCH 130/361] fix: dont show git errors in terminal (#11064)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/utils/change_log.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py
index 53c83cfe99..c75b3289db 100644
--- a/frappe/utils/change_log.py
+++ b/frappe/utils/change_log.py
@@ -118,8 +118,9 @@ def get_versions():
 def get_app_branch(app):
 	'''Returns branch of an app'''
 	try:
+		null_stream = open(os.devnull, 'wb')
 		result = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app),
-			shell=True)
+			shell=True, stdin=null_stream, stderr=null_stream)
 		result = safe_decode(result)
 		result = result.strip()
 		return result
@@ -128,8 +129,9 @@ def get_app_branch(app):
 
 def get_app_last_commit_ref(app):
 	try:
+		null_stream = open(os.devnull, 'wb')
 		result = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app),
-			shell=True)
+			shell=True, stdin=null_stream, stderr=null_stream)
 		result = safe_decode(result)
 		result = result.strip()
 		return result

From 34fdf63b9097d5ad0cfb366121242c0f8480e460 Mon Sep 17 00:00:00 2001
From: Glen Whitney <glen@studioinfinity.org>
Date: Tue, 4 Aug 2020 05:10:06 -0700
Subject: [PATCH 131/361] fix(Link Card): refresh incomplete_dependencies as
 needed (#11169)

When _prepare_item in Workspace.get_cards notices that the dependencies
  for an item have been completed, it should clear the incomplete_dependencies
  field in case a value has been saved in that attribute.

  Fixes #11168.
---
 frappe/desk/desktop.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py
index 68b57a93d4..ae9d070976 100644
--- a/frappe/desk/desktop.py
+++ b/frappe/desk/desktop.py
@@ -221,6 +221,8 @@ class Workspace:
 				incomplete_dependencies = [d for d in item.dependencies if not _doctype_contains_a_record(d)]
 				if len(incomplete_dependencies):
 					item.incomplete_dependencies = incomplete_dependencies
+				else:
+					item.incomplete_dependencies = ""
 
 			if item.onboard:
 				# Mark Spotlights for initial

From f5c61d04da27d8258caad8631c5f7f4edf3ffa85 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchampfaris@users.noreply.github.com>
Date: Tue, 4 Aug 2020 17:40:18 +0530
Subject: [PATCH 132/361] fix: Patch to generate theme files in public folder
 (#11148)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/patches.txt                                |  1 +
 .../generate_theme_files_in_public_folder.py      | 15 +++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 frappe/patches/v13_0/generate_theme_files_in_public_folder.py

diff --git a/frappe/patches.txt b/frappe/patches.txt
index b207a325cb..75750ab59c 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -296,3 +296,4 @@ frappe.patches.v13_0.update_duration_options
 frappe.patches.v13_0.replace_old_data_import # 2020-06-24
 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts
 frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
+frappe.patches.v13_0.generate_theme_files_in_public_folder
diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
new file mode 100644
index 0000000000..c5a64780cd
--- /dev/null
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+
+def execute():
+	themes = frappe.db.get_all(
+		"Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")}
+	)
+	for theme in themes:
+		doc = frappe.get_doc("Website Theme", theme.name)
+		doc.generate_bootstrap_theme()
+		doc.save()

From 0748312e3950516bb08c76f200359757cd2c6759 Mon Sep 17 00:00:00 2001
From: Aditya Hase <aditya@adityahase.com>
Date: Tue, 4 Aug 2020 17:41:34 +0530
Subject: [PATCH 133/361] fix(werkzeug): Avoid duplicate lines from werkzeug in
 bench start logs (#11171)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/app.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/app.py b/frappe/app.py
index 450e3b5f17..725bec183a 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -256,9 +256,11 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 		'SERVER_NAME': 'localhost:8000'
 	}
 
+	log = logging.getLogger('werkzeug')
+	log.propagate = False
+
 	in_test_env = os.environ.get('CI')
 	if in_test_env:
-		log = logging.getLogger('werkzeug')
 		log.setLevel(logging.ERROR)
 
 	run_simple('0.0.0.0', int(port), application,

From cc22350b04f73b6321e1fac6ccf62f57ee9776fb Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Tue, 4 Aug 2020 17:42:47 +0530
Subject: [PATCH 134/361] fix: remove Video DocType from Core module (#11151)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/automation/desk_page/tools/tools.json |   5 +-
 frappe/core/doctype/video/__init__.py        |   0
 frappe/core/doctype/video/test_video.py      |  10 --
 frappe/core/doctype/video/video.js           |   8 --
 frappe/core/doctype/video/video.json         | 106 -------------------
 frappe/core/doctype/video/video.py           |  10 --
 frappe/public/js/frappe/views/breadcrumbs.js |   1 -
 7 files changed, 3 insertions(+), 137 deletions(-)
 delete mode 100644 frappe/core/doctype/video/__init__.py
 delete mode 100644 frappe/core/doctype/video/test_video.py
 delete mode 100644 frappe/core/doctype/video/video.js
 delete mode 100644 frappe/core/doctype/video/video.json
 delete mode 100644 frappe/core/doctype/video/video.py

diff --git a/frappe/automation/desk_page/tools/tools.json b/frappe/automation/desk_page/tools/tools.json
index 2164a4ce38..3fbaf62d02 100644
--- a/frappe/automation/desk_page/tools/tools.json
+++ b/frappe/automation/desk_page/tools/tools.json
@@ -3,7 +3,7 @@
   {
    "hidden": 0,
    "label": "Tools",
-   "links": "[\n    {\n        \"description\": \"Documents assigned to you and by you.\",\n        \"label\": \"To Do\",\n        \"name\": \"ToDo\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Event and other calendars.\",\n        \"label\": \"Calendar\",\n        \"link\": \"List/Event/Calendar\",\n        \"name\": \"Event\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Private and public Notes.\",\n        \"label\": \"Note\",\n        \"name\": \"Note\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"label\": \"Files\",\n        \"name\": \"File\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"label\": \"Video\",\n        \"name\": \"Video\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Activity log of all users.\",\n        \"label\": \"Activity\",\n        \"name\": \"activity\",\n        \"type\": \"page\"\n    }\n]"
+   "links": "[\n    {\n        \"description\": \"Documents assigned to you and by you.\",\n        \"label\": \"To Do\",\n        \"name\": \"ToDo\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Event and other calendars.\",\n        \"label\": \"Calendar\",\n        \"link\": \"List/Event/Calendar\",\n        \"name\": \"Event\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Private and public Notes.\",\n        \"label\": \"Note\",\n        \"name\": \"Note\",\n        \"onboard\": 1,\n        \"type\": \"doctype\"\n    },\n    {\n        \"label\": \"Files\",\n        \"name\": \"File\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Activity log of all users.\",\n        \"label\": \"Activity\",\n        \"name\": \"activity\",\n        \"type\": \"page\"\n    }\n]"
   },
   {
    "hidden": 0,
@@ -29,10 +29,11 @@
  "docstatus": 0,
  "doctype": "Desk Page",
  "extends_another_page": 0,
+ "hide_custom": 0,
  "idx": 0,
  "is_standard": 1,
  "label": "Tools",
- "modified": "2020-04-20 18:21:14.152537",
+ "modified": "2020-07-21 19:32:18.480700",
  "modified_by": "Administrator",
  "module": "Automation",
  "name": "Tools",
diff --git a/frappe/core/doctype/video/__init__.py b/frappe/core/doctype/video/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/core/doctype/video/test_video.py b/frappe/core/doctype/video/test_video.py
deleted file mode 100644
index 0bed1e98d6..0000000000
--- a/frappe/core/doctype/video/test_video.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# See license.txt
-from __future__ import unicode_literals
-
-# import frappe
-import unittest
-
-class TestVideo(unittest.TestCase):
-	pass
diff --git a/frappe/core/doctype/video/video.js b/frappe/core/doctype/video/video.js
deleted file mode 100644
index 36ea240a36..0000000000
--- a/frappe/core/doctype/video/video.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Video', {
-	// refresh: function(frm) {
-
-	// }
-});
diff --git a/frappe/core/doctype/video/video.json b/frappe/core/doctype/video/video.json
deleted file mode 100644
index 26a407c05c..0000000000
--- a/frappe/core/doctype/video/video.json
+++ /dev/null
@@ -1,106 +0,0 @@
-{
- "actions": [],
- "allow_import": 1,
- "allow_rename": 1,
- "autoname": "field:title",
- "creation": "2018-10-17 05:47:13.087395",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "title",
-  "provider",
-  "url",
-  "column_break_4",
-  "publish_date",
-  "duration",
-  "section_break_7",
-  "description"
- ],
- "fields": [
-  {
-   "fieldname": "title",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Title",
-   "reqd": 1,
-   "unique": 1
-  },
-  {
-   "fieldname": "provider",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Provider",
-   "options": "YouTube\nVimeo",
-   "reqd": 1
-  },
-  {
-   "fieldname": "url",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "URL",
-   "reqd": 1
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "publish_date",
-   "fieldtype": "Date",
-   "label": "Publish Date"
-  },
-  {
-   "fieldname": "duration",
-   "fieldtype": "Data",
-   "label": "Duration"
-  },
-  {
-   "fieldname": "section_break_7",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "description",
-   "fieldtype": "Text Editor",
-   "in_list_view": 1,
-   "label": "Description",
-   "reqd": 1
-  }
- ],
- "links": [],
- "modified": "2020-04-22 12:09:49.057403",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Video",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "if_owner": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "All",
-   "share": 1,
-   "write": 1
-  },
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/core/doctype/video/video.py b/frappe/core/doctype/video/video.py
deleted file mode 100644
index fdbd3a1abe..0000000000
--- a/frappe/core/doctype/video/video.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-# import frappe
-from frappe.model.document import Document
-
-class Video(Document):
-	pass
diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js
index 0058310e3f..dbe591f6fd 100644
--- a/frappe/public/js/frappe/views/breadcrumbs.js
+++ b/frappe/public/js/frappe/views/breadcrumbs.js
@@ -6,7 +6,6 @@ frappe.breadcrumbs = {
 
 	preferred: {
 		"File": "",
-		"Video": "",
 		"Dashboard": "Customization",
 		"Dashboard Chart": "Customization",
 		"Dashboard Chart Source": "Customization"

From 5d4feb5dbb264611d2d257eea8f8f45b9636fbae Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Tue, 4 Aug 2020 17:45:14 +0530
Subject: [PATCH 135/361] feat: Convenience APIs for tags (#11088)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/model/document.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 69a781d6d1..316c576f55 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -1307,6 +1307,16 @@ class Document(BaseDocument):
 		users = set([assignment.owner for assignment in assignments])
 		return users
 
+	def add_tag(self, tag):
+		"""Add a Tag to this document"""
+		from frappe.desk.doctype.tag.tag import DocTags
+		DocTags(self.doctype).add(self.name, tag)
+
+	def get_tags(self):
+		"""Return a list of Tags attached to this document"""
+		from frappe.desk.doctype.tag.tag import DocTags
+		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
+
 def execute_action(doctype, name, action, **kwargs):
 	"""Execute an action on a document (called by background worker)"""
 	doc = frappe.get_doc(doctype, name)

From 805e415502164c49aeef5a6332208ef2075c258d Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Wed, 5 Aug 2020 14:57:48 +0530
Subject: [PATCH 136/361] fix: Dashboard fixes (#11186)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/desk/doctype/dashboard_chart/dashboard_chart.js |  6 ------
 frappe/desk/doctype/dashboard_chart/dashboard_chart.py |  2 +-
 frappe/desk/doctype/number_card/number_card.py         |  2 +-
 frappe/public/js/frappe/widgets/utils.js               | 10 ++++++++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
index 8d89cc2f31..7f26bd9101 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
@@ -23,7 +23,6 @@ frappe.ui.form.on('Dashboard Chart', {
 		frm.chart_filters = null;
 
 		if (!frappe.boot.developer_mode && frm.doc.is_standard) {
-			frm.set_df_property('chart_options_section', 'hidden', 1);
 			frm.disable_form();
 		}
 
@@ -57,11 +56,6 @@ frappe.ui.form.on('Dashboard Chart', {
 		if (frm.doc.report_name) {
 			frm.trigger('set_chart_report_filters');
 		}
-
-		if (!frappe.boot.developer_mode) {
-			frm.set_df_property("custom_options", "hidden", 1);
-		}
-
 	},
 
 	is_standard: function(frm) {
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index 88c9623945..7be97f8e24 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -143,7 +143,7 @@ def add_chart_to_dashboard(args):
 	dashboard_link = frappe.new_doc('Dashboard Chart Link')
 	dashboard_link.chart = args.chart_name or args.name
 
-	if args.set_standard:
+	if args.set_standard and dashboard.is_standard:
 		chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart)
 		chart.is_standard = 1
 		chart.module = dashboard.module
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 68ed79e64b..8b79cf4052 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -180,7 +180,7 @@ def add_card_to_dashboard(args):
 	dashboard_link = frappe.new_doc('Number Card Link')
 	dashboard_link.card = args.name
 
-	if args.set_standard:
+	if args.set_standard and dashboard.is_standard:
 		card = frappe.get_doc('Number Card', dashboard_link.card)
 		card.is_standard = 1
 		card.module = dashboard.module
diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index 22b3167977..4599b4adc8 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -122,7 +122,8 @@ function shorten_number(number, country) {
 	const number_system = get_number_system(country);
 	let x = Math.abs(Math.round(number));
 	for (const map of number_system) {
-		if (x >= map.divisor) {
+		const condition = map.condition ? map.condition(x) : x >= map.divisor;
+		if (condition) {
 			return Math.round(number/map.divisor) +  ' ' + map.symbol;
 		}
 	}
@@ -152,9 +153,14 @@ function get_number_system(country) {
 			{
 				divisor: 1.0e+6,
 				symbol: 'M'
+			},
+			{
+				divisor: 1.0e+3,
+				symbol: 'K',
+				condition: (num) => num.toFixed().length > 5
 			}]
 	};
 	return number_system_map[country];
 }
 
-export { generate_route, generate_grid, build_summary_item, shorten_number };
\ No newline at end of file
+export { generate_route, generate_grid, build_summary_item, shorten_number };

From 4225b55041c4da41a961bf5ca13c5f339b81e3ae Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 5 Aug 2020 15:33:30 +0530
Subject: [PATCH 137/361] feat(minor): Allow web form to follow document
 permissions

---
 frappe/website/doctype/web_form/web_form.json | 11 ++++-
 frappe/website/doctype/web_form/web_form.py   | 45 ++++++++++---------
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json
index e6b7632e4d..77786734e7 100644
--- a/frappe/website/doctype/web_form/web_form.json
+++ b/frappe/website/doctype/web_form/web_form.json
@@ -16,6 +16,7 @@
   "route_to_success_link",
   "allow_edit",
   "allow_multiple",
+  "apply_document_permissions",
   "show_in_grid",
   "allow_delete",
   "allow_print",
@@ -346,14 +347,20 @@
    "fieldname": "custom_css_section",
    "fieldtype": "Section Break",
    "label": "Custom CSS"
+  },
+  {
+   "default": "0",
+   "fieldname": "apply_document_permissions",
+   "fieldtype": "Check",
+   "label": "Apply Document Permissions"
   }
  ],
  "has_web_view": 1,
  "icon": "icon-edit",
  "is_published_field": "published",
  "links": [],
- "modified": "2019-12-24 14:15:43.497431",
- "modified_by": "faris@erpnext.com",
+ "modified": "2020-06-30 21:49:18.237443",
+ "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Form",
  "owner": "Administrator",
diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index 4fc3c99037..bb0e03ddae 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -134,7 +134,7 @@ def get_context(context):
 		if frappe.session.user == "Guest" and frappe.form_dict.name:
 			frappe.throw(_("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError)
 
-		if frappe.form_dict.name and not has_web_form_permission(self.doc_type, frappe.form_dict.name):
+		if frappe.form_dict.name and not self.has_web_form_permission(self.doc_type, frappe.form_dict.name):
 			frappe.throw(_("You don't have the permissions to access this document"), frappe.PermissionError)
 
 		self.reset_field_parent()
@@ -347,6 +347,27 @@ def get_context(context):
 			frappe.throw(_('Mandatory Information missing:') + '<br><br>'
 				+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
 
+	def has_web_form_permission(self, doctype, name, ptype='read'):
+		if frappe.session.user=="Guest":
+			return False
+
+		if self.apply_document_permissions:
+			return frappe.get_doc(doctype, name).has_permission()
+
+		# owner matches
+		elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user:
+			return True
+
+		elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype):
+			return True
+
+		elif check_webform_perm(doctype, name):
+			return True
+
+		else:
+			return False
+
+
 
 @frappe.whitelist(allow_guest=True)
 def accept(web_form, data, docname=None, for_payment=False):
@@ -395,7 +416,7 @@ def accept(web_form, data, docname=None, for_payment=False):
 		doc.run_method('validate_payment')
 
 	if doc.name:
-		if has_web_form_permission(doc.doctype, doc.name, "write"):
+		if web_form.has_web_form_permission(doc.doctype, doc.name, "write"):
 			doc.save(ignore_permissions=True)
 		else:
 			# only if permissions are present
@@ -482,24 +503,6 @@ def delete_multiple(web_form_name, docnames):
 		raise frappe.PermissionError("You do not have permisssion to delete " + ", ".join(restricted_docnames))
 
 
-def has_web_form_permission(doctype, name, ptype='read'):
-	if frappe.session.user=="Guest":
-		return False
-
-	# owner matches
-	elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user:
-		return True
-
-	elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype):
-		return True
-
-	elif check_webform_perm(doctype, name):
-		return True
-
-	else:
-		return False
-
-
 def check_webform_perm(doctype, name):
 	doc = frappe.get_doc(doctype, name)
 	if hasattr(doc, "has_webform_permission"):
@@ -536,7 +539,7 @@ def get_form_data(doctype, docname=None, web_form_name=None):
 
 	if docname:
 		doc = frappe.get_doc(doctype, docname)
-		if has_web_form_permission(doctype, docname, ptype='read'):
+		if web_form.has_web_form_permission(doctype, docname, ptype='read'):
 			out.doc = doc
 		else:
 			frappe.throw(_("Not permitted"), frappe.PermissionError)

From cd1ab8e23c4a1de4bbd5137c34a864af53788237 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Wed, 5 Aug 2020 19:41:41 +0530
Subject: [PATCH 138/361] feat: Add validate_and_sanitize_search_inputs
 decorator (#11194)

* feat: Add validate_and_sanitize_search_inputs decorator

Co-authored-by: Nabin Hait <nabinhait@gmail.com>
Co-authored-by: Prssanna Desai <prssud@gmail.com>
Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>

* refactor: Move validate_and_sanitize_search_inputs to init

* refactor: Move validate_and_sanitize_search_inputs to search.py

* test: validate_and_sanitize_search_inputs decorator

* chore: Add wrapt module

* refactor: Use @wrapt to define validate_and_sanitize_search_inputs decorator

* test: Add a case to make sure frappe.call works as well

Co-authored-by: Nabin Hait <nabinhait@gmail.com>
Co-authored-by: Prssanna Desai <prssud@gmail.com>
Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>
---
 frappe/__init__.py          |  4 ++++
 frappe/desk/search.py       | 13 +++++++++++++
 frappe/tests/test_search.py | 28 ++++++++++++++++++++++++++++
 requirements.txt            |  3 ++-
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 45f40f7783..9e744c2c07 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1707,3 +1707,7 @@ def mock(type, size=1, locale='en'):
 
 	from frappe.chat.util import squashify
 	return squashify(results)
+
+def validate_and_sanitize_search_inputs(fn):
+	from frappe.desk.search import validate_and_sanitize_search_inputs as func
+	return func(fn)
diff --git a/frappe/desk/search.py b/frappe/desk/search.py
index b4b54b4b6e..798e499bb9 100644
--- a/frappe/desk/search.py
+++ b/frappe/desk/search.py
@@ -10,6 +10,7 @@ from frappe.handler import is_whitelisted
 from frappe import _
 from six import string_types
 import re
+import wrapt
 
 UNTRANSLATED_DOCTYPES = ["DocType", "Role"]
 
@@ -206,3 +207,15 @@ def scrub_custom_query(query, key, txt):
 	if '%s' in query:
 		query = query.replace('%s', ((txt or '') + '%'))
 	return query
+
+@wrapt.decorator
+def validate_and_sanitize_search_inputs(fn, instance, args, kwargs):
+	kwargs.update(dict(zip(fn.__code__.co_varnames, args)))
+	sanitize_searchfield(kwargs['searchfield'])
+	kwargs['start'] = cint(kwargs['start'])
+	kwargs['page_len'] = cint(kwargs['page_len'])
+
+	if kwargs['doctype'] and not frappe.db.exists('DocType', kwargs['doctype']):
+		return []
+
+	return fn(**kwargs)
\ No newline at end of file
diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py
index 5cab827218..192fe9dd25 100644
--- a/frappe/tests/test_search.py
+++ b/frappe/tests/test_search.py
@@ -50,3 +50,31 @@ class TestSearch(unittest.TestCase):
 
 	def tearDown(self):
 		frappe.local.lang = 'en'
+
+	def test_validate_and_sanitize_search_inputs(self):
+
+		# should raise error if searchfield is injectable
+		self.assertRaises(frappe.DataError,
+			get_data, *('User', 'Random', 'select * from tabSessions) --', '1', '10', dict()))
+
+		# page_len and start should be converted to int
+		self.assertListEqual(get_data('User', 'Random', 'email', 'name or (select * from tabSessions)', '10', dict()),
+			['User', 'Random', 'email', 0, 10, {}])
+		self.assertListEqual(get_data('User', 'Random', 'email', page_len='2', start='10', filters=dict()),
+			['User', 'Random', 'email', 10, 2, {}])
+
+		# DocType can be passed as None which should be accepted
+		self.assertListEqual(get_data(None, 'Random', 'email', '2', '10', dict()),
+			[None, 'Random', 'email', 2, 10, {}])
+
+		# return empty string if passed doctype is invalid
+		self.assertListEqual(get_data("Random DocType", 'Random', 'email', '2', '10', dict()), [])
+
+		# should not fail if function is called via frappe.call with extra arguments
+		args = ("Random DocType", 'Random', 'email', '2', '10', dict())
+		kwargs = {'as_dict': False}
+		self.assertListEqual(frappe.call('frappe.tests.test_search.get_data', *args, **kwargs), [])
+
+@frappe.validate_and_sanitize_search_inputs
+def get_data(doctype, txt, searchfield, start, page_len, filters):
+	return [doctype, txt, searchfield, start, page_len, filters]
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 33cd292bf6..2995553541 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -69,4 +69,5 @@ Whoosh==2.7.4
 xlrd==1.2.0
 zxcvbn-python==4.4.24
 pycryptodome==3.9.8
-paytmchecksum==1.7.0
\ No newline at end of file
+paytmchecksum==1.7.0
+wrapt==1.10.11
\ No newline at end of file

From 21a0cafe6209334906441c07c7f2e085afc82ade Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Wed, 5 Aug 2020 19:42:19 +0530
Subject: [PATCH 139/361] fix: Validate and sanitise whitelisted search methods
 using decorator (#11197)

* fix: Validate and sanitize whitelisted search methods using decorator

Co-authored-by: Prssanna Desai <prssud@gmail.com>
Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>

* refactor: Replace decorator name

Co-authored-by: Prssanna Desai <prssud@gmail.com>
Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>

Co-authored-by: Prssanna Desai <prssud@gmail.com>
Co-authored-by: Shivam Mishra <scmmishra@users.noreply.github.com>
---
 frappe/automation/doctype/auto_repeat/auto_repeat.py             | 1 +
 frappe/contacts/address_and_contact.py                           | 1 +
 frappe/contacts/doctype/address/address.py                       | 1 +
 frappe/contacts/doctype/contact/contact.py                       | 1 +
 frappe/core/doctype/user/user.py                                 | 1 +
 frappe/core/doctype/user_permission/user_permission.py           | 1 +
 .../permitted_documents_for_user/permitted_documents_for_user.py | 1 +
 frappe/desk/doctype/dashboard_chart/dashboard_chart.py           | 1 +
 frappe/desk/doctype/number_card/number_card.py                   | 1 +
 frappe/email/__init__.py                                         | 1 +
 10 files changed, 10 insertions(+)

diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py
index a946fcc81c..c09e347e71 100644
--- a/frappe/automation/doctype/auto_repeat/auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py
@@ -374,6 +374,7 @@ def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, e
 
 # method for reference_doctype filter
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters):
 	res = frappe.db.get_all('Property Setter', {
 		'property': 'allow_auto_repeat',
diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py
index 51f13fb1a1..3ca9547188 100644
--- a/frappe/contacts/address_and_contact.py
+++ b/frappe/contacts/address_and_contact.py
@@ -147,6 +147,7 @@ def delete_contact_and_address(doctype, docname):
 				doc.delete()
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters):
 	if not txt: txt = ""
 
diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py
index 57dea8284c..e82ab9b26e 100644
--- a/frappe/contacts/doctype/address/address.py
+++ b/frappe/contacts/doctype/address/address.py
@@ -231,6 +231,7 @@ def get_company_address(company):
 	return ret
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def address_query(doctype, txt, searchfield, start, page_len, filters):
 	from frappe.desk.reportview import get_match_cond
 
diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py
index 8240940d2f..f82946dc5e 100644
--- a/frappe/contacts/doctype/contact/contact.py
+++ b/frappe/contacts/doctype/contact/contact.py
@@ -183,6 +183,7 @@ def update_contact(doc, method):
 		contact.save(ignore_permissions=True)
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def contact_query(doctype, txt, searchfield, start, page_len, filters):
 	from frappe.desk.reportview import get_match_cond
 
diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index 64bff32189..ef2de679ae 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -812,6 +812,7 @@ def reset_password(user):
 		return 'not found'
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def user_query(doctype, txt, searchfield, start, page_len, filters):
 	from frappe.desk.reportview import get_match_cond
 
diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py
index 325c41622d..ba14583c2f 100644
--- a/frappe/core/doctype/user_permission/user_permission.py
+++ b/frappe/core/doctype/user_permission/user_permission.py
@@ -120,6 +120,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None):
 	return has_same_user_permission
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters):
 	linked_doctypes_map = get_linked_doctypes(doctype, True)
 
diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
index 5ca21b811e..97209cd8ea 100644
--- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
+++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py
@@ -42,6 +42,7 @@ def get_columns_and_fields(doctype):
 	return columns, fields
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def query_doctypes(doctype, txt, searchfield, start, page_len, filters):
 	user = filters.get("user")
 	user_perms = frappe.utils.user.UserPermissions(user)
diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
index 7be97f8e24..4ea61ec6a9 100644
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@@ -358,6 +358,7 @@ def get_year_ending(date):
 	return add_to_date(date, days=-1)
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters):
 	or_filters = {'owner': frappe.session.user, 'is_public': 1}
 	return frappe.db.get_list('Dashboard Chart',
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 8b79cf4052..d4a2b00c57 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -129,6 +129,7 @@ def create_number_card(args):
 	return doc
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters):
 	meta = frappe.get_meta(doctype)
 	searchfields = meta.get_search_fields()
diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py
index 0b874a03bb..b05aef7639 100644
--- a/frappe/email/__init__.py
+++ b/frappe/email/__init__.py
@@ -58,6 +58,7 @@ def relink(name, reference_doctype=None, reference_name=None):
 			name = %s""", (reference_doctype, reference_name, name))
 
 @frappe.whitelist()
+@frappe.validate_and_sanitize_search_inputs
 def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters):
 	user_perms = frappe.utils.user.UserPermissions(frappe.session.user)
 	user_perms.build_permissions()

From 34c3d57767611db622779efe5cf0454b8a866f92 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Wed, 5 Aug 2020 18:31:11 +0000
Subject: [PATCH 140/361] feat: validate json when saving desk page (#11095)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/desk/doctype/desk_page/desk_page.py | 9 +++++++++
 frappe/utils/data.py                       | 9 +++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/frappe/desk/doctype/desk_page/desk_page.py b/frappe/desk/doctype/desk_page/desk_page.py
index f14535cb5f..cc2db53481 100644
--- a/frappe/desk/doctype/desk_page/desk_page.py
+++ b/frappe/desk/doctype/desk_page/desk_page.py
@@ -5,14 +5,23 @@
 from __future__ import unicode_literals
 import frappe
 from frappe import _
+from frappe.utils.data import validate_json_string
 from frappe.modules.export_file import export_to_files
 from frappe.model.document import Document
 
 class DeskPage(Document):
 	def validate(self):
+		self.validate_cards_json()
 		if (self.is_standard and not frappe.conf.developer_mode and not disable_saving_as_standard()):
 			frappe.throw(_("You need to be in developer mode to edit this document"))
 
+	def validate_cards_json(self):
+		for card in self.cards:
+			try:
+				validate_json_string(card.links)
+			except frappe.ValidationError:
+				frappe.throw(_("Invalid JSON in card links for {0}").format(frappe.bold(card.label)))
+
 	def on_update(self):
 		if disable_saving_as_standard():
 			return
diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index d1b409d1fc..bda201f083 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -8,6 +8,7 @@ import frappe
 from dateutil.parser._parser import ParserError
 import subprocess
 import operator
+import json
 import re, datetime, math, time
 import babel.dates
 from babel.core import UnknownLocaleError
@@ -1236,8 +1237,6 @@ def is_subset(list_a, list_b):
 def generate_hash(*args, **kwargs):
 	return frappe.generate_hash(*args, **kwargs)
 
-
-
 def guess_date_format(date_string):
 	DATE_FORMATS = [
 		r"%d-%m-%Y",
@@ -1310,3 +1309,9 @@ def guess_date_format(date_string):
 
 		if date_format and time_format:
 			return (date_format + ' ' + time_format).strip()
+
+def validate_json_string(string):
+	try:
+		json.loads(string)
+	except (TypeError, ValueError):
+		raise frappe.ValidationError
\ No newline at end of file

From ef4a28e10c0de0083c57278b615e587600b64e3b Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 6 Aug 2020 09:32:04 +0530
Subject: [PATCH 141/361] fix: add a distinct identifier for public files
 backup (#11201)

---
 frappe/integrations/offsite_backup_utils.py | 2 +-
 frappe/utils/backups.py                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index 9de176b2d0..0b3c47064e 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -53,7 +53,7 @@ def get_latest_backup_file(with_files=False):
 	latest_site_config = get_latest('*.json')
 
 	if with_files:
-		latest_public_file_bak = get_latest('*-files.tar')
+		latest_public_file_bak = get_latest('*-public-files.tar')
 		latest_private_file_bak = get_latest('*-private-files.tar')
 		return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak
 
diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 7eb00ceccd..08b911c9ae 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -82,7 +82,7 @@ class BackupGenerator:
 	def set_backup_file_name(self):
 		#Generate a random name using today's date and a 8 digit random number
 		for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
-		for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar"
+		for_public_files = self.todays_date + "-" + self.site_slug + "-public-files.tar"
 		for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar"
 		backup_path = get_backup_path()
 

From 4f37f9a4528ba50b82f2000a4381c8ff813ff138 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Thu, 6 Aug 2020 09:33:39 +0530
Subject: [PATCH 142/361] test: Try committing after Role creation

---
 frappe/workflow/doctype/workflow/test_workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index d7f7939d3d..84adc3a096 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -126,6 +126,7 @@ def create_todo_workflow():
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)
+		frappe.db.commit()
 		frappe.cache().hdel('roles', frappe.session.user)
 		workflow = frappe.new_doc('Workflow')
 		workflow.workflow_name = 'Test ToDo'

From b23e2cc31108df1440aaa3e3910370a5d0769d73 Mon Sep 17 00:00:00 2001
From: Saurabh <saurabh6790@gmail.com>
Date: Thu, 6 Aug 2020 19:09:35 +0530
Subject: [PATCH 143/361] fix: handle encoding exceptions while appending email
 to sent email folder

---
 frappe/email/doctype/email_account/email_account.py | 6 ++++--
 frappe/email/queue.py                               | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py
index cf8c6e80c6..197b3ee376 100755
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@@ -700,8 +700,10 @@ class EmailAccount(Document):
 		email_server.connect()
 
 		if email_server.imap:
-			email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message)
-
+			try:
+				email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message.encode())
+			except Exception:
+				frappe.log_error()
 
 @frappe.whitelist()
 def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None):
diff --git a/frappe/email/queue.py b/frappe/email/queue.py
index 8bffc108b9..25b6f93f83 100755
--- a/frappe/email/queue.py
+++ b/frappe/email/queue.py
@@ -460,7 +460,7 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False):
 			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
 
 		if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient:
-			smtpserver.email_account.append_email_to_sent_folder(encode(message))
+			smtpserver.email_account.append_email_to_sent_folder(message)
 
 	except (smtplib.SMTPServerDisconnected,
 			smtplib.SMTPConnectError,

From 91d0e47f1dacf544533dc840b848cb5a59646a6a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:03:38 +0530
Subject: [PATCH 144/361] feat: optimize while storing index

---
 frappe/modules/full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index fce9983907..a9b93a1265 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -67,7 +67,7 @@ def build_index(index_name, documents):
 				title=document.title, path=document.path, content=document.content
 			)
 
-	writer.commit()
+	writer.commit(optimize=True)
 
 
 def search(index_name, text, scope=None, limit=20):

From aa0cf2bad4ba4a0ed749e0187bab7f39fcb5cd92 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:05:08 +0530
Subject: [PATCH 145/361] feat: added re-index method

---
 frappe/modules/full_text_search.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index a9b93a1265..d06d925d08 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -101,6 +101,26 @@ def search(index_name, text, scope=None, limit=20):
 
 	return out
 
+def reindex_path(index_name, path):
+	document = get_document_to_index(path):
+	reindex(index_name, document)
+
+def reindex(index_name, document):
+	index_dir = get_index_path(index_name)
+	ix = open_dir(index_dir)
+
+	with ix.searcher() as searcher:
+		writer = ix.writer()
+		
+		# Remove the index of the particular file
+		writer.delete_by_term('path', document.path)
+
+		# that wasn't indexed before. So index it!
+		writer.add_document(
+			title=document.title, path=document.path, content=document.content
+		)
+
+		writer.commit(optimize=True)
 
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From d5d32770136fc71e6d0de8bbc3a291b825d916a0 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:05:55 +0530
Subject: [PATCH 146/361] feat: move search portable

---
 frappe/templates/doc.html    | 66 +--------------------------
 frappe/website/js/website.js | 86 +++++++++++++++++++++++++++++++++++-
 2 files changed, 87 insertions(+), 65 deletions(-)

diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index 3e1cc5509a..1c70aace98 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -22,7 +22,7 @@
 			</div>
 			<div class="col-12 col-lg-8">
 				<div class="doc-search-container">
-					<div class="doc-search">
+					<div class="doc-search" id="search-container">
 						<div class="dropdown">
 							<div class="search-icon">
 								<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
@@ -117,73 +117,11 @@ id="page-{{ name or route | e }}" data-path="{{ pathname | e }}"
 {%- block script -%}
 <script>
 	frappe.ready(() => {
-		setup_search();
+		frappe.setup_search('#search-container', '{{ docs_search_scope or "" }}');
 
 		$('.web-footer .container')
 			.removeClass('container')
 			.addClass('container-fluid doc-container');
 	});
-
-	function setup_search() {
-		let $dropdown = $('.doc-search .dropdown');
-		let $dropdown_menu = $('.doc-search .dropdown-menu');
-		let $input = $('.doc-search input');
-
-		$(document).on('keypress', e => {
-			if (e.key === '/') {
-				e.preventDefault();
-				$input.focus();
-			}
-		});
-
-		$input.on('input', frappe.utils.debounce(() => {
-			if (!$input.val()) {
-				clear_dropdown();
-				return;
-			}
-
-			frappe.call({
-				method: 'frappe.modules.full_text_search.web_search',
-				args: {
-					index_name: 'web_routes',
-					scope: '{{ docs_search_scope or "" }}' || null,
-					query: $input.val(),
-					limit: 5
-				}
-			}).then(r => {
-				let results = r.message || [];
-				let dropdown_html;
-				if (results.length == 0) {
-					dropdown_html = `<div class="dropdown-item">No results found</div>`;
-				} else {
-					dropdown_html = results.map(r => {
-						return `<a class="dropdown-item" href="/${r.path}">
-							<h6>${r.title_highlights || r.title}</h6>
-							<div style="white-space: normal;">${r.content_highlights}</div>
-						</a>`
-					}).join('')
-				}
-				$dropdown_menu.html(dropdown_html);
-				$dropdown_menu.addClass('show');
-			});
-		}, 500));
-
-		$input.on('focus', () => {
-			if (!$input.val()) {
-				clear_dropdown();
-			}
-		});
-
-		$input.on('blur', () => {
-			setTimeout(() => {
-				clear_dropdown();
-			}, 300);
-		});
-
-		function clear_dropdown() {
-			$dropdown_menu.html('');
-			$dropdown_menu.removeClass('show');
-		}
-	}
 </script>
 {%- endblock -%}
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index cd9609cfd3..b69077f482 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -380,9 +380,93 @@ $.extend(frappe, {
 	}
 });
 
+frappe.setup_search = function (target, search_scope) {
+	if (typeof target === "string") {
+		target = $(target);
+	}
+
+	let $search_input = $(`<div class="dropdown">
+			<div class="search-icon">
+				<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
+					fill="none"
+					stroke="currentColor" stroke-width="2" stroke-linecap="round"
+					stroke-linejoin="round"
+					class="feather feather-search">
+					<circle cx="11" cy="11" r="8"></circle>
+					<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
+				</svg>
+			</div>
+			<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+			<div class="overflow-hidden shadow dropdown-menu w-100">
+			</div>
+		</div>`);
+
+	target.empty();
+	$search_input.appendTo(target);
+
+	let $dropdown = $search_input.find('.dropdown');
+	let $dropdown_menu = $search_input.find('.dropdown-menu');
+	let $input = $search_input.find('input');
+
+	$(document).on('keypress', e => {
+		if (e.key === '/') {
+			e.preventDefault();
+			$input.focus();
+		}
+	});
+
+	$input.on('input', frappe.utils.debounce(() => {
+		if (!$input.val()) {
+			clear_dropdown();
+			return;
+		}
+
+		frappe.call({
+			method: 'frappe.modules.full_text_search.web_search',
+			args: {
+				index_name: 'web_routes',
+				scope: search_scope || null,
+				query: $input.val(),
+				limit: 5
+			}
+		}).then(r => {
+			let results = r.message || [];
+			let dropdown_html;
+			if (results.length == 0) {
+				dropdown_html = `<div class="dropdown-item">No results found</div>`;
+			} else {
+				dropdown_html = results.map(r => {
+					return `<a class="dropdown-item" href="/${r.path}">
+						<h6>${r.title_highlights || r.title}</h6>
+						<div style="white-space: normal;">${r.content_highlights}</div>
+					</a>`
+				}).join('')
+			}
+			$dropdown_menu.html(dropdown_html);
+			$dropdown_menu.addClass('show');
+		});
+	}, 500));
+
+	$input.on('focus', () => {
+		if (!$input.val()) {
+			clear_dropdown();
+		}
+	});
+
+	$input.on('blur', () => {
+		setTimeout(() => {
+			clear_dropdown();
+		}, 300);
+	});
+
+	function clear_dropdown() {
+		$dropdown_menu.html('');
+		$dropdown_menu.removeClass('show');
+	}
+}
+
 
 // Utility functions
-
 window.valid_email = function(id) {
 	// eslint-disable-next-line
 	// copied regex from frappe/utils.js validate_type

From f33c05d128b518fbcffa84ab108ac7872a21ca81 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:08:11 +0530
Subject: [PATCH 147/361] chore: better comments

---
 frappe/modules/full_text_search.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index d06d925d08..84e0d1cee5 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -106,16 +106,18 @@ def reindex_path(index_name, path):
 	reindex(index_name, document)
 
 def reindex(index_name, document):
+	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
 
+	# initiate search scope to find previous index
 	with ix.searcher() as searcher:
 		writer = ix.writer()
 		
 		# Remove the index of the particular file
 		writer.delete_by_term('path', document.path)
 
-		# that wasn't indexed before. So index it!
+		# re index it!
 		writer.add_document(
 			title=document.title, path=document.path, content=document.content
 		)

From ac887875935d0ffc9620fa799e385368bba4fcbd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:28:18 +0530
Subject: [PATCH 148/361] fix: syntax

---
 frappe/modules/full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 84e0d1cee5..5355546c36 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -102,7 +102,7 @@ def search(index_name, text, scope=None, limit=20):
 	return out
 
 def reindex_path(index_name, path):
-	document = get_document_to_index(path):
+	document = get_document_to_index(path)
 	reindex(index_name, document)
 
 def reindex(index_name, document):

From 32fbab7d183929efc6d4e711233da8779a098699 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:46:54 +0530
Subject: [PATCH 149/361] feat: remove re_index by path

* get_document_to_index sets user to administrator, this is not desirable when running desk, keeping this API available is not wise
---
 frappe/modules/full_text_search.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 5355546c36..7ccc68f877 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -100,11 +100,7 @@ def search(index_name, text, scope=None, limit=20):
 			)
 
 	return out
-
-def reindex_path(index_name, path):
-	document = get_document_to_index(path)
-	reindex(index_name, document)
-
+	
 def reindex(index_name, document):
 	# open index
 	index_dir = get_index_path(index_name)

From 9db450f815b18090ca9f04d01e68d541a7bd6f4e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 20:47:03 +0530
Subject: [PATCH 150/361] feat: add remove index option

---
 frappe/modules/full_text_search.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 7ccc68f877..022200f3cc 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -120,5 +120,17 @@ def reindex(index_name, document):
 
 		writer.commit(optimize=True)
 
+def remove_from_index(index_name, path):
+	# open index
+	index_dir = get_index_path(index_name)
+	ix = open_dir(index_dir)
+
+	# initiate search scope to find previous index
+	with ix.searcher() as searcher:
+		writer = ix.writer()
+		
+		# Remove the index of the particular file
+		writer.delete_by_term('path', path)
+
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From 702a6f103abbc0fa4150787de645944c7b983fd1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 21:02:42 +0530
Subject: [PATCH 151/361] fix: commit after removing index

---
 frappe/modules/full_text_search.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 022200f3cc..03fb8d8bd6 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -132,5 +132,7 @@ def remove_from_index(index_name, path):
 		# Remove the index of the particular file
 		writer.delete_by_term('path', path)
 
+		writer.commit(optimize=True)
+
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)

From f0547ce0d0fa51e37eb59eaff2e926583d72f83e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 17 Jun 2020 23:10:31 +0530
Subject: [PATCH 152/361] feat: added index hooks to website generator

---
 frappe/modules/full_text_search.py  |  8 ++++++--
 frappe/website/website_generator.py | 16 +++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 03fb8d8bd6..e3519690c5 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -100,8 +100,12 @@ def search(index_name, text, scope=None, limit=20):
 			)
 
 	return out
+
+def update_index_for_path(index_name, path):
+	document = get_document_to_index(path)
+	update_index(index_name, document)
 	
-def reindex(index_name, document):
+def update_index(index_name, document):
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
@@ -120,7 +124,7 @@ def reindex(index_name, document):
 
 		writer.commit(optimize=True)
 
-def remove_from_index(index_name, path):
+def remove_document_from_index(index_name, path):
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 1a32ef9ffa..e8a13e970c 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -7,6 +7,7 @@ from frappe.model.document import Document
 from frappe.website.utils import cleanup_page_name
 from frappe.website.render import clear_cache
 from frappe.modules import get_module_name
+from frappe.modules.full_text_search import update_index_for_path, remove_document_from_index
 
 class WebsiteGenerator(Document):
 	website = frappe._dict()
@@ -82,11 +83,18 @@ class WebsiteGenerator(Document):
 		pass
 
 	def on_update(self):
+		old_doc = self.get_doc_before_save()
+		if old_doc.route != self.route:
+			remove_document_from_index("web_routes", old_doc.route)
 		self.send_indexing_request()
 
+	def on_change(self):
+		self.update_website_search_index()
+
 	def on_trash(self):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
+		remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""
@@ -129,4 +137,10 @@ class WebsiteGenerator(Document):
 
 			url = frappe.utils.get_url(self.route)
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
-				url=url, operation_type=operation_type)
\ No newline at end of file
+				url=url, operation_type=operation_type)
+
+	def update_website_search_index(self):
+		if not self.is_website_published():
+			remove_document_from_index("web_routes", self.route)
+		else:
+			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file

From 841914c110f324515c3be6536196c6b8cbc2280a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:10:55 +0530
Subject: [PATCH 153/361] feat: allow disabling indexing for certain routes

---
 frappe/website/doctype/web_form/web_form.py |  3 ++
 frappe/website/website_generator.py         | 32 ++++++++++++++++++---
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index a3867f1396..4ac91c852f 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -343,6 +343,9 @@ def get_context(context):
 			frappe.throw(_('Mandatory Information missing:') + '<br><br>'
 				+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
 
+	def enable_website_search_indexing(self):
+		return False
+
 
 @frappe.whitelist(allow_guest=True)
 def accept(web_form, data, docname=None, for_payment=False):
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index e8a13e970c..075e5644ac 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -83,18 +83,20 @@ class WebsiteGenerator(Document):
 		pass
 
 	def on_update(self):
-		old_doc = self.get_doc_before_save()
-		if old_doc.route != self.route:
-			remove_document_from_index("web_routes", old_doc.route)
 		self.send_indexing_request()
+		self.remove_old_route_from_index()
 
 	def on_change(self):
+		# Update the index on change
+		# On change is triggered last in the event lifecycle
 		self.update_website_search_index()
 
 	def on_trash(self):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
-		remove_document_from_index("web_routes", self.route)
+		# On deleting the doc, remove the page from the web_routes index
+		if self.enable_website_search_indexing():
+			remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""
@@ -139,8 +141,30 @@ class WebsiteGenerator(Document):
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
 				url=url, operation_type=operation_type)
 
+	# Override this method to disable indexing
+	def enable_website_search_indexing(self):
+		return True
+
+	def remove_old_route_from_index(self):
+		"""Remove page from the website index if the route has changed."""
+		if not self.enable_website_search_indexing():
+			return
+		old_doc = self.get_doc_before_save()
+		# Check if the route is changed
+		if old_doc.route != self.route:
+			# Remove the route from index if the route has changed
+			remove_document_from_index("web_routes", old_doc.route)
+
 	def update_website_search_index(self):
+		"""
+			Update the full test index executed on document change event.
+			- remove document from index if document is unpublished
+			- update index otherwise
+		"""
+		if not self.enable_website_search_indexing():
+			return
 		if not self.is_website_published():
+			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
 		else:
 			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file

From f9b59a7b1029e2f756af08cc9c34e070ed7ffbca Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:20:52 +0530
Subject: [PATCH 154/361] chore: add docstrings

---
 frappe/modules/full_text_search.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index e3519690c5..6f7be30999 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -102,10 +102,24 @@ def search(index_name, text, scope=None, limit=20):
 	return out
 
 def update_index_for_path(index_name, path):
+	"""Wraps `update_index` method, gets the document from path
+	and updates the index. This function changes the current user
+	and should only be run as administrator or in a background job.
+
+	Args:
+		index_name (str): name of the index
+		path (str): route of the page to be updated 
+	"""
 	document = get_document_to_index(path)
 	update_index(index_name, document)
 	
 def update_index(index_name, document):
+	"""Update search index for a document
+
+	Args:
+		index_name (str): Name of the index to be updated
+		document (dict): A dictionary with title, path and content
+	"""
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)
@@ -125,6 +139,12 @@ def update_index(index_name, document):
 		writer.commit(optimize=True)
 
 def remove_document_from_index(index_name, path):
+	"""Remove document from search index
+
+	Args:
+		index_name (str): name of the index
+		path (str): route of the page to be removed 
+	"""
 	# open index
 	index_dir = get_index_path(index_name)
 	ix = open_dir(index_dir)

From d2d37a1e0493bf8c275f3ae3f4dfb5747a47eac1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:34:28 +0530
Subject: [PATCH 155/361] feat: move whoosh full_text_search build to migrate
 from hooks

---
 frappe/hooks.py   | 2 +-
 frappe/migrate.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/hooks.py b/frappe/hooks.py
index 1f209f00a2..0850cd6f91 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -273,7 +273,7 @@ setup_wizard_exception = [
 
 before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute']
 after_migrate = [
-	'frappe.modules.full_text_search.build_index_for_all_routes'
+	'frappe.website.doctype.website_theme.website_theme.generate_theme_files_if_not_exist'
 ]
 
 otp_methods = ['OTP App','Email','SMS']
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 9ec23d8ae7..88c7bfdbb2 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -19,6 +19,7 @@ from frappe.website import render
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
+from frappe.modules import full_text_search
 from frappe.utils import global_search
 
 
@@ -82,6 +83,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 
 		# add static pages to global search
 		global_search.update_global_search_for_all_web_pages()
+		full_text_search.build_index_for_all_routes()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

From 51b8e467add2b756f27013ce65e2de0dfc82ee63 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 18 Jun 2020 13:34:43 +0530
Subject: [PATCH 156/361] feat: add flag to disable search index rebuild on
 migrate

---
 frappe/commands/site.py | 10 ++++++++--
 frappe/migrate.py       |  7 ++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 26eb455338..d64a4609e3 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,8 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
+@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False):
+def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -284,7 +285,12 @@ def migrate(context, rebuild_website=False, skip_failing=False):
 		frappe.init(site=site)
 		frappe.connect()
 		try:
-			migrate(context.verbose, rebuild_website=rebuild_website, skip_failing=skip_failing)
+			migrate(
+				context.verbose,
+				rebuild_website=rebuild_website,
+				skip_failing=skip_failing,
+				skip_search_index=skip_search_index
+			)
 		finally:
 			frappe.destroy()
 	if not context.sites:
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 88c7bfdbb2..544e0f6b6f 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -23,7 +23,7 @@ from frappe.modules import full_text_search
 from frappe.utils import global_search
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -82,8 +82,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 		render.clear_cache()
 
 		# add static pages to global search
-		global_search.update_global_search_for_all_web_pages()
-		full_text_search.build_index_for_all_routes()
+		if not skip_search_index:
+			global_search.update_global_search_for_all_web_pages()
+			full_text_search.build_index_for_all_routes()
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()

From 8bb473143b82ad925476627081e57653e685ca49 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:20:03 +0530
Subject: [PATCH 157/361] refactor: FullTextSearch class

---
 frappe/modules/full_text_search.py | 303 ++++++++++++++++-------------
 1 file changed, 163 insertions(+), 140 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 6f7be30999..f0c83b7cb3 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -10,153 +10,176 @@ from whoosh.query import Prefix
 from bs4 import BeautifulSoup
 from frappe.website.render import render_page
 from frappe.utils import set_request, cint
-from frappe.utils.global_search import get_routes_to_index
+import os
+
+class FullTextSearch:
+	def __init__(self, index_name):
+		self.index_name = index_name
+		self.index_path = get_index_path(index_name)
+		self.schema = Schema(
+			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
+		)
+
+	def get_routes_to_index(self):
+		routes = get_static_pages_from_all_apps()
+		return routes
+
+	def build(self):
+		print("Building search index for all web routes...")
+		routes = self.get_routes_to_index()
+		self.documents = [self.get_document_to_index(route) for route in routes]
+		self.build_index()
+
+	def get_document_to_index(self, route):
+		frappe.set_user("Guest")
+		frappe.local.no_cache = True
+
+		try:
+			set_request(method="GET", path=route)
+			content = render_page(route)
+			soup = BeautifulSoup(content, "html.parser")
+			page_content = soup.find(class_="page_content")
+			text_content = page_content.text if page_content else ""
+			title = soup.title.text.strip() if soup.title else route
+
+			frappe.set_user("Administrator")
+
+			return frappe._dict(title=title, content=text_content, path=route)
+		except (
+			frappe.PermissionError,
+			frappe.DoesNotExistError,
+			frappe.ValidationError,
+			Exception,
+		):
+			pass
+
+	def update_index_for_path(self, path):
+		"""Wraps `update_index` method, gets the document from path
+		and updates the index. This function changes the current user
+		and should only be run as administrator or in a background job.
+
+		Args:
+			self (object): FullTextSearch Instance
+			path (str): route of the page to be updated 
+		"""
+		document = self.get_document_to_index(path)
+		self.update_index(document)
+
+	def update_index(self, document):
+		"""Update search index for a document
+
+		Args:
+			self (object): FullTextSearch Instance
+			document (dict): A dictionary with title, path and content
+		"""
+		ix = open_dir(self.index_path)
+
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term('path', document.path)
+			writer.add_document(
+				title=document.title, path=document.path, content=document.content
+			)
+			writer.commit(optimize=True)
+
+	def remove_document_from_index(self, path):
+		"""Remove document from search index
+
+		Args:
+			self (object): FullTextSearch Instance
+			path (str): route of the page to be removed 
+		"""
+		ix = open_dir(self.index_path)
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term('path', path)
+			writer.commit(optimize=True)
+
+	
+	def build_index(self):
+		ix = create_in(self.index_path, self.schema)
+		writer = ix.writer()
+
+		for document in self.documents:
+			if document:
+				writer.add_document(
+					title=document.title, path=document.path, content=document.content
+				)
+
+		writer.commit(optimize=True)
+
+	def search(self, text, scope=None, limit=20):
+		ix = open_dir(self.index_path)
+
+		results = None
+		out = []
+		
+		with ix.searcher() as searcher:
+			parser = MultifieldParser(["title", "content"], ix.schema)
+			parser.remove_plugin_class(FieldsPlugin)
+			parser.remove_plugin_class(WildcardPlugin)
+			query = parser.parse(text)
+
+			filter_scoped = None
+			if scope:
+				filter_scoped = Prefix("path", scope)
+			results = searcher.search(query, limit=limit, filter=filter_scoped)
+
+			for r in results:
+				title_highlights = r.highlights("title")
+				content_highlights = r.highlights("content")
+				out.append(
+					frappe._dict(
+						title=r["title"],
+						path=r["path"],
+						title_highlights=title_highlights,
+						content_highlights=content_highlights,
+					)
+				)
+
+		return out
 
 
-def build_index_for_all_routes():
-	print("Building search index for all web routes...")
-	routes = get_routes_to_index()
-	documents = [get_document_to_index(route) for route in routes]
-	build_index("web_routes", documents)
+def get_doctype_routes_with_web_view():
+	doctype_with_web_views = frappe.get_all("DocType", { "has_web_view": 1 })
+	return []
 
+def get_static_pages_from_all_apps():
+	apps = frappe.get_installed_apps()
+
+	routes_to_index = []
+	for app in apps:
+		base = frappe.get_app_path(app, 'www')
+		path_to_index = frappe.get_app_path(app, 'www')
+
+		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
+			for f in filenames:
+				if f.endswith(('.md', '.html')):
+					filepath = os.path.join(dirpath, f)
+
+					route = os.path.relpath(filepath, base)
+					route = route.split('.')[0]
+
+					if route.endswith('index'):
+						route = route.rsplit('index', 1)[0]
+
+					routes_to_index.append(route)
+
+	return routes_to_index
 
 @frappe.whitelist(allow_guest=True)
 def web_search(index_name, query, scope=None, limit=20):
 	limit = cint(limit)
-	return search(index_name, query, scope, limit)
-
-
-def get_document_to_index(route):
-	frappe.set_user("Guest")
-	frappe.local.no_cache = True
-
-	try:
-		set_request(method="GET", path=route)
-		content = render_page(route)
-		soup = BeautifulSoup(content, "html.parser")
-		page_content = soup.find(class_="page_content")
-		text_content = page_content.text if page_content else ""
-		title = soup.title.text.strip() if soup.title else route
-
-		frappe.set_user("Administrator")
-
-		return frappe._dict(title=title, content=text_content, path=route)
-	except (
-		frappe.PermissionError,
-		frappe.DoesNotExistError,
-		frappe.ValidationError,
-		Exception,
-	):
-		pass
-
-
-def build_index(index_name, documents):
-	schema = Schema(
-		title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
-	)
-
-	index_dir = get_index_path(index_name)
-	frappe.create_folder(index_dir)
-
-	ix = create_in(index_dir, schema)
-	writer = ix.writer()
-
-	for document in documents:
-		if document:
-			writer.add_document(
-				title=document.title, path=document.path, content=document.content
-			)
-
-	writer.commit(optimize=True)
-
-
-def search(index_name, text, scope=None, limit=20):
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	results = None
-	out = []
-	with ix.searcher() as searcher:
-		parser = MultifieldParser(["title", "content"], ix.schema)
-		parser.remove_plugin_class(FieldsPlugin)
-		parser.remove_plugin_class(WildcardPlugin)
-		query = parser.parse(text)
-
-		filter_scoped = None
-		if scope:
-			filter_scoped = Prefix("path", scope)
-		results = searcher.search(query, limit=limit, filter=filter_scoped)
-
-		for r in results:
-			title_highlights = r.highlights("title")
-			content_highlights = r.highlights("content")
-			out.append(
-				frappe._dict(
-					title=r["title"],
-					path=r["path"],
-					title_highlights=title_highlights,
-					content_highlights=content_highlights,
-				)
-			)
-
-	return out
-
-def update_index_for_path(index_name, path):
-	"""Wraps `update_index` method, gets the document from path
-	and updates the index. This function changes the current user
-	and should only be run as administrator or in a background job.
-
-	Args:
-		index_name (str): name of the index
-		path (str): route of the page to be updated 
-	"""
-	document = get_document_to_index(path)
-	update_index(index_name, document)
-	
-def update_index(index_name, document):
-	"""Update search index for a document
-
-	Args:
-		index_name (str): Name of the index to be updated
-		document (dict): A dictionary with title, path and content
-	"""
-	# open index
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	# initiate search scope to find previous index
-	with ix.searcher() as searcher:
-		writer = ix.writer()
-		
-		# Remove the index of the particular file
-		writer.delete_by_term('path', document.path)
-
-		# re index it!
-		writer.add_document(
-			title=document.title, path=document.path, content=document.content
-		)
-
-		writer.commit(optimize=True)
-
-def remove_document_from_index(index_name, path):
-	"""Remove document from search index
-
-	Args:
-		index_name (str): name of the index
-		path (str): route of the page to be removed 
-	"""
-	# open index
-	index_dir = get_index_path(index_name)
-	ix = open_dir(index_dir)
-
-	# initiate search scope to find previous index
-	with ix.searcher() as searcher:
-		writer = ix.writer()
-		
-		# Remove the index of the particular file
-		writer.delete_by_term('path', path)
-
-		writer.commit(optimize=True)
+	fts = FullTextSearch(index_name)
+	return fts.search(query, scope, limit)
 
 def get_index_path(index_name):
 	return frappe.get_site_path("indexes", index_name)
+
+def update_index_for_path(index_name, path):
+	fts = FullTextSearch(index_name)
+	return fts.update_index_for_path(path)
+
+def remove_document_from_index(index_name, path):
+	fts = FullTextSearch(index_name)
+	return fts.remove_document_from_index(path)

From 59ce616153272fa641283f0991d434273be1e394 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:29:34 +0530
Subject: [PATCH 158/361] chore: added docstrings

---
 frappe/modules/full_text_search.py | 31 +++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index f0c83b7cb3..2c48df4d7b 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -13,6 +13,8 @@ from frappe.utils import set_request, cint
 import os
 
 class FullTextSearch:
+	""" Frappe Wrapper for Whoosh """
+
 	def __init__(self, index_name):
 		self.index_name = index_name
 		self.index_path = get_index_path(index_name)
@@ -21,16 +23,32 @@ class FullTextSearch:
 		)
 
 	def get_routes_to_index(self):
+		"""Get all routes to be indexed, this includes the static pages
+		in www/ and routes from published documents
+
+		Returns:
+			self (object): FullTextSearch Instance
+		"""
 		routes = get_static_pages_from_all_apps()
 		return routes
 
 	def build(self):
+		"""	Build search index for all web routes """
+
 		print("Building search index for all web routes...")
 		routes = self.get_routes_to_index()
 		self.documents = [self.get_document_to_index(route) for route in routes]
 		self.build_index()
 
 	def get_document_to_index(self, route):
+		"""Render a page and parse it using BeautifulSoup
+
+		Args:
+			path (str): route of the page to be parsed 
+
+		Returns:
+			document (_dict): A dictionary with title, path and content
+		"""
 		frappe.set_user("Guest")
 		frappe.local.no_cache = True
 
@@ -70,7 +88,7 @@ class FullTextSearch:
 
 		Args:
 			self (object): FullTextSearch Instance
-			document (dict): A dictionary with title, path and content
+			document (_dict): A dictionary with title, path and content
 		"""
 		ix = open_dir(self.index_path)
 
@@ -97,6 +115,7 @@ class FullTextSearch:
 
 	
 	def build_index(self):
+		"""Build index for all parsed documents"""
 		ix = create_in(self.index_path, self.schema)
 		writer = ix.writer()
 
@@ -109,6 +128,16 @@ class FullTextSearch:
 		writer.commit(optimize=True)
 
 	def search(self, text, scope=None, limit=20):
+		"""Search from the current index
+
+		Args:
+			text (str): String to search for
+			scope (str, optional): Scope to limit the search. Defaults to None.
+			limit (int, optional): Limit number of search results. Defaults to 20.
+
+		Returns:
+			[List(_dict)]: Search results
+		"""
 		ix = open_dir(self.index_path)
 
 		results = None

From eb9435c0124941d6e550424a9287f640a2c6722e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:55:33 +0530
Subject: [PATCH 159/361] refactor: rename enable_website_search_indexing to
 allow_website_search_indexing

---
 frappe/website/doctype/web_form/web_form.py | 2 +-
 frappe/website/website_generator.py         | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py
index 4ac91c852f..cbec84752e 100644
--- a/frappe/website/doctype/web_form/web_form.py
+++ b/frappe/website/doctype/web_form/web_form.py
@@ -343,7 +343,7 @@ def get_context(context):
 			frappe.throw(_('Mandatory Information missing:') + '<br><br>'
 				+ '<br>'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing]))
 
-	def enable_website_search_indexing(self):
+	def allow_website_search_indexing(self):
 		return False
 
 
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 075e5644ac..12743af62e 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -95,7 +95,7 @@ class WebsiteGenerator(Document):
 		self.clear_cache()
 		self.send_indexing_request('URL_DELETED')
 		# On deleting the doc, remove the page from the web_routes index
-		if self.enable_website_search_indexing():
+		if self.allow_website_search_indexing():
 			remove_document_from_index("web_routes", self.route)
 
 	def is_website_published(self):
@@ -142,12 +142,12 @@ class WebsiteGenerator(Document):
 				url=url, operation_type=operation_type)
 
 	# Override this method to disable indexing
-	def enable_website_search_indexing(self):
+	def allow_website_search_indexing(self):
 		return True
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
-		if not self.enable_website_search_indexing():
+		if self.allow_website_search_indexing():
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
@@ -161,7 +161,7 @@ class WebsiteGenerator(Document):
 			- remove document from index if document is unpublished
 			- update index otherwise
 		"""
-		if not self.enable_website_search_indexing():
+		if not self.allow_website_search_indexing():
 			return
 		if not self.is_website_published():
 			# If the website is not published

From 90456a979e45fde047bfd864f9647095ce85e399 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 14:55:54 +0530
Subject: [PATCH 160/361] feat: allow indexing only if route is present

---
 frappe/website/website_generator.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 12743af62e..f8289f53f8 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -143,7 +143,7 @@ class WebsiteGenerator(Document):
 
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return True
+		return bool(self.route)
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
@@ -151,7 +151,7 @@ class WebsiteGenerator(Document):
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
-		if old_doc.route != self.route:
+		if old_doc and old_doc.route != self.route:
 			# Remove the route from index if the route has changed
 			remove_document_from_index("web_routes", old_doc.route)
 
@@ -163,8 +163,10 @@ class WebsiteGenerator(Document):
 		"""
 		if not self.allow_website_search_indexing():
 			return
-		if not self.is_website_published():
+		
+		if self.is_website_published():
+			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
+		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
-		else:
-			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
\ No newline at end of file
+			
\ No newline at end of file

From a60a6fc04a5e32a53a54b4e9f98051a72b42d7ee Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 19 Jun 2020 15:33:29 +0530
Subject: [PATCH 161/361] feat: function to get routes from published docitems

---
 frappe/modules/full_text_search.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index 2c48df4d7b..f626ea8490 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -30,6 +30,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 		"""
 		routes = get_static_pages_from_all_apps()
+		routes += get_doctype_routes_with_web_view()
 		return routes
 
 	def build(self):
@@ -170,8 +171,17 @@ class FullTextSearch:
 
 
 def get_doctype_routes_with_web_view():
-	doctype_with_web_views = frappe.get_all("DocType", { "has_web_view": 1 })
-	return []
+	all_routes = []
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	fields = ["name", "is_published_field"]
+	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
+
+	for doctype in doctype_with_web_views:
+		if doctype.is_published_field:
+			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
+			all_routes += [route.route for route in routes]
+
+	return all_routes
 
 def get_static_pages_from_all_apps():
 	apps = frappe.get_installed_apps()

From 19f42d187759d75f4d04d4766353ef93e8057ff2 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 13:29:11 +0530
Subject: [PATCH 162/361] feat: added index all routes function

---
 frappe/migrate.py                  | 4 ++--
 frappe/modules/full_text_search.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 544e0f6b6f..40c9bd4068 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -83,8 +83,8 @@ Otherwise, check the server logs and ensure that all the required services are r
 
 		# add static pages to global search
 		if not skip_search_index:
-			global_search.update_global_search_for_all_web_pages()
-			full_text_search.build_index_for_all_routes()
+			# global_search.update_global_search_for_all_web_pages()
+			full_text_search.build_index_for_all_routes("web_routes")
 
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()
diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
index f626ea8490..b1003ecc97 100644
--- a/frappe/modules/full_text_search.py
+++ b/frappe/modules/full_text_search.py
@@ -222,3 +222,7 @@ def update_index_for_path(index_name, path):
 def remove_document_from_index(index_name, path):
 	fts = FullTextSearch(index_name)
 	return fts.remove_document_from_index(path)
+
+def build_index_for_all_routes(index_name):
+	fts = FullTextSearch(index_name)
+	return fts.build()
\ No newline at end of file

From 16a2ef78ff3c3a5b33b7cd9e9ef99c5c7f19231a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 13:39:39 +0530
Subject: [PATCH 163/361] style: add missing semicolon

---
 frappe/website/js/website.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index b69077f482..1c739cc6ca 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -439,8 +439,8 @@ frappe.setup_search = function (target, search_scope) {
 					return `<a class="dropdown-item" href="/${r.path}">
 						<h6>${r.title_highlights || r.title}</h6>
 						<div style="white-space: normal;">${r.content_highlights}</div>
-					</a>`
-				}).join('')
+					</a>`;
+				}).join('');
 			}
 			$dropdown_menu.html(dropdown_html);
 			$dropdown_menu.addClass('show');
@@ -463,7 +463,7 @@ frappe.setup_search = function (target, search_scope) {
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 	}
-}
+};
 
 
 // Utility functions

From edf13c92e10d9cba9784047899f8e7a765a7610a Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 22 Jun 2020 16:29:59 +0530
Subject: [PATCH 164/361] feat: separate search style rules from doc layout

---
 frappe/public/scss/doc.scss     | 35 --------------------------------
 frappe/public/scss/search.scss  | 36 +++++++++++++++++++++++++++++++++
 frappe/public/scss/website.scss |  1 +
 frappe/templates/doc.html       |  2 +-
 4 files changed, 38 insertions(+), 36 deletions(-)
 create mode 100644 frappe/public/scss/search.scss

diff --git a/frappe/public/scss/doc.scss b/frappe/public/scss/doc.scss
index 13a59ba45b..f258e2ee47 100644
--- a/frappe/public/scss/doc.scss
+++ b/frappe/public/scss/doc.scss
@@ -81,45 +81,10 @@ $navbar-height-lg: 4.5rem;
 }
 
 .doc-search {
-	position: relative;
-	width: 100%;
-
 	@include media-breakpoint-up(lg) {
 		padding-left: 4rem;
 		padding-right: 4rem;
 	}
-
-	.search-icon {
-		position: absolute;
-		left: 0;
-		top: 0;
-		width: 2.5rem;
-		height: 100%;
-		display: flex;
-		justify-content: center;
-		align-items: center;
-	}
-
-	svg {
-		color: $gray-600;
-	}
-
-	input {
-		padding-left: 2.5rem;
-	}
-
-	.dropdown-menu {
-		.dropdown-item {
-			padding: 1rem 0.75rem;
-		}
-
-		.match {
-			background-color: $primary-light;
-			color: $primary;
-			font-weight: 500;
-			padding: 0 0.125rem;
-		}
-	}
 }
 
 .doc-sidebar {
diff --git a/frappe/public/scss/search.scss b/frappe/public/scss/search.scss
new file mode 100644
index 0000000000..929b73032d
--- /dev/null
+++ b/frappe/public/scss/search.scss
@@ -0,0 +1,36 @@
+.website-search {
+	position: relative;
+	width: 100%;
+
+	.search-icon {
+		position: absolute;
+		left: 0;
+		top: 0;
+		width: 2.5rem;
+		height: 100%;
+		display: flex;
+		justify-content: center;
+		align-items: center;
+	}
+
+	svg {
+		color: $gray-600;
+	}
+
+	input {
+		padding-left: 2.5rem;
+	}
+
+	.dropdown-menu {
+		.dropdown-item {
+			padding: 1rem 0.75rem;
+		}
+
+		.match {
+			background-color: $primary-light;
+			color: $primary;
+			font-weight: 500;
+			padding: 0 0.125rem;
+		}
+	}
+}
\ No newline at end of file
diff --git a/frappe/public/scss/website.scss b/frappe/public/scss/website.scss
index 3c11d23252..e64c090ea8 100644
--- a/frappe/public/scss/website.scss
+++ b/frappe/public/scss/website.scss
@@ -10,6 +10,7 @@
 @import 'markdown';
 @import 'sidebar';
 @import 'portal';
+@import 'search';
 @import 'doc';
 
 .ql-editor.read-mode {
diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index 1c70aace98..f45cf3d29e 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -22,7 +22,7 @@
 			</div>
 			<div class="col-12 col-lg-8">
 				<div class="doc-search-container">
-					<div class="doc-search" id="search-container">
+					<div class="website-search doc-search" id="search-container">
 						<div class="dropdown">
 							<div class="search-icon">
 								<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"

From 5145c58b815dc048d15f15beaa98037adb991eb3 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 23 Jun 2020 13:09:02 +0530
Subject: [PATCH 165/361] feat: [breaking] remove global search from migrate

* Run search last as it changes the session user
---
 frappe/migrate.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 40c9bd4068..26264bad6c 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -20,7 +20,6 @@ from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.modules import full_text_search
-from frappe.utils import global_search
 
 
 def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
@@ -81,11 +80,6 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# syncs statics
 		render.clear_cache()
 
-		# add static pages to global search
-		if not skip_search_index:
-			# global_search.update_global_search_for_all_web_pages()
-			full_text_search.build_index_for_all_routes("web_routes")
-
 		# updating installed applications data
 		frappe.get_single('Installed Applications').update_versions()
 
@@ -94,6 +88,11 @@ Otherwise, check the server logs and ensure that all the required services are r
 			for fn in frappe.get_hooks('after_migrate', app_name=app):
 				frappe.get_attr(fn)()
 
+		# add static pages to global search
+		if not skip_search_index:
+			# Run this last as it updates the current session
+			full_text_search.build_index_for_all_routes("web_routes")
+
 		frappe.db.commit()
 
 		clear_notifications()

From 602366e285eaac231d2836efa04a951c21df9c9f Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:27:41 +0530
Subject: [PATCH 166/361] feat: separate FTS and website search inside
 frappe.search

---
 frappe/modules/full_text_search.py | 228 -----------------------------
 frappe/search/__init__.py          |  13 ++
 frappe/search/full_text_search.py  | 128 ++++++++++++++++
 frappe/search/website_search.py    | 123 ++++++++++++++++
 4 files changed, 264 insertions(+), 228 deletions(-)
 delete mode 100644 frappe/modules/full_text_search.py
 create mode 100644 frappe/search/__init__.py
 create mode 100644 frappe/search/full_text_search.py
 create mode 100644 frappe/search/website_search.py

diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py
deleted file mode 100644
index b1003ecc97..0000000000
--- a/frappe/modules/full_text_search.py
+++ /dev/null
@@ -1,228 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals
-import frappe
-from whoosh.index import create_in, open_dir
-from whoosh.fields import TEXT, ID, Schema
-from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
-from whoosh.query import Prefix
-from bs4 import BeautifulSoup
-from frappe.website.render import render_page
-from frappe.utils import set_request, cint
-import os
-
-class FullTextSearch:
-	""" Frappe Wrapper for Whoosh """
-
-	def __init__(self, index_name):
-		self.index_name = index_name
-		self.index_path = get_index_path(index_name)
-		self.schema = Schema(
-			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
-		)
-
-	def get_routes_to_index(self):
-		"""Get all routes to be indexed, this includes the static pages
-		in www/ and routes from published documents
-
-		Returns:
-			self (object): FullTextSearch Instance
-		"""
-		routes = get_static_pages_from_all_apps()
-		routes += get_doctype_routes_with_web_view()
-		return routes
-
-	def build(self):
-		"""	Build search index for all web routes """
-
-		print("Building search index for all web routes...")
-		routes = self.get_routes_to_index()
-		self.documents = [self.get_document_to_index(route) for route in routes]
-		self.build_index()
-
-	def get_document_to_index(self, route):
-		"""Render a page and parse it using BeautifulSoup
-
-		Args:
-			path (str): route of the page to be parsed 
-
-		Returns:
-			document (_dict): A dictionary with title, path and content
-		"""
-		frappe.set_user("Guest")
-		frappe.local.no_cache = True
-
-		try:
-			set_request(method="GET", path=route)
-			content = render_page(route)
-			soup = BeautifulSoup(content, "html.parser")
-			page_content = soup.find(class_="page_content")
-			text_content = page_content.text if page_content else ""
-			title = soup.title.text.strip() if soup.title else route
-
-			frappe.set_user("Administrator")
-
-			return frappe._dict(title=title, content=text_content, path=route)
-		except (
-			frappe.PermissionError,
-			frappe.DoesNotExistError,
-			frappe.ValidationError,
-			Exception,
-		):
-			pass
-
-	def update_index_for_path(self, path):
-		"""Wraps `update_index` method, gets the document from path
-		and updates the index. This function changes the current user
-		and should only be run as administrator or in a background job.
-
-		Args:
-			self (object): FullTextSearch Instance
-			path (str): route of the page to be updated 
-		"""
-		document = self.get_document_to_index(path)
-		self.update_index(document)
-
-	def update_index(self, document):
-		"""Update search index for a document
-
-		Args:
-			self (object): FullTextSearch Instance
-			document (_dict): A dictionary with title, path and content
-		"""
-		ix = open_dir(self.index_path)
-
-		with ix.searcher() as searcher:
-			writer = ix.writer()
-			writer.delete_by_term('path', document.path)
-			writer.add_document(
-				title=document.title, path=document.path, content=document.content
-			)
-			writer.commit(optimize=True)
-
-	def remove_document_from_index(self, path):
-		"""Remove document from search index
-
-		Args:
-			self (object): FullTextSearch Instance
-			path (str): route of the page to be removed 
-		"""
-		ix = open_dir(self.index_path)
-		with ix.searcher() as searcher:
-			writer = ix.writer()
-			writer.delete_by_term('path', path)
-			writer.commit(optimize=True)
-
-	
-	def build_index(self):
-		"""Build index for all parsed documents"""
-		ix = create_in(self.index_path, self.schema)
-		writer = ix.writer()
-
-		for document in self.documents:
-			if document:
-				writer.add_document(
-					title=document.title, path=document.path, content=document.content
-				)
-
-		writer.commit(optimize=True)
-
-	def search(self, text, scope=None, limit=20):
-		"""Search from the current index
-
-		Args:
-			text (str): String to search for
-			scope (str, optional): Scope to limit the search. Defaults to None.
-			limit (int, optional): Limit number of search results. Defaults to 20.
-
-		Returns:
-			[List(_dict)]: Search results
-		"""
-		ix = open_dir(self.index_path)
-
-		results = None
-		out = []
-		
-		with ix.searcher() as searcher:
-			parser = MultifieldParser(["title", "content"], ix.schema)
-			parser.remove_plugin_class(FieldsPlugin)
-			parser.remove_plugin_class(WildcardPlugin)
-			query = parser.parse(text)
-
-			filter_scoped = None
-			if scope:
-				filter_scoped = Prefix("path", scope)
-			results = searcher.search(query, limit=limit, filter=filter_scoped)
-
-			for r in results:
-				title_highlights = r.highlights("title")
-				content_highlights = r.highlights("content")
-				out.append(
-					frappe._dict(
-						title=r["title"],
-						path=r["path"],
-						title_highlights=title_highlights,
-						content_highlights=content_highlights,
-					)
-				)
-
-		return out
-
-
-def get_doctype_routes_with_web_view():
-	all_routes = []
-	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
-	fields = ["name", "is_published_field"]
-	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
-
-	for doctype in doctype_with_web_views:
-		if doctype.is_published_field:
-			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
-			all_routes += [route.route for route in routes]
-
-	return all_routes
-
-def get_static_pages_from_all_apps():
-	apps = frappe.get_installed_apps()
-
-	routes_to_index = []
-	for app in apps:
-		base = frappe.get_app_path(app, 'www')
-		path_to_index = frappe.get_app_path(app, 'www')
-
-		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
-			for f in filenames:
-				if f.endswith(('.md', '.html')):
-					filepath = os.path.join(dirpath, f)
-
-					route = os.path.relpath(filepath, base)
-					route = route.split('.')[0]
-
-					if route.endswith('index'):
-						route = route.rsplit('index', 1)[0]
-
-					routes_to_index.append(route)
-
-	return routes_to_index
-
-@frappe.whitelist(allow_guest=True)
-def web_search(index_name, query, scope=None, limit=20):
-	limit = cint(limit)
-	fts = FullTextSearch(index_name)
-	return fts.search(query, scope, limit)
-
-def get_index_path(index_name):
-	return frappe.get_site_path("indexes", index_name)
-
-def update_index_for_path(index_name, path):
-	fts = FullTextSearch(index_name)
-	return fts.update_index_for_path(path)
-
-def remove_document_from_index(index_name, path):
-	fts = FullTextSearch(index_name)
-	return fts.remove_document_from_index(path)
-
-def build_index_for_all_routes(index_name):
-	fts = FullTextSearch(index_name)
-	return fts.build()
\ No newline at end of file
diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py
new file mode 100644
index 0000000000..462418ca41
--- /dev/null
+++ b/frappe/search/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+from frappe.utils import cint
+from frappe.search.website_search import WebsiteSearch
+from frappe.search.website_search import FullTextSearch
+
+@frappe.whitelist(allow_guest=True)
+def web_search(query, scope=None, limit=20):
+	limit = cint(limit)
+	ws = WebsiteSearch(index_name="web_routes")
+	return ws.search(query, scope, limit)
\ No newline at end of file
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
new file mode 100644
index 0000000000..13fcf22e9b
--- /dev/null
+++ b/frappe/search/full_text_search.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+from whoosh.index import create_in, open_dir
+from whoosh.fields import TEXT, ID, Schema
+from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
+from whoosh.query import Prefix
+import os
+
+class FullTextSearch:
+	""" Frappe Wrapper for Whoosh """
+
+	def __init__(self, index_name):
+		self.index_name = index_name
+		self.index_path = get_index_path(index_name)
+		self.schema = self.get_schema()
+		self.id = self.get_id()
+
+	def get_schema(self):
+		return Schema(name=ID(stored=True), content=TEXT(stored=True))
+
+	def get_id(self):
+		return "name"
+
+	def get_items_to_index(self):
+		"""Get all documents to be indexed conforming to the schema"""
+		return []
+
+	def get_document_to_index(self):
+		return {}
+
+	def build(self):
+		"""	Build search index for all documents """
+
+		print("Building search index for all web routes...")
+		self.documents = self.get_items_to_index()
+		self.build_index()
+
+	def update_index_by_name(self, doc_name):
+		"""Wraps `update_index` method, gets the document from name
+		and updates the index. This function changes the current user
+		and should only be run as administrator or in a background job.
+
+		Args:
+			self (object): FullTextSearch Instance
+			doc_name (str): name of the document to be updated
+		"""
+		document = self.get_document_to_index(doc_name)
+		self.update_index(document)
+
+	def remove_document_from_index(self, doc_name):
+		"""Remove document from search index
+
+		Args:
+			self (object): FullTextSearch Instance
+			doc_name (str): name of the document to be removed
+		"""
+		ix = open_dir(self.index_path)
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term(self.id, doc_name)
+			writer.commit(optimize=True)
+
+	def update_index(self, document):
+		"""Update search index for a document
+
+		Args:
+			self (object): FullTextSearch Instance
+			document (_dict): A dictionary with title, path and content
+		"""
+		ix = open_dir(self.index_path)
+
+		with ix.searcher() as searcher:
+			writer = ix.writer()
+			writer.delete_by_term(self.id, document[self.id])
+			writer.add_document(**document)
+			writer.commit(optimize=True)
+
+
+	def build_index(self):
+		"""Build index for all parsed documents"""
+		frappe.create_folder(self.index_path)
+		ix = create_in(self.index_path, self.schema)
+		writer = ix.writer()
+
+		for document in self.documents:
+			if document:
+				writer.add_document(**document)
+
+		writer.commit(optimize=True)
+
+	def search(self, text, scope=None, limit=20):
+		"""Search from the current index
+
+		Args:
+			text (str): String to search for
+			scope (str, optional): Scope to limit the search. Defaults to None.
+			limit (int, optional): Limit number of search results. Defaults to 20.
+
+		Returns:
+			[List(_dict)]: Search results
+		"""
+		ix = open_dir(self.index_path)
+
+		results = None
+		out = []
+
+		with ix.searcher() as searcher:
+			parser = MultifieldParser(["title", "content"], ix.schema)
+			parser.remove_plugin_class(FieldsPlugin)
+			parser.remove_plugin_class(WildcardPlugin)
+			query = parser.parse(text)
+
+			filter_scoped = None
+			if scope:
+				filter_scoped = Prefix(self.id, scope)
+			results = searcher.search(query, limit=limit, filter=filter_scoped)
+
+			for r in results:
+				out.append(self.parse_result(r))
+
+		return out
+
+def get_index_path(index_name):
+	return frappe.get_site_path("indexes", index_name)
\ No newline at end of file
diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
new file mode 100644
index 0000000000..bb8d0d8fa4
--- /dev/null
+++ b/frappe/search/website_search.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from bs4 import BeautifulSoup
+from whoosh.fields import TEXT, ID, Schema
+from frappe.search.full_text_search import FullTextSearch
+from frappe.website.render import render_page
+from frappe.utils import set_request
+import os
+
+INDEX_NAME = "web_routes"
+
+class WebsiteSearch(FullTextSearch):
+	""" Wrapper for WebsiteSearch """
+
+	def get_schema(self):
+		return Schema(
+			title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)
+		)
+
+	def get_id(self):
+		return "path"
+
+	def get_items_to_index(self):
+		"""Get all routes to be indexed, this includes the static pages
+		in www/ and routes from published documents
+
+		Returns:
+			self (object): FullTextSearch Instance
+		"""
+		routes = get_static_pages_from_all_apps()
+		routes += get_doctype_routes_with_web_view()
+		print(routes)
+		documents = [self.get_document_to_index(route) for route in routes]
+		return documents
+
+	def get_document_to_index(self, route):
+		"""Render a page and parse it using BeautifulSoup
+
+		Args:
+			path (str): route of the page to be parsed
+
+		Returns:
+			document (_dict): A dictionary with title, path and content
+		"""
+		frappe.set_user("Guest")
+		frappe.local.no_cache = True
+
+		try:
+			set_request(method="GET", path=route)
+			content = render_page(route)
+			soup = BeautifulSoup(content, "html.parser")
+			page_content = soup.find(class_="page_content")
+			text_content = page_content.text if page_content else ""
+			title = soup.title.text.strip() if soup.title else route
+
+			frappe.set_user("Administrator")
+
+			return frappe._dict(title=title, content=text_content, path=route)
+		except Exception:
+			pass
+
+	def parse_result(self, result):
+		title_highlights = result.highlights("title")
+		content_highlights = result.highlights("content")
+
+		return frappe._dict(
+			title=result["title"],
+			path=result["path"],
+			title_highlights=title_highlights,
+			content_highlights=content_highlights,
+		)
+
+
+def get_doctype_routes_with_web_view():
+	all_routes = []
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	fields = ["name", "is_published_field"]
+	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
+
+	for doctype in doctype_with_web_views:
+		if doctype.is_published_field:
+			routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route")
+			all_routes += [route.route for route in routes]
+
+	return all_routes
+
+def get_static_pages_from_all_apps():
+	apps = frappe.get_installed_apps()
+
+	routes_to_index = []
+	for app in apps:
+		base = frappe.get_app_path(app, 'www')
+		path_to_index = frappe.get_app_path(app, 'www')
+
+		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
+			for f in filenames:
+				if f.endswith(('.md', '.html')):
+					filepath = os.path.join(dirpath, f)
+
+					route = os.path.relpath(filepath, base)
+					route = route.split('.')[0]
+
+					if route.endswith('index'):
+						route = route.rsplit('index', 1)[0]
+
+					routes_to_index.append(route)
+
+	return routes_to_index
+
+def update_index_for_path(INDEX_NAME, path):
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.update_index_by_name(path)
+
+def remove_document_from_index(INDEX_NAME, path):
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.remove_document_from_index(path)
+
+def build_index_for_all_routes():
+	ws = WebsiteSearch(INDEX_NAME)
+	return ws.build()
\ No newline at end of file

From b2100279d6adc3336532ab35d509f860f24d228e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:27:56 +0530
Subject: [PATCH 167/361] feat: update code for new search

---
 frappe/migrate.py                   | 4 ++--
 frappe/website/js/website.js        | 4 ++--
 frappe/website/website_generator.py | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 26264bad6c..51840e2506 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -19,7 +19,7 @@ from frappe.website import render
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
-from frappe.modules import full_text_search
+from frappe.search.website_search import build_index_for_all_routes
 
 
 def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
@@ -91,7 +91,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# add static pages to global search
 		if not skip_search_index:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes("web_routes")
+			full_text_search.build_index_for_all_routes()
 
 		frappe.db.commit()
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 1c739cc6ca..6c8b29059e 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -420,11 +420,11 @@ frappe.setup_search = function (target, search_scope) {
 			clear_dropdown();
 			return;
 		}
+		console.log(search_scope);
 
 		frappe.call({
-			method: 'frappe.modules.full_text_search.web_search',
+			method: 'frappe.search.web_search',
 			args: {
-				index_name: 'web_routes',
 				scope: search_scope || null,
 				query: $input.val(),
 				limit: 5
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index f8289f53f8..7b5d46d47d 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -7,7 +7,7 @@ from frappe.model.document import Document
 from frappe.website.utils import cleanup_page_name
 from frappe.website.render import clear_cache
 from frappe.modules import get_module_name
-from frappe.modules.full_text_search import update_index_for_path, remove_document_from_index
+from frappe.search.website_search import update_index_for_path, remove_document_from_index
 
 class WebsiteGenerator(Document):
 	website = frappe._dict()
@@ -163,10 +163,9 @@ class WebsiteGenerator(Document):
 		"""
 		if not self.allow_website_search_indexing():
 			return
-		
+
 		if self.is_website_published():
 			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
 		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)
-			
\ No newline at end of file

From 24a984a0eab8d6e17b7191c36fbce5a011b59f7d Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:28:05 +0530
Subject: [PATCH 168/361] feat: add test for FTS

---
 frappe/search/test_full_text_search.py | 129 +++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 frappe/search/test_full_text_search.py

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
new file mode 100644
index 0000000000..bb0d2cef02
--- /dev/null
+++ b/frappe/search/test_full_text_search.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+import frappe
+import unittest
+from frappe.search import FullTextSearch
+
+class TestFullTextSearch(unittest.TestCase):
+
+	def setUp(self):
+		index = get_index()
+		index.build()
+		self.index = index
+
+	def test_search_term(self):
+		# Search Wikipedia
+		res = self.index.search("multilingual online encyclopedia")
+		self.assertEqual(res[0], 'wikipedia')
+
+		res = self.index.search("Linux is a family")
+		self.assertEqual(res[0], 'linux')
+
+		res = self.index.search("Enterprise Resource Planning")
+		self.assertEqual(res[0], 'erpnext')
+
+	def test_search_limit(self):
+		res = self.index.search("CommonSearchTerm")
+		self.assertEqual(len(res), 5)
+
+		res = self.index.search("CommonSearchTerm", limit=3)
+		self.assertEqual(len(res), 3)
+
+		res = self.index.search("CommonSearchTerm", limit=20)
+		self.assertEqual(len(res), 5)
+
+	def test_search_scope(self):
+		# Search outside scope
+		res = self.index.search("multilingual online encyclopedia", scope=["os"])
+		self.assertEqual(len(res), 0)
+
+		# Search inside scope
+		res = self.index.search("CommonSearchTerm", scope=["os"])
+		self.assertEqual(len(res), 2)
+		self.assertTrue('os/linux' in res)
+		self.assertTrue('os/gnu' in res)
+
+	def test_remove_document_from_index(self):
+		self.remove_document_from_index("os/gnu")
+		res = self.index.search("GNU")
+		self.assertEqual(len(res), 0)
+
+	def test_update_index(self):
+		# Update existing index
+		self.update_index({
+			'name': "sw/erpnext",
+			'content': """AwesomeERPNext"""
+		})
+
+		res = self.index.search("CommonSearchTerm")
+		self.assertTrue('sw/erpnext' not in res)
+
+		res = self.index.search("AwesomeERPNext")
+		self.assertEqual(res[0], "sw/erpnext")
+
+		# Update new doc
+		self.update_index({
+			'name': "sw/frappebooks",
+			'content': """DesktopAccounting"""
+		})
+
+		res = self.index.search("DesktopAccounting")
+		self.assertEqual(res[0], "sw/frappebooks")
+
+
+
+class TestWrapper(FullTextSearch):
+	def get_items_to_index(self):
+		return get_documents()
+
+	def get_document_to_index(self, name):
+		documents = get_documents()
+		for doc in documents:
+			if doc["name"] == name:
+				return doc
+
+	def parse_result(self, result):
+		return result["name"]
+
+
+def get_index():
+	return TestWrapper("test_frappe_index")
+
+def get_documents():
+	docs = []
+	docs.append({
+		'name': "site/wikipedia",
+		'content': """Wikipedia is a multilingual online encyclopedia created and maintained
+			as an open collaboration project by a community of volunteer editors using a wiki-based editing system.
+			It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "os/linux",
+		'content': """Linux is a family of open source Unix-like operating systems based on the
+			Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds.
+			Linux is typically packaged in a Linux distribution. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "os/gnu",
+		'content': """GNU is an operating system and an extensive collection of computer software.
+			GNU is composed wholly of free software, most of which is licensed under the GNU Project's own
+			General Public License. GNU is a recursive acronym for "GNU's Not Unix! ",
+			chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "sw/erpnext",
+		'content': """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by
+			Frappe Technologies Pvt. Ltd. and is built on MariaDB database system using a Python based server-side framework.
+			ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm"""
+	})
+
+	docs.append({
+		'name': "sw/frappe",
+		'content': """Frappe Framework is a full-stack web framework, that includes everything you need to build and
+			deploy business applications with Rich Admin Interface. CommonSearchTerm"""
+	})
+
+	return docs
\ No newline at end of file

From 9b06881702aa0e21b0f5843680011c7be26064e4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:30:38 +0530
Subject: [PATCH 169/361] chore: remove log/print statements

---
 frappe/search/website_search.py | 2 +-
 frappe/website/js/website.js    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index bb8d0d8fa4..3b9b5d266b 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -32,7 +32,7 @@ class WebsiteSearch(FullTextSearch):
 		"""
 		routes = get_static_pages_from_all_apps()
 		routes += get_doctype_routes_with_web_view()
-		print(routes)
+
 		documents = [self.get_document_to_index(route) for route in routes]
 		return documents
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 6c8b29059e..aff752476c 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -420,7 +420,6 @@ frappe.setup_search = function (target, search_scope) {
 			clear_dropdown();
 			return;
 		}
-		console.log(search_scope);
 
 		frappe.call({
 			method: 'frappe.search.web_search',

From bc8e23ca151fa6a8b9cde62ae653a44fc106fc0c Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 14:30:51 +0530
Subject: [PATCH 170/361] fix: update shortcut for search

---
 frappe/templates/doc.html    | 2 +-
 frappe/website/js/website.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html
index f45cf3d29e..3a566a1227 100644
--- a/frappe/templates/doc.html
+++ b/frappe/templates/doc.html
@@ -34,7 +34,7 @@
 									<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
 								</svg>
 							</div>
-							<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+							<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
 							<div class="overflow-hidden shadow dropdown-menu w-100">
 							</div>
 						</div>
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index aff752476c..2fc052a73b 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -396,7 +396,7 @@ frappe.setup_search = function (target, search_scope) {
 					<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
 				</svg>
 			</div>
-			<input type="search" class="form-control" placeholder="Search the docs (Press ? to focus)" />
+			<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
 			<div class="overflow-hidden shadow dropdown-menu w-100">
 			</div>
 		</div>`);

From 7b6463c40eed684a182a300e8ae77fe367c0eb51 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 15:42:46 +0530
Subject: [PATCH 171/361] feat: add keyboard navigation to search dropdown

---
 frappe/public/scss/search.scss |  4 +++
 frappe/website/js/website.js   | 61 ++++++++++++++++++++++++++++++----
 2 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/frappe/public/scss/search.scss b/frappe/public/scss/search.scss
index 929b73032d..aee2457f31 100644
--- a/frappe/public/scss/search.scss
+++ b/frappe/public/scss/search.scss
@@ -24,6 +24,10 @@
 	.dropdown-menu {
 		.dropdown-item {
 			padding: 1rem 0.75rem;
+
+			&:focus {
+				background-color: $gray-100;
+			}
 		}
 
 		.match {
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 2fc052a73b..dfe397db4f 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -385,7 +385,7 @@ frappe.setup_search = function (target, search_scope) {
 		target = $(target);
 	}
 
-	let $search_input = $(`<div class="dropdown">
+	let $search_input = $(`<div class="dropdown" id="dropdownMenuSearch">
 			<div class="search-icon">
 				<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24"
 					fill="none"
@@ -397,16 +397,18 @@ frappe.setup_search = function (target, search_scope) {
 				</svg>
 			</div>
 			<input type="search" class="form-control" placeholder="Search the docs (Press / to focus)" />
-			<div class="overflow-hidden shadow dropdown-menu w-100">
+			<div class="overflow-hidden shadow dropdown-menu w-100" aria-labelledby="dropdownMenuSearch">
 			</div>
 		</div>`);
 
 	target.empty();
 	$search_input.appendTo(target);
 
-	let $dropdown = $search_input.find('.dropdown');
+	// let $dropdown = $search_input.find('.dropdown');
 	let $dropdown_menu = $search_input.find('.dropdown-menu');
 	let $input = $search_input.find('input');
+	let dropdownItems;
+	let offsetIndex = 0;
 
 	$(document).on('keypress', e => {
 		if (e.key === '/') {
@@ -443,25 +445,70 @@ frappe.setup_search = function (target, search_scope) {
 			}
 			$dropdown_menu.html(dropdown_html);
 			$dropdown_menu.addClass('show');
+			dropdownItems = $dropdown_menu.find(".dropdown-item");
 		});
 	}, 500));
 
 	$input.on('focus', () => {
 		if (!$input.val()) {
 			clear_dropdown();
+		} else {
+			$input.trigger('input');
 		}
 	});
 
-	$input.on('blur', () => {
-		setTimeout(() => {
-			clear_dropdown();
-		}, 300);
+	$input.keydown(function(e){
+		// up: 38, down: 40
+		if (e.which == 40) {
+			navigate(0);
+		}
 	});
 
+	$dropdown_menu.keydown(function(e){
+		// up: 38, down: 40
+		if (e.which == 38) {
+			navigate(-1);
+		} else if (e.which == 40) {
+			navigate(1);
+		} else if (e.which == 27) {
+			setTimeout(() => {
+				clear_dropdown();
+			}, 300);
+		}
+	});
+
+	// Clear dropdown when clicked
+	$(window).click(function() {
+		clear_dropdown();
+	});
+
+	$search_input.click(function(event){
+		event.stopPropagation();
+	});
+
+	// Navigate the list
+	var navigate = function(diff) {
+		offsetIndex += diff;
+
+		if (offsetIndex >= dropdownItems.length)
+			offsetIndex = 0;
+		if (offsetIndex < 0)
+			offsetIndex = dropdownItems.length - 1;
+		$input.off('blur');
+		dropdownItems.eq(offsetIndex).focus();
+	}
+
 	function clear_dropdown() {
+		offsetIndex = 0
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
+		dropdownItems = undefined;
 	}
+
+	// Remove focus state on hover
+	$dropdown_menu.mouseover(function(e){
+		dropdownItems.blur();
+	});
 };
 
 

From 05ac0591dfca5226bb68aa46017321b3176d6b23 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:00:32 +0530
Subject: [PATCH 172/361] refactor: use glob to crawl static files

Co-authored-by: Chinmay D. Pai <chinmaydpai@gmail.com>
---
 frappe/search/website_search.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index 3b9b5d266b..8412afdf35 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -88,26 +88,20 @@ def get_doctype_routes_with_web_view():
 	return all_routes
 
 def get_static_pages_from_all_apps():
+	from glob import glob
 	apps = frappe.get_installed_apps()
 
 	routes_to_index = []
 	for app in apps:
-		base = frappe.get_app_path(app, 'www')
 		path_to_index = frappe.get_app_path(app, 'www')
 
-		for dirpath, _, filenames in os.walk(path_to_index, topdown=True):
-			for f in filenames:
-				if f.endswith(('.md', '.html')):
-					filepath = os.path.join(dirpath, f)
-
-					route = os.path.relpath(filepath, base)
-					route = route.split('.')[0]
-
-					if route.endswith('index'):
-						route = route.rsplit('index', 1)[0]
-
-					routes_to_index.append(route)
-
+		files_to_index = glob(path_to_index + '/**/*.html', recursive=True)
+		files_to_index.extend(glob(path_to_index + '/**/*.md', recursive=True))
+		for file in files_to_index:
+			route = os.path.relpath(file, path_to_index).split('.')[0]
+			if route.endswith('index'):
+				route = route.rsplit('index', 1)[0]
+			routes_to_index.append(route)
 	return routes_to_index
 
 def update_index_for_path(INDEX_NAME, path):

From 9e3d3de432987bcd766e61637823375d85425141 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:17:40 +0530
Subject: [PATCH 173/361] feat: add site indexing command

---
 frappe/commands/site.py           | 19 ++++++++++++++++++-
 frappe/search/full_text_search.py |  2 --
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index d64a4609e3..88e90af4dc 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -661,6 +661,22 @@ def start_ngrok(context):
 		frappe.destroy()
 		ngrok.kill()
 
+@click.command('build-search-index')
+@pass_context
+def build_search_index(context):
+	from frappe.search.website_search import build_index_for_all_routes
+	site = get_site(context)
+	if not site:
+		raise SiteNotSpecifiedError
+
+	print('Building search index for {}'.format(site))
+	frappe.init(site=site)
+	frappe.connect()
+	try:
+		build_index_for_all_routes()
+	finally:
+		frappe.destroy()
+
 commands = [
 	add_system_manager,
 	backup,
@@ -686,5 +702,6 @@ commands = [
 	start_recording,
 	stop_recording,
 	add_to_hosts,
-	start_ngrok
+	start_ngrok,
+	build_search_index
 ]
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 13fcf22e9b..a8125ee59e 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -34,8 +34,6 @@ class FullTextSearch:
 
 	def build(self):
 		"""	Build search index for all documents """
-
-		print("Building search index for all web routes...")
 		self.documents = self.get_items_to_index()
 		self.build_index()
 

From 7521c3d14f42c4646b6211dd2fb19dd5448d6dcb Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:25:59 +0530
Subject: [PATCH 174/361] feat: add allow web indexing check to doctype

---
 frappe/core/doctype/doctype/doctype.json | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json
index 379ea227cb..698289140e 100644
--- a/frappe/core/doctype/doctype/doctype.json
+++ b/frappe/core/doctype/doctype/doctype.json
@@ -70,6 +70,7 @@
   "web_view",
   "has_web_view",
   "allow_guest_to_view",
+  "index_web_pages_for_search",
   "route",
   "is_published_field",
   "advanced",
@@ -517,12 +518,18 @@
    "fieldname": "email_settings_sb",
    "fieldtype": "Section Break",
    "label": "Email Settings"
+  },
+  {
+   "default": "1",
+   "fieldname": "index_web_pages_for_search",
+   "fieldtype": "Check",
+   "label": "Index Web Pages for Search"
   }
  ],
  "icon": "fa fa-bolt",
  "idx": 6,
  "links": [],
- "modified": "2020-03-27 14:51:44.581128",
+ "modified": "2020-07-21 16:20:57.028802",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

From 02ddaa5ee7c101fb2ca40d6c0ac5982bcc88a982 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:26:26 +0530
Subject: [PATCH 175/361] feat: mark web search indexing check

---
 .../email/doctype/newsletter/newsletter.json  |   3 +-
 .../doctype/blog_category/blog_category.json  | 251 +++++-------------
 .../website/doctype/blog_post/blog_post.json  |   3 +-
 .../doctype/help_article/help_article.json    |  39 +--
 frappe/website/doctype/web_form/web_form.json |   4 +-
 frappe/website/doctype/web_page/web_page.json |   3 +-
 6 files changed, 82 insertions(+), 221 deletions(-)

diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json
index 01f75be954..1ec64826da 100644
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@@ -132,10 +132,11 @@
  "has_web_view": 1,
  "icon": "fa fa-envelope",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2020-05-12 18:09:40.137138",
+ "modified": "2020-07-21 16:25:17.687476",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",
diff --git a/frappe/website/doctype/blog_category/blog_category.json b/frappe/website/doctype/blog_category/blog_category.json
index 5921cef2d5..975ae73829 100644
--- a/frappe/website/doctype/blog_category/blog_category.json
+++ b/frappe/website/doctype/blog_category/blog_category.json
@@ -1,196 +1,83 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 1, 
- "allow_import": 1, 
- "allow_rename": 0, 
- "autoname": "field:category_name", 
- "beta": 0, 
- "creation": "2013-03-08 09:41:11", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
+ "actions": [],
+ "allow_guest_to_view": 1,
+ "allow_import": 1,
+ "autoname": "field:category_name",
+ "creation": "2013-03-08 09:41:11",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "engine": "InnoDB",
+ "field_order": [
+  "category_name",
+  "title",
+  "published",
+  "route"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "category_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Category Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "category_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Category Name",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "title", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Title", 
-   "length": 0, 
-   "no_copy": 1, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Title",
+   "no_copy": 1,
+   "reqd": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "published", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Published", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "published",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Published"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "published", 
-   "fieldname": "route", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Route", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
+   "depends_on": "published",
+   "fieldname": "route",
+   "fieldtype": "Data",
+   "label": "Route",
    "unique": 1
   }
- ], 
- "has_web_view": 1, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-tag", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_published_field": "published", 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-03-06 16:29:05.035486", 
- "modified_by": "Administrator", 
- "module": "Website", 
- "name": "Blog Category", 
- "owner": "Administrator", 
+ ],
+ "has_web_view": 1,
+ "icon": "fa fa-tag",
+ "idx": 1,
+ "index_web_pages_for_search": 1,
+ "is_published_field": "published",
+ "links": [],
+ "modified": "2020-07-21 16:25:18.759447",
+ "modified_by": "Administrator",
+ "module": "Website",
+ "name": "Blog Category",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Website Manager", 
-   "set_user_permissions": 1, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Website Manager",
+   "set_user_permissions": 1,
+   "share": 1,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "Blogger", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Blogger"
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/website/doctype/blog_post/blog_post.json b/frappe/website/doctype/blog_post/blog_post.json
index 8821be1555..c3220788b7 100644
--- a/frappe/website/doctype/blog_post/blog_post.json
+++ b/frappe/website/doctype/blog_post/blog_post.json
@@ -189,10 +189,11 @@
  "has_web_view": 1,
  "icon": "fa fa-quote-left",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 5,
- "modified": "2020-06-01 13:37:57.465434",
+ "modified": "2020-07-21 16:25:17.154911",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Post",
diff --git a/frappe/website/doctype/help_article/help_article.json b/frappe/website/doctype/help_article/help_article.json
index ca659692c6..2c69de6ff2 100644
--- a/frappe/website/doctype/help_article/help_article.json
+++ b/frappe/website/doctype/help_article/help_article.json
@@ -4,6 +4,7 @@
  "allow_import": 1,
  "creation": "2014-10-30 14:25:53.780105",
  "doctype": "DocType",
+ "engine": "InnoDB",
  "field_order": [
   "title",
   "category",
@@ -15,11 +16,7 @@
   "content",
   "likes",
   "route",
-  "owner",
-  "feedback",
-  "helpful",
-  "cb_00",
-  "not_helpful"
+  "owner"
  ],
  "fields": [
   {
@@ -27,8 +24,7 @@
    "fieldtype": "Data",
    "in_global_search": 1,
    "label": "Title",
-   "reqd": 1,
-   "search_index": 1
+   "reqd": 1
   },
   {
    "fieldname": "category",
@@ -90,39 +86,14 @@
    "fieldtype": "Link",
    "label": "Owner",
    "options": "User"
-  },
-  {
-   "collapsible": 1,
-   "fieldname": "feedback",
-   "fieldtype": "Section Break",
-   "label": "Feedback"
-  },
-  {
-   "default": "0",
-   "fieldname": "helpful",
-   "fieldtype": "Int",
-   "in_list_view": 1,
-   "label": "Helpful",
-   "read_only": 1
-  },
-  {
-   "fieldname": "cb_00",
-   "fieldtype": "Column Break"
-  },
-  {
-   "default": "0",
-   "fieldname": "not_helpful",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Not Helpful",
-   "read_only": 1
   }
  ],
  "has_web_view": 1,
  "icon": "icon-file-alt",
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-05-08 10:48:19.997789",
+ "modified": "2020-07-21 16:25:18.577325",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Help Article",
diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json
index e6b7632e4d..f4de1e518f 100644
--- a/frappe/website/doctype/web_form/web_form.json
+++ b/frappe/website/doctype/web_form/web_form.json
@@ -352,8 +352,8 @@
  "icon": "icon-edit",
  "is_published_field": "published",
  "links": [],
- "modified": "2019-12-24 14:15:43.497431",
- "modified_by": "faris@erpnext.com",
+ "modified": "2020-07-21 16:25:37.028459",
+ "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Form",
  "owner": "Administrator",
diff --git a/frappe/website/doctype/web_page/web_page.json b/frappe/website/doctype/web_page/web_page.json
index 4766c24045..8f03237363 100644
--- a/frappe/website/doctype/web_page/web_page.json
+++ b/frappe/website/doctype/web_page/web_page.json
@@ -288,10 +288,11 @@
  "has_web_view": 1,
  "icon": "fa fa-file-alt",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
  "max_attachments": 20,
- "modified": "2020-04-25 20:40:39.253548",
+ "modified": "2020-07-21 16:25:17.899069",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Page",

From 59fe1873139ffae182eab797f95736e1849214c1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:28:43 +0530
Subject: [PATCH 176/361] feat: update search and website generator to check
 index_web_pages_for_search

---
 frappe/search/website_search.py     | 2 +-
 frappe/website/website_generator.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index 8412afdf35..f8dd2a20da 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -76,7 +76,7 @@ class WebsiteSearch(FullTextSearch):
 
 def get_doctype_routes_with_web_view():
 	all_routes = []
-	filters = { "has_web_view": 1, "allow_guest_to_view": 1 }
+	filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1}
 	fields = ["name", "is_published_field"]
 	doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields)
 
diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 7b5d46d47d..94bc4b24be 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -141,9 +141,10 @@ class WebsiteGenerator(Document):
 			frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \
 				url=url, operation_type=operation_type)
 
+	# Change the field value in doctype
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return bool(self.route)
+		return self.index_web_pages_for_search
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""

From 8b7ae16cddb570193216fd2dffbc476724e332e6 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 21 Jul 2020 16:32:54 +0530
Subject: [PATCH 177/361] feat: make search indexing optional in migrate

---
 frappe/commands/site.py | 6 +++---
 frappe/migrate.py       | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 88e90af4dc..54773d6e50 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,9 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
+@click.option('--index-site', is_flag=True, help="Index public web pages")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
+def migrate(context, rebuild_website=False, skip_failing=False, index_site=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -289,7 +289,7 @@ def migrate(context, rebuild_website=False, skip_failing=False, skip_search_inde
 				context.verbose,
 				rebuild_website=rebuild_website,
 				skip_failing=skip_failing,
-				skip_search_index=skip_search_index
+				index_site=index_site
 			)
 		finally:
 			frappe.destroy()
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 51840e2506..63a87a769d 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -22,7 +22,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.search.website_search import build_index_for_all_routes
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, index_site=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -89,9 +89,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 				frappe.get_attr(fn)()
 
 		# add static pages to global search
-		if not skip_search_index:
+		if index_site:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes()
+			build_index_for_all_routes()
 
 		frappe.db.commit()
 

From 20f06f0d24de9300d3cdacfb204d17b18a72adc6 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 24 Jul 2020 18:59:56 +0530
Subject: [PATCH 178/361] chore: revert "feat: make search indexing optional in
 migrate"

This reverts commit b5fe9d4f7be8dae3e3e946b4327bf44b9a35448c.
---
 frappe/commands/site.py | 6 +++---
 frappe/migrate.py       | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 54773d6e50..88e90af4dc 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,9 +274,9 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--index-site', is_flag=True, help="Index public web pages")
+@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
 @pass_context
-def migrate(context, rebuild_website=False, skip_failing=False, index_site=False):
+def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
 	from frappe.migrate import migrate
 
@@ -289,7 +289,7 @@ def migrate(context, rebuild_website=False, skip_failing=False, index_site=False
 				context.verbose,
 				rebuild_website=rebuild_website,
 				skip_failing=skip_failing,
-				index_site=index_site
+				skip_search_index=skip_search_index
 			)
 		finally:
 			frappe.destroy()
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 63a87a769d..51840e2506 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -22,7 +22,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.search.website_search import build_index_for_all_routes
 
 
-def migrate(verbose=True, rebuild_website=False, skip_failing=False, index_site=False):
+def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
 	- run patches
@@ -89,9 +89,9 @@ Otherwise, check the server logs and ensure that all the required services are r
 				frappe.get_attr(fn)()
 
 		# add static pages to global search
-		if index_site:
+		if not skip_search_index:
 			# Run this last as it updates the current session
-			build_index_for_all_routes()
+			full_text_search.build_index_for_all_routes()
 
 		frappe.db.commit()
 

From 156ca5baf13097a1906638f7945a234022a56e63 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 24 Jul 2020 19:03:17 +0530
Subject: [PATCH 179/361] fix: update help text

---
 frappe/commands/site.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index 88e90af4dc..b72d98c433 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -274,7 +274,7 @@ def disable_user(context, email):
 @click.command('migrate')
 @click.option('--rebuild-website', help="Rebuild webpages after migration")
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
-@click.option('--skip-search-index', is_flag=True, help="Skip patches that fail to run")
+@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents")
 @pass_context
 def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"

From 6f0c4a4bf18abec7f3d9469a7e85df257e3abbf4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:01:03 +0530
Subject: [PATCH 180/361] fix: import for build_index_for_all_routes

---
 frappe/migrate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 51840e2506..3af06e8c91 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -91,7 +91,8 @@ Otherwise, check the server logs and ensure that all the required services are r
 		# add static pages to global search
 		if not skip_search_index:
 			# Run this last as it updates the current session
-			full_text_search.build_index_for_all_routes()
+			print('Building search index for {}'.format(frappe.local.site))
+			build_index_for_all_routes()
 
 		frappe.db.commit()
 

From ef0182cf148170dd5d1dd71367a5637a33e57cba Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:03:45 +0530
Subject: [PATCH 181/361] fix: remove generate theme fn from hooks

* ref - a6eae5665
---
 frappe/hooks.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/frappe/hooks.py b/frappe/hooks.py
index 0850cd6f91..894e72a121 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -272,9 +272,6 @@ setup_wizard_exception = [
 ]
 
 before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute']
-after_migrate = [
-	'frappe.website.doctype.website_theme.website_theme.generate_theme_files_if_not_exist'
-]
 
 otp_methods = ['OTP App','Email','SMS']
 user_privacy_documents = [

From a18b81e39db12a3f251719d6c03f949dc3cfce0d Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 14:07:05 +0530
Subject: [PATCH 182/361] chore: linting fixes

---
 frappe/search/__init__.py              |  2 +-
 frappe/search/full_text_search.py      |  5 ++---
 frappe/search/test_full_text_search.py |  3 +--
 frappe/website/js/website.js           | 12 ++++++------
 4 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py
index 462418ca41..0436775417 100644
--- a/frappe/search/__init__.py
+++ b/frappe/search/__init__.py
@@ -4,7 +4,7 @@
 import frappe
 from frappe.utils import cint
 from frappe.search.website_search import WebsiteSearch
-from frappe.search.website_search import FullTextSearch
+from frappe.search.full_text_search import FullTextSearch
 
 @frappe.whitelist(allow_guest=True)
 def web_search(query, scope=None, limit=20):
diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index a8125ee59e..e073a2cfc1 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -8,7 +8,6 @@ from whoosh.index import create_in, open_dir
 from whoosh.fields import TEXT, ID, Schema
 from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
 from whoosh.query import Prefix
-import os
 
 class FullTextSearch:
 	""" Frappe Wrapper for Whoosh """
@@ -57,7 +56,7 @@ class FullTextSearch:
 			doc_name (str): name of the document to be removed
 		"""
 		ix = open_dir(self.index_path)
-		with ix.searcher() as searcher:
+		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, doc_name)
 			writer.commit(optimize=True)
@@ -71,7 +70,7 @@ class FullTextSearch:
 		"""
 		ix = open_dir(self.index_path)
 
-		with ix.searcher() as searcher:
+		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, document[self.id])
 			writer.add_document(**document)
diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index bb0d2cef02..feaa8ca84a 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -1,8 +1,7 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-import frappe
 import unittest
-from frappe.search import FullTextSearch
+from frappe.search.full_text_search import FullTextSearch
 
 class TestFullTextSearch(unittest.TestCase):
 
diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index dfe397db4f..21e1bb93f4 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -457,14 +457,14 @@ frappe.setup_search = function (target, search_scope) {
 		}
 	});
 
-	$input.keydown(function(e){
+	$input.keydown(function(e) {
 		// up: 38, down: 40
 		if (e.which == 40) {
 			navigate(0);
 		}
 	});
 
-	$dropdown_menu.keydown(function(e){
+	$dropdown_menu.keydown(function(e) {
 		// up: 38, down: 40
 		if (e.which == 38) {
 			navigate(-1);
@@ -482,7 +482,7 @@ frappe.setup_search = function (target, search_scope) {
 		clear_dropdown();
 	});
 
-	$search_input.click(function(event){
+	$search_input.click(function(event) {
 		event.stopPropagation();
 	});
 
@@ -499,14 +499,14 @@ frappe.setup_search = function (target, search_scope) {
 	}
 
 	function clear_dropdown() {
-		offsetIndex = 0
+		offsetIndex = 0;
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 		dropdownItems = undefined;
-	}
+	};
 
 	// Remove focus state on hover
-	$dropdown_menu.mouseover(function(e){
+	$dropdown_menu.mouseover(function() {
 		dropdownItems.blur();
 	});
 };

From 9c35b7fc884e2ee3246a38dbc027f17daf486417 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:36:16 +0530
Subject: [PATCH 183/361] chore: remove unused variables

---
 frappe/search/website_search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index f8dd2a20da..ca6f0fe56b 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -104,11 +104,11 @@ def get_static_pages_from_all_apps():
 			routes_to_index.append(route)
 	return routes_to_index
 
-def update_index_for_path(INDEX_NAME, path):
+def update_index_for_path(path):
 	ws = WebsiteSearch(INDEX_NAME)
 	return ws.update_index_by_name(path)
 
-def remove_document_from_index(INDEX_NAME, path):
+def remove_document_from_index(path):
 	ws = WebsiteSearch(INDEX_NAME)
 	return ws.remove_document_from_index(path)
 

From 68e4509792182f7c39de73d154ad0657384eda54 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:36:27 +0530
Subject: [PATCH 184/361] chore: update comment

---
 frappe/migrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/migrate.py b/frappe/migrate.py
index 3af06e8c91..6d64799fdd 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -88,7 +88,7 @@ Otherwise, check the server logs and ensure that all the required services are r
 			for fn in frappe.get_hooks('after_migrate', app_name=app):
 				frappe.get_attr(fn)()
 
-		# add static pages to global search
+		# build web_routes index
 		if not skip_search_index:
 			# Run this last as it updates the current session
 			print('Building search index for {}'.format(frappe.local.site))

From 1b525f0f869428ae0ba528d0b07ca0bfdb9f6b94 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 27 Jul 2020 18:39:00 +0530
Subject: [PATCH 185/361] chore: linting fixes

---
 frappe/website/js/website.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js
index 21e1bb93f4..113e04c5b7 100644
--- a/frappe/website/js/website.js
+++ b/frappe/website/js/website.js
@@ -496,14 +496,14 @@ frappe.setup_search = function (target, search_scope) {
 			offsetIndex = dropdownItems.length - 1;
 		$input.off('blur');
 		dropdownItems.eq(offsetIndex).focus();
-	}
+	};
 
 	function clear_dropdown() {
 		offsetIndex = 0;
 		$dropdown_menu.html('');
 		$dropdown_menu.removeClass('show');
 		dropdownItems = undefined;
-	};
+	}
 
 	// Remove focus state on hover
 	$dropdown_menu.mouseover(function() {

From 6e8772b42dc0bd579ffdfe0fd828abe704adf788 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 29 Jul 2020 21:17:25 +0530
Subject: [PATCH 186/361] feat: don't index blog category

---
 frappe/website/doctype/blog_category/blog_category.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/website/doctype/blog_category/blog_category.json b/frappe/website/doctype/blog_category/blog_category.json
index 975ae73829..b2180047cd 100644
--- a/frappe/website/doctype/blog_category/blog_category.json
+++ b/frappe/website/doctype/blog_category/blog_category.json
@@ -48,10 +48,9 @@
  "has_web_view": 1,
  "icon": "fa fa-tag",
  "idx": 1,
- "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-07-21 16:25:18.759447",
+ "modified": "2020-07-29 21:14:47.210446",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Category",

From 560e9dc8dda1efec2fae5dbc577728d27bd9c746 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 29 Jul 2020 21:40:24 +0530
Subject: [PATCH 187/361] fix: fetch index_web_pages_for_search from meta

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 94bc4b24be..52ae61cd75 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -144,7 +144,7 @@ class WebsiteGenerator(Document):
 	# Change the field value in doctype
 	# Override this method to disable indexing
 	def allow_website_search_indexing(self):
-		return self.index_web_pages_for_search
+		return self.meta.index_web_pages_for_search
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""

From 6da04fcb97a6f6b45b23754f0e6327139583534e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 30 Jul 2020 21:31:11 +0530
Subject: [PATCH 188/361] fix: arguments for update_index_for_path

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 52ae61cd75..d02e88c5ba 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -166,7 +166,7 @@ class WebsiteGenerator(Document):
 			return
 
 		if self.is_website_published():
-			frappe.enqueue(update_index_for_path, index_name="web_routes", path=self.route)
+			frappe.enqueue(update_index_for_path, path=self.route)
 		elif self.route:
 			# If the website is not published
 			remove_document_from_index("web_routes", self.route)

From 788711bdad66d384a454ff8ee86aa68bab24b8bd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 31 Jul 2020 13:35:36 +0530
Subject: [PATCH 189/361] fix: arguments for remove_document_from_index

---
 frappe/website/website_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index d02e88c5ba..427a264725 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -96,7 +96,7 @@ class WebsiteGenerator(Document):
 		self.send_indexing_request('URL_DELETED')
 		# On deleting the doc, remove the page from the web_routes index
 		if self.allow_website_search_indexing():
-			remove_document_from_index("web_routes", self.route)
+			remove_document_from_index(self.route)
 
 	def is_website_published(self):
 		"""Return true if published in website"""

From eab9a27b61e565c3290880d3daf54ac7afb9e4f2 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 6 Aug 2020 19:14:44 +0530
Subject: [PATCH 190/361] feat: separate create_index function

---
 frappe/search/full_text_search.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index e073a2cfc1..9fb478140f 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -76,11 +76,13 @@ class FullTextSearch:
 			writer.add_document(**document)
 			writer.commit(optimize=True)
 
+	def create_index(self):
+		frappe.create_folder(self.index_path)
+		return create_in(self.index_path, self.schema)
 
 	def build_index(self):
 		"""Build index for all parsed documents"""
-		frappe.create_folder(self.index_path)
-		ix = create_in(self.index_path, self.schema)
+		ix = create_index()
 		writer = ix.writer()
 
 		for document in self.documents:

From 38909db9559d832f2c4391d19bf4ca7c865d2b76 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 6 Aug 2020 19:14:52 +0530
Subject: [PATCH 191/361] feat: create index if not exists

---
 frappe/search/full_text_search.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 9fb478140f..4dc405a5d2 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -4,7 +4,7 @@
 from __future__ import unicode_literals
 import frappe
 
-from whoosh.index import create_in, open_dir
+from whoosh.index import create_in, open_dir, EmptyIndexError
 from whoosh.fields import TEXT, ID, Schema
 from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin
 from whoosh.query import Prefix
@@ -55,7 +55,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			doc_name (str): name of the document to be removed
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 		with ix.searcher():
 			writer = ix.writer()
 			writer.delete_by_term(self.id, doc_name)
@@ -68,7 +68,7 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			document (_dict): A dictionary with title, path and content
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 
 		with ix.searcher():
 			writer = ix.writer()
@@ -76,13 +76,19 @@ class FullTextSearch:
 			writer.add_document(**document)
 			writer.commit(optimize=True)
 
+	def get_index(self):
+		try:
+			return open_dir(self.index_path)
+		except EmptyIndexError:
+			return self.create_index()
+
 	def create_index(self):
 		frappe.create_folder(self.index_path)
 		return create_in(self.index_path, self.schema)
 
 	def build_index(self):
 		"""Build index for all parsed documents"""
-		ix = create_index()
+		ix = self.create_index()
 		writer = ix.writer()
 
 		for document in self.documents:
@@ -102,7 +108,7 @@ class FullTextSearch:
 		Returns:
 			[List(_dict)]: Search results
 		"""
-		ix = open_dir(self.index_path)
+		ix = self.get_index()
 
 		results = None
 		out = []

From 170d3c221bcba186042c50ce21a52e1b7733c63f Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 6 Aug 2020 20:26:34 +0530
Subject: [PATCH 192/361] feat: Configurable navbar logo and dropdowns

---
 frappe/core/doctype/navbar_item/__init__.py   |  0
 .../core/doctype/navbar_item/navbar_item.js   |  8 ++
 .../core/doctype/navbar_item/navbar_item.json | 87 ++++++++++++++++++
 .../core/doctype/navbar_item/navbar_item.py   | 10 ++
 .../doctype/navbar_item/test_navbar_item.py   | 10 ++
 .../core/doctype/navbar_settings/__init__.py  |  0
 .../navbar_settings/navbar_settings.js        |  8 ++
 .../navbar_settings/navbar_settings.json      | 91 +++++++++++++++++++
 .../navbar_settings/navbar_settings.py        | 65 +++++++++++++
 .../navbar_settings/test_navbar_settings.py   | 10 ++
 .../v13_0/add_standard_navbar_items.py        |  9 ++
 11 files changed, 298 insertions(+)
 create mode 100644 frappe/core/doctype/navbar_item/__init__.py
 create mode 100644 frappe/core/doctype/navbar_item/navbar_item.js
 create mode 100644 frappe/core/doctype/navbar_item/navbar_item.json
 create mode 100644 frappe/core/doctype/navbar_item/navbar_item.py
 create mode 100644 frappe/core/doctype/navbar_item/test_navbar_item.py
 create mode 100644 frappe/core/doctype/navbar_settings/__init__.py
 create mode 100644 frappe/core/doctype/navbar_settings/navbar_settings.js
 create mode 100644 frappe/core/doctype/navbar_settings/navbar_settings.json
 create mode 100644 frappe/core/doctype/navbar_settings/navbar_settings.py
 create mode 100644 frappe/core/doctype/navbar_settings/test_navbar_settings.py
 create mode 100644 frappe/patches/v13_0/add_standard_navbar_items.py

diff --git a/frappe/core/doctype/navbar_item/__init__.py b/frappe/core/doctype/navbar_item/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/core/doctype/navbar_item/navbar_item.js b/frappe/core/doctype/navbar_item/navbar_item.js
new file mode 100644
index 0000000000..bd4274db49
--- /dev/null
+++ b/frappe/core/doctype/navbar_item/navbar_item.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Navbar Item', {
+	// refresh: function(frm) {
+
+	// }
+});
diff --git a/frappe/core/doctype/navbar_item/navbar_item.json b/frappe/core/doctype/navbar_item/navbar_item.json
new file mode 100644
index 0000000000..3bfea52558
--- /dev/null
+++ b/frappe/core/doctype/navbar_item/navbar_item.json
@@ -0,0 +1,87 @@
+{
+ "actions": [],
+ "creation": "2020-08-01 23:38:41.783206",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "item_label",
+  "item_type",
+  "route",
+  "action",
+  "hidden",
+  "is_standard"
+ ],
+ "fields": [
+  {
+   "columns": 2,
+   "fieldname": "item_label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Item Label",
+   "mandatory_depends_on": "eval:doc.item_type == 'Route' || doc.item_type == 'Action'",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "columns": 2,
+   "fieldname": "item_type",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Item Type",
+   "options": "Route\nAction\nSeparator",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "hidden",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Hidden",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "is_standard",
+   "fieldtype": "Check",
+   "label": "Is Standard",
+   "read_only": 1,
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "columns": 4,
+   "depends_on": "eval:doc.item_type == 'Route'",
+   "fieldname": "route",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Route",
+   "mandatory_depends_on": "eval:doc.item_type == 'Route'",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "depends_on": "eval:doc.item_type == 'Action'",
+   "fieldname": "action",
+   "fieldtype": "Data",
+   "label": "Action",
+   "mandatory_depends_on": "eval:doc.item_type == 'Action'",
+   "show_days": 1,
+   "show_seconds": 1
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-08-06 16:32:49.597060",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Navbar Item",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/core/doctype/navbar_item/navbar_item.py b/frappe/core/doctype/navbar_item/navbar_item.py
new file mode 100644
index 0000000000..614aee8eaf
--- /dev/null
+++ b/frappe/core/doctype/navbar_item/navbar_item.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class NavbarItem(Document):
+	pass
diff --git a/frappe/core/doctype/navbar_item/test_navbar_item.py b/frappe/core/doctype/navbar_item/test_navbar_item.py
new file mode 100644
index 0000000000..192e8fe42a
--- /dev/null
+++ b/frappe/core/doctype/navbar_item/test_navbar_item.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestNavbarItem(unittest.TestCase):
+	pass
diff --git a/frappe/core/doctype/navbar_settings/__init__.py b/frappe/core/doctype/navbar_settings/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.js b/frappe/core/doctype/navbar_settings/navbar_settings.js
new file mode 100644
index 0000000000..e2c157fe6a
--- /dev/null
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Navbar Settings', {
+	// refresh: function(frm) {
+
+	// }
+});
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.json b/frappe/core/doctype/navbar_settings/navbar_settings.json
new file mode 100644
index 0000000000..8fc0c83c82
--- /dev/null
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.json
@@ -0,0 +1,91 @@
+{
+ "actions": [],
+ "creation": "2020-08-01 23:41:12.577160",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "logo_section",
+  "app_logo",
+  "column_break_3",
+  "logo_width",
+  "section_break_2",
+  "settings_dropdown",
+  "help_dropdown"
+ ],
+ "fields": [
+  {
+   "fieldname": "app_logo",
+   "fieldtype": "Attach Image",
+   "label": "Application Logo",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "settings_dropdown",
+   "fieldtype": "Table",
+   "label": "Settings Dropdown",
+   "options": "Navbar Item",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "help_dropdown",
+   "fieldtype": "Table",
+   "label": "Help Dropdown",
+   "options": "Navbar Item",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "section_break_2",
+   "fieldtype": "Section Break",
+   "label": "Dropdowns",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "logo_section",
+   "fieldtype": "Section Break",
+   "label": "Application Logo",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break",
+   "show_days": 1,
+   "show_seconds": 1
+  },
+  {
+   "fieldname": "logo_width",
+   "fieldtype": "Int",
+   "label": "Logo Width",
+   "show_days": 1,
+   "show_seconds": 1
+  }
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-08-06 18:11:29.955835",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Navbar Settings",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py
new file mode 100644
index 0000000000..5765fd9f0f
--- /dev/null
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+from frappe import _
+
+class NavbarSettings(Document):
+	def validate(self):
+		self.validate_standard_navbar_items()
+
+	def validate_standard_navbar_items(self):
+		doc_before_save = self.get_doc_before_save()
+
+		before_save_items = [item for item in \
+			doc_before_save.help_dropdown + doc_before_save.settings_dropdown if item.is_standard]
+
+		after_save_items = [item for item in \
+			self.help_dropdown + self.settings_dropdown if item.is_standard]
+
+		if not frappe.flags.in_patch and (len(before_save_items) != len(after_save_items)):
+			frappe.throw(_("Cannot delete standard navbar items"))
+
+@frappe.whitelist()
+def get_app_logo():
+	app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo')
+	if not app_logo:
+		app_logo = frappe.get_hooks('app_logo_url')[-1]
+
+	return app_logo
+
+def get_navbar_items():
+	navbar_settings = frappe.get_single('Navbar Settings')
+
+	navbar_items = frappe._dict()
+	navbar_items.help_dropdown = []
+	navbar_items.settings_dropdown = []
+
+	for item in navbar_settings.settings_dropdown:
+		navbar_items.settings_dropdown.append({
+			'label': item.item_label,
+			'item_type': item.item_type,
+			'route': item.route,
+			'hidden': item.hidden,
+			'action': item.action
+		})
+
+	for item in navbar_settings.help_dropdown:
+		navbar_items.help_dropdown.append({
+			'label': item.item_label,
+			'item_type': item.item_type,
+			'route': item.route,
+			'hidden': item.hidden,
+			'action': item.action
+		})
+
+	navbar_items.logo_width = navbar_settings.logo_width
+
+	return navbar_items
+
+
+
+
diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py
new file mode 100644
index 0000000000..ed423b0f27
--- /dev/null
+++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestNavbarSettings(unittest.TestCase):
+	pass
diff --git a/frappe/patches/v13_0/add_standard_navbar_items.py b/frappe/patches/v13_0/add_standard_navbar_items.py
new file mode 100644
index 0000000000..9982e6e3f5
--- /dev/null
+++ b/frappe/patches/v13_0/add_standard_navbar_items.py
@@ -0,0 +1,9 @@
+from __future__ import unicode_literals
+import frappe
+from frappe.utils.install import add_standard_navbar_items
+
+def execute():
+	# Add standard navbar items for ERPNext in Navbar Settings
+	frappe.reload_doc('core', 'doctype', 'navbar_settings')
+	frappe.reload_doc('core', 'doctype', 'navbar_item')
+	add_standard_navbar_items()
\ No newline at end of file

From 2e3e9bf283cf569594478cdb9fffa838519f830f Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 6 Aug 2020 20:27:12 +0530
Subject: [PATCH 193/361] fix: Add navbar items in boot

---
 frappe/boot.py                                |  2 +
 .../public/js/frappe/ui/toolbar/navbar.html   | 68 +++++++++++--------
 frappe/public/js/frappe/ui/toolbar/toolbar.js | 13 +++-
 3 files changed, 55 insertions(+), 28 deletions(-)

diff --git a/frappe/boot.py b/frappe/boot.py
index b552d7d703..df870adc60 100644
--- a/frappe/boot.py
+++ b/frappe/boot.py
@@ -21,6 +21,7 @@ from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabl
 from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points
 from frappe.model.base_document import get_controller
 from frappe.social.doctype.post.post import frequently_visited_links
+from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_items
 
 def get_bootinfo():
 	"""build and return boot info"""
@@ -59,6 +60,7 @@ def get_bootinfo():
 	load_print(bootinfo, doclist)
 	doclist.extend(get_meta_bundle("Page"))
 	bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1})
+	bootinfo.navbar_items = get_navbar_items()
 
 	# ipinfo
 	if frappe.session.data.get('ipinfo'):
diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html
index bc1c14dc47..b66049183e 100644
--- a/frappe/public/js/frappe/ui/toolbar/navbar.html
+++ b/frappe/public/js/frappe/ui/toolbar/navbar.html
@@ -5,13 +5,12 @@
 				<i class="octicon octicon-three-bars"></i>
 			</a>
 			<a class="navbar-brand navbar-home hidden-xs hidden-sm" href="#">
-				<img class="app-logo" src="{{ frappe.app.logo_url }}">
+				<img class="app-logo" style="width: {{navbar_items.logo_width || 24}}px" src="{{ frappe.app.logo_url }}">
 			</a>
 			<ul class="nav navbar-nav" id="navbar-breadcrumbs">
 			</ul>
 		</div>
 		<div class="navbar-center ellipsis" style="display: none;"></div>
-
 		<ul class="nav navbar-nav navbar-right">
 			<li class="visible-xs">
 				<a class="navbar-search-button" href="#" data-toggle="modal" data-target="#search-modal"><i class="octicon octicon-search"></i></a>
@@ -24,25 +23,25 @@
 					{%= __("Settings") %}</span>
 				<b class="caret hidden-xs hidden-sm"></b></a>
 				<ul class="dropdown-menu" id="toolbar-user" role="menu">
-					<li><a href="#user-profile">
-						{%= __("My Profile") %}</a></li>
-					<li><a href="#Form/User/{%= encodeURIComponent(frappe.session.user) %}">
-						{%= __("My Settings") %}</a></li>
-					<li><a href="#" onclick="return frappe.ui.toolbar.setup_session_defaults();">
-							{%= __("Session Defaults") %}</a></li>
-					<li class="navbar-reload">
-						<a href="" onclick="return frappe.ui.toolbar.clear_cache();">
-						{%= __("Reload") %}</a></li>
-					<li><a href="/index" target="_blank" rel="noopener noreferrer">
-						{%= __("View Website") %}</a></li>
-					<li class="navbar-toggle-full-width">
-						<a href="#" onclick="return false">{%= __("Toggle Full Width") %}</a>
-					</li>
-					<li><a href="#background_jobs">
-						{%= __("Background Jobs") %}</a></li>
-					<li class="divider"></li>
-					<li><a href="#" onclick="return frappe.app.logout();">
-						{%= __("Logout") %}</a></li>
+					{% for item in navbar_items.settings_dropdown %}
+						{% if (!item.hidden) { %}
+							{% if (item.route) { %}
+								<li>
+									<a href="{{ item.route }}">
+										{%= __(item.label) %}
+									</a>
+								</li>
+							{% } else if (item.action) { %}
+								<li>
+									<a onclick="return {{ item.action }}">
+										{%= __(item.label) %}
+									</a>
+								</li>
+							{% } else { %}
+								<li class="divider"></li>
+							{% } %}
+						{% } %}
+					{% endfor %}
 				</ul>
 			</li>
 
@@ -53,13 +52,28 @@
 					<span class="visible-xs visible-sm standard-image help-icon">?</span>
 				</a>
 				<ul class="dropdown-menu" id="toolbar-help" role="menu">
+					<!-- For custom links added via jquery -->
 					<li id="help-links"></li>
-					<li class="divider"></li>
 					<li class="divider documentation-links"></li>
-					<li><a href="#" onclick="return frappe.ui.toolbar.show_about();">
-						{%= __("About") %}</a></li>
-					<li><a href="#" onclick="return frappe.ui.toolbar.show_shortcuts(event);">
-						{%= __("Keyboard Shortcuts") %}</a></li>
+					{% for item in navbar_items.help_dropdown %}
+						{% if (!item.hidden) { %}
+							{% if (item.route) { %}
+								<li>
+									<a href="{{ item.route }}">
+										{%= __(item.label) %}
+									</a>
+								</li>
+							{% } else if (item.action) { %}
+								<li>
+									<a onclick="return {{ item.action }}">
+										{%= __(item.label) %}
+									</a>
+								</li>
+							{% } else { %}
+								<li class="divider"></li>
+							{% } %}
+						{% } %}
+					{% endfor %}
 				</ul>
 			</li>
 
@@ -112,4 +126,4 @@
 			</form>
 		</div>
 	</div>
-</div>
+</div>
\ No newline at end of file
diff --git a/frappe/public/js/frappe/ui/toolbar/toolbar.js b/frappe/public/js/frappe/ui/toolbar/toolbar.js
index e774550a69..5b80d75f4e 100644
--- a/frappe/public/js/frappe/ui/toolbar/toolbar.js
+++ b/frappe/public/js/frappe/ui/toolbar/toolbar.js
@@ -7,7 +7,8 @@ frappe.provide('frappe.search');
 frappe.ui.toolbar.Toolbar = Class.extend({
 	init: function() {
 		$('header').append(frappe.render_template("navbar", {
-			avatar: frappe.avatar(frappe.session.user)
+			avatar: frappe.avatar(frappe.session.user),
+			navbar_items: frappe.boot.navbar_items
 		}));
 		$('.dropdown-toggle').dropdown();
 
@@ -236,6 +237,16 @@ frappe.ui.toolbar.show_about = function() {
 	return false;
 };
 
+frappe.ui.toolbar.route_to_user = function() {
+	frappe.set_route('Form', 'User', frappe.session.user);
+}
+
+frappe.ui.toolbar.view_website = function() {
+	let website_tab = window.open();
+	website_tab.opener = null;
+	website_tab.location = '/index';
+}
+
 frappe.ui.toolbar.setup_session_defaults = function() {
 	let fields = [];
 	frappe.call({

From 01213e0af414ae6db63c1484a50cdff08a6e7623 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 6 Aug 2020 20:27:36 +0530
Subject: [PATCH 194/361] fix: Get app logo from Navbar Settings

---
 frappe/public/js/frappe/desk.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js
index 2e80dbfd85..10aeb66a2a 100644
--- a/frappe/public/js/frappe/desk.js
+++ b/frappe/public/js/frappe/desk.js
@@ -427,9 +427,9 @@ frappe.Application = Class.extend({
 	},
 
 	set_app_logo_url: function() {
-		return frappe.call('frappe.client.get_hooks', { hook: 'app_logo_url' })
+		return frappe.call('frappe.core.doctype.navbar_settings.navbar_settings.get_app_logo')
 			.then(r => {
-				frappe.app.logo_url = (r.message || []).slice(-1)[0];
+				frappe.app.logo_url = r.message;
 				if (window.cordova) {
 					let host = frappe.request.url;
 					host = host.slice(0, host.length - 1);

From cbe4a999c5396172cd6e4abc606afe16ba587d42 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 6 Aug 2020 20:28:29 +0530
Subject: [PATCH 195/361] fix: Add standard navbar items in patch and after
 install

---
 frappe/patches.txt      |  1 +
 frappe/utils/install.py | 81 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/frappe/patches.txt b/frappe/patches.txt
index b207a325cb..e13db34411 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -296,3 +296,4 @@ frappe.patches.v13_0.update_duration_options
 frappe.patches.v13_0.replace_old_data_import # 2020-06-24
 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts
 frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
+frappe.patches.v13_0.add_standard_navbar_items
\ No newline at end of file
diff --git a/frappe/utils/install.py b/frappe/utils/install.py
index bb384fb300..dbb77abd66 100644
--- a/frappe/utils/install.py
+++ b/frappe/utils/install.py
@@ -43,6 +43,8 @@ def after_install():
 	with open(frappe.get_site_path('.test_log'), 'w') as f:
 		f.write('')
 
+	add_standard_navbar_items()
+
 	frappe.db.commit()
 
 def install_basic_docs():
@@ -164,3 +166,82 @@ def add_country_and_currency(name, country):
 			"number_format": country.number_format,
 			"docstatus": 0
 		}).db_insert()
+
+def add_standard_navbar_items():
+	navbar_settings = frappe.get_single("Navbar Settings")
+
+	standard_navbar_items = [
+		{
+			'item_label': 'My Profile',
+			'item_type': 'Route',
+			'route': '#user-profile',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'My Settings',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.route_to_user()',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'Session Defaults',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.setup_session_defaults()',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'Reload',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.clear_cache()',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'View Website',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.view_website()',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'Background Jobs',
+			'item_type': 'Route',
+			'route': '#background_jobs',
+			'is_standard': 1
+		},
+		{
+			'item_type': 'Separator',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'Logout',
+			'item_type': 'Action',
+			'action': 'frappe.app.logout()',
+			'is_standard': 1
+		}
+	]
+
+	standard_help_items = [
+		{
+			'item_type': 'Separator',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'About',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.show_about()',
+			'is_standard': 1
+		},
+		{
+			'item_label': 'Keyboard Shortcuts',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.show_shortcuts(event)',
+			'is_standard': 1
+		}
+	]
+
+	for item in standard_navbar_items:
+		navbar_settings.append('settings_dropdown', item)
+
+	for item in standard_help_items:
+		navbar_settings.append('help_dropdown', item)
+
+	navbar_settings.save()

From 0c92e9d5f783b0f7699763183febf9d4aa607609 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 6 Aug 2020 21:22:27 +0530
Subject: [PATCH 196/361] fix: Validate method for standard items

---
 frappe/core/doctype/navbar_settings/navbar_settings.py | 2 +-
 frappe/public/js/frappe/ui/toolbar/toolbar.js          | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py
index 5765fd9f0f..b95482d6d9 100644
--- a/frappe/core/doctype/navbar_settings/navbar_settings.py
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.py
@@ -20,7 +20,7 @@ class NavbarSettings(Document):
 		after_save_items = [item for item in \
 			self.help_dropdown + self.settings_dropdown if item.is_standard]
 
-		if not frappe.flags.in_patch and (len(before_save_items) != len(after_save_items)):
+		if not frappe.flags.in_patch and (len(before_save_items) > len(after_save_items)):
 			frappe.throw(_("Cannot delete standard navbar items"))
 
 @frappe.whitelist()
diff --git a/frappe/public/js/frappe/ui/toolbar/toolbar.js b/frappe/public/js/frappe/ui/toolbar/toolbar.js
index 5b80d75f4e..a69edd7210 100644
--- a/frappe/public/js/frappe/ui/toolbar/toolbar.js
+++ b/frappe/public/js/frappe/ui/toolbar/toolbar.js
@@ -239,13 +239,13 @@ frappe.ui.toolbar.show_about = function() {
 
 frappe.ui.toolbar.route_to_user = function() {
 	frappe.set_route('Form', 'User', frappe.session.user);
-}
+};
 
 frappe.ui.toolbar.view_website = function() {
 	let website_tab = window.open();
 	website_tab.opener = null;
 	website_tab.location = '/index';
-}
+};
 
 frappe.ui.toolbar.setup_session_defaults = function() {
 	let fields = [];

From b92ce9a245ad9582a833b4d0ce62bf1230e99a68 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <42651287+deepeshgarg007@users.noreply.github.com>
Date: Fri, 7 Aug 2020 09:25:37 +0530
Subject: [PATCH 197/361] feat: Allow sending emails via Server Script (#11162)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/utils/safe_exec.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index 826aed6082..d97e770d88 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -70,6 +70,7 @@ def get_safe_globals():
 			render_template=frappe.render_template,
 			msgprint=frappe.msgprint,
 			throw=frappe.throw,
+			sendmail = frappe.sendmail,
 
 			user=user,
 			get_fullname=frappe.utils.get_fullname,

From 5a52d542115af203020e2ba14dbd6f17f5569895 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 7 Aug 2020 12:47:07 +0530
Subject: [PATCH 198/361] fix: fetch correct backup files

---
 frappe/utils/backups.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 7eb00ceccd..12d584f894 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -102,15 +102,17 @@ class BackupGenerator:
 
 		for this_file in file_list:
 			this_file = cstr(this_file)
+			file_name = os.path.splitext(this_file)[0]
 			this_file_path = os.path.join(get_backup_path(), this_file)
+
 			if not is_file_old(this_file_path, older_than):
-				if "-private-files" in this_file_path:
-					backup_path_private_files = this_file_path
-				elif "-files" in this_file_path:
-					backup_path_files = this_file_path
-				elif "-database" in this_file_path:
+				if file_name.endswith("{}-database.sql".format(self.site_slug)):
 					backup_path_db = this_file_path
-				elif "site_config" in this_file_path:
+				elif file_name.endswith("{}-files".format(self.site_slug)):
+					backup_path_files = this_file_path
+				elif file_name.endswith("{}-private-files".format(self.site_slug)):
+					backup_path_private_files = this_file_path
+				elif file_name.endswith("{}-site_config_backup".format(self.site_slug)):
 					site_config_backup_path = this_file_path
 
 		return (backup_path_db, backup_path_files, backup_path_private_files, site_config_backup_path)

From 62e718a8138e01d7de66df9d054602cecf0cd561 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 7 Aug 2020 13:13:49 +0530
Subject: [PATCH 199/361] fix: Remove duplicate code

Use BackupGenerator instead of custom code for integrations
---
 frappe/integrations/offsite_backup_utils.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index 9de176b2d0..6e64bc5428 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -44,20 +44,14 @@ def get_recipients(doctype, email_field):
 
 
 def get_latest_backup_file(with_files=False):
-
-	def get_latest(file_ext):
-		file_list = glob.glob(os.path.join(get_backups_path(), file_ext))
-		return max(file_list, key=os.path.getctime) if file_list else None
-
-	latest_file = get_latest('*.sql.gz')
-	latest_site_config = get_latest('*.json')
+	from frappe.utils.backups import BackupGenerator
+	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name, frappe.conf.db_password, db_host=frappe.db.host, db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
+	database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
 
 	if with_files:
-		latest_public_file_bak = get_latest('*-files.tar')
-		latest_private_file_bak = get_latest('*-private-files.tar')
-		return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak
+		return database, config, public, private
 
-	return latest_file, latest_site_config
+	return database, config
 
 
 def get_file_size(file_path, unit):

From 5b3fa971b54be7641b369d30a6c68f8acf290a13 Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Fri, 7 Aug 2020 13:14:29 +0530
Subject: [PATCH 200/361] Revert "fix: add a distinct identifier for public
 files backup (#11201)"

This reverts commit ef4a28e10c0de0083c57278b615e587600b64e3b.
---
 frappe/integrations/offsite_backup_utils.py | 2 +-
 frappe/utils/backups.py                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index 0b3c47064e..9de176b2d0 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -53,7 +53,7 @@ def get_latest_backup_file(with_files=False):
 	latest_site_config = get_latest('*.json')
 
 	if with_files:
-		latest_public_file_bak = get_latest('*-public-files.tar')
+		latest_public_file_bak = get_latest('*-files.tar')
 		latest_private_file_bak = get_latest('*-private-files.tar')
 		return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak
 
diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 08b911c9ae..7eb00ceccd 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -82,7 +82,7 @@ class BackupGenerator:
 	def set_backup_file_name(self):
 		#Generate a random name using today's date and a 8 digit random number
 		for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz"
-		for_public_files = self.todays_date + "-" + self.site_slug + "-public-files.tar"
+		for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar"
 		for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar"
 		backup_path = get_backup_path()
 

From ce9c23ba609e5f04cf15aba195a78b81b5a7508d Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Fri, 7 Aug 2020 14:48:59 +0530
Subject: [PATCH 201/361] fix: return handler in _handler (#11215)

---
 frappe/public/js/frappe/form/script_manager.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/script_manager.js b/frappe/public/js/frappe/form/script_manager.js
index 1238bf141c..4456c7d39c 100644
--- a/frappe/public/js/frappe/form/script_manager.js
+++ b/frappe/public/js/frappe/form/script_manager.js
@@ -19,7 +19,7 @@ frappe.ui.form.on = frappe.ui.form.on_change = function(doctype, fieldname, hand
 
 		let _handler = (...args) => {
 			try {
-				handler(...args);
+				return handler(...args);
 			} catch (error) {
 				console.error(handler);
 				throw error;

From 19118ba45f3a16298405bf061bf0d098b18d813d Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 7 Aug 2020 20:55:07 +0530
Subject: [PATCH 202/361] feat: add API to add participants to event

---
 frappe/desk/doctype/event/event.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py
index 45dc5d86c7..54905bed6a 100644
--- a/frappe/desk/doctype/event/event.py
+++ b/frappe/desk/doctype/event/event.py
@@ -82,6 +82,27 @@ class Event(Document):
 		communication.add_link(participant.reference_doctype, participant.reference_docname)
 		communication.save(ignore_permissions=True)
 
+	def add_participant(self, doctype, docname):
+		"""Add a single participant to event participants
+
+		Args:
+			doctype (string): Reference Doctype
+			docname (string): Reference Docname
+		"""
+		self.append("event_participants", {
+			"reference_doctype": doctype,
+			"reference_docname": docname,
+		})
+
+	def add_participants(self, participants):
+		"""Add participant entry
+
+		Args:
+			participants ([Array]): Array of a dict with doctype and docname
+		"""
+		for participant in  participants:
+			self.add_participant(participant["doctype"], participant["docname"])
+
 @frappe.whitelist()
 def delete_communication(event, reference_doctype, reference_docname):
 	deleted_participant = frappe.get_doc(reference_doctype, reference_docname)

From 80733d4124b5ed5dc0a82d7a77b5f6593dd4b67b Mon Sep 17 00:00:00 2001
From: Shridhar Patil <shridharpatil2792@gmail.com>
Date: Fri, 7 Aug 2020 21:52:04 +0530
Subject: [PATCH 203/361] fix: raise incorrect site path error instead of
 sys.exit (#11206)

---
 frappe/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 9e744c2c07..fdf6079220 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -231,8 +231,7 @@ def get_site_config(sites_path=None, site_path=None):
 		if os.path.exists(site_config):
 			config.update(get_file_json(site_config))
 		elif local.site and not local.flags.new_site:
-			print("Site {0} does not exist".format(local.site))
-			sys.exit(1)
+			raise IncorrectSitePath("{0} does not exist".format(local.site))
 
 	return _dict(config)
 

From 8d745a97edc2b1b954bca86039c0ead9411eaa32 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 7 Aug 2020 21:54:09 +0530
Subject: [PATCH 204/361] fix: Get latest available backup obeying older_than

---
 frappe/utils/backups.py | 52 ++++++++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 12d584f894..71906f540f 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -7,7 +7,9 @@ from __future__ import print_function, unicode_literals
 
 import os
 import json
+from calendar import timegm
 from datetime import datetime
+from glob import glob
 
 import frappe
 from frappe import _, conf
@@ -94,28 +96,40 @@ class BackupGenerator:
 			self.backup_path_private_files = os.path.join(backup_path, for_private_files)
 
 	def get_recent_backup(self, older_than):
-		file_list = os.listdir(get_backup_path())
-		backup_path_files = None
-		backup_path_db = None
-		backup_path_private_files = None
-		site_config_backup_path = None
+		backup_path = get_backup_path()
 
-		for this_file in file_list:
-			this_file = cstr(this_file)
-			file_name = os.path.splitext(this_file)[0]
-			this_file_path = os.path.join(get_backup_path(), this_file)
+		file_type_slugs = {
+			"database": "*-{}-database.sql.gz",
+			"public": "*-{}-files.tar",
+			"private": "*-{}-private-files.tar",
+			"config": "*-{}-site_config_backup.json"
+		}
 
-			if not is_file_old(this_file_path, older_than):
-				if file_name.endswith("{}-database.sql".format(self.site_slug)):
-					backup_path_db = this_file_path
-				elif file_name.endswith("{}-files".format(self.site_slug)):
-					backup_path_files = this_file_path
-				elif file_name.endswith("{}-private-files".format(self.site_slug)):
-					backup_path_private_files = this_file_path
-				elif file_name.endswith("{}-site_config_backup".format(self.site_slug)):
-					site_config_backup_path = this_file_path
+		def backup_time(file_path):
+			file_name = file_path.split(os.sep)[-1]
+			file_timestamp = file_name.split("-")[0]
+			return timegm(datetime.strptime(file_timestamp, "%Y%m%d_%H%M%S").utctimetuple())
 
-		return (backup_path_db, backup_path_files, backup_path_private_files, site_config_backup_path)
+		def get_latest(file_pattern):
+			file_list = glob(os.path.join(backup_path, file_pattern))
+			if file_list:
+				return max(file_list, key=backup_time)
+
+		def old_enough(file_path):
+			if file_path:
+				if not os.path.isfile(file_path) or is_file_old(file_path, older_than):
+					return None
+				return file_path
+
+		latest_backups = {
+			file_type: get_latest(pattern.format(self.site_slug)) for file_type, pattern in file_type_slugs.items()
+		}
+
+		recent_backups = {
+			file_type: old_enough(file_name) for file_type, file_name in latest_backups.items()
+		}
+
+		return recent_backups.get("database"), recent_backups.get("public"), recent_backups.get("private"), recent_backups.get("config")
 
 	def zip_files(self):
 		for folder in ("public", "private"):

From 4a331016319471e07a2fabac7312bcb8347176b2 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Sat, 8 Aug 2020 19:47:39 +0530
Subject: [PATCH 205/361] fix: Check if frappe.local has "request" to avoid
 attribute error

---
 frappe/auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/auth.py b/frappe/auth.py
index 64fea36748..998e97fe24 100644
--- a/frappe/auth.py
+++ b/frappe/auth.py
@@ -338,7 +338,7 @@ class CookieManager:
 			self.set_cookie("country", frappe.session.session_country)
 
 	def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"):
-		if not secure:
+		if not secure and hasattr(frappe.local, 'request'):
 			secure = frappe.local.request.scheme == "https"
 		self.cookies[key] = {
 			"value": value,

From 52d6925c49c7acee15189fcde58b92a55a17f97c Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Sat, 8 Aug 2020 19:47:54 +0530
Subject: [PATCH 206/361] test: Fallback to site URL if host_name is none

---
 .../doctype/data_migration_run/test_data_migration_run.py     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
index ba4a255b97..c6c3ea138c 100644
--- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
+++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
@@ -108,7 +108,7 @@ def create_plan():
 		'connector_name': 'Local Connector',
 		'connector_type': 'Frappe',
 		# connect to same host.
-		'hostname': frappe.conf.host_name,
+		'hostname': frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site),
 		'username': 'Administrator',
-		'password': 'admin'
+		'password': frappe.conf.get("admin_password") or 'admin'
 	}).insert(ignore_if_duplicate=True)

From 7d8695c7d348155ab337619df1a208ffc3e6ec76 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Sat, 8 Aug 2020 23:46:45 +0530
Subject: [PATCH 207/361] fix: Provide fallback array for test_mails to avoid
 unexpected trace in travis

---
 frappe/email/doctype/email_account/email_account.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py
index cf8c6e80c6..29cd890bf1 100755
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@@ -251,7 +251,7 @@ class EmailAccount(Document):
 			email_server = None
 
 			if frappe.local.flags.in_test:
-				incoming_mails = test_mails
+				incoming_mails = test_mails or []
 			else:
 				email_sync_rule = self.build_email_sync_rule()
 

From 0919dbeccd8c7367f0a26e2af3ebf75b22814f9a Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Sun, 9 Aug 2020 00:06:25 +0530
Subject: [PATCH 208/361] test: Manually delete jobs instead of deleting job
 type

- This removes unwanted traceback of "Scheduled Job Type" not found
while running tests
---
 frappe/tests/test_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py
index e554fd23be..30818e6f17 100644
--- a/frappe/tests/test_scheduler.py
+++ b/frappe/tests/test_scheduler.py
@@ -49,7 +49,7 @@ class TestScheduler(TestCase):
 
 		# 2nd job not loaded
 		self.assertFalse(job.enqueue())
-		job.delete()
+		frappe.db.sql('DELETE FROM `tabScheduled Job Log` WHERE `scheduled_job_type`=%s', job.name)
 
 	def test_is_dormant(self):
 		self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00')))

From be8566e1acdbcbf941b49c47b66b5b6a250c2232 Mon Sep 17 00:00:00 2001
From: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
Date: Sun, 9 Aug 2020 14:23:25 +0200
Subject: [PATCH 209/361] feat: add currency Congolese franc

---
 frappe/geo/country_info.json | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json
index 1bf5ada467..6a56107333 100644
--- a/frappe/geo/country_info.json
+++ b/frappe/geo/country_info.json
@@ -620,7 +620,12 @@
  },
  "Congo, The Democratic Republic of the": {
   "code": "cd",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "currency": "CDF",
+  "currency_name": "Congolese franc",
+  "currency_symbol": "FC",
+  "currency_fraction": "Centime",
+  "currency_fraction_units": 100
  },
  "Cook Islands": {
   "code": "ck",

From be4d3ae43492e72f2fe94bd5369e59e7626c74b8 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 4 Aug 2020 17:16:49 +0530
Subject: [PATCH 210/361] feat: allow wide attribute in msgprint and throw

---
 frappe/__init__.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index c6e762919f..add18cdc58 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -299,7 +299,7 @@ def log(msg):
 
 	debug_log.append(as_unicode(msg))
 
-def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None):
+def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None, alert=False, primary_action=None, is_minimizable=None, wide=None):
 	"""Print a message to the user (via HTTP response).
 	Messages are sent in the `__server_messages` property in the
 	response JSON and shown in a pop-up / modal.
@@ -309,6 +309,8 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None,
 	:param raise_exception: [optional] Raise given exception and show message.
 	:param as_table: [optional] If `msg` is a list of lists, render as HTML table.
 	:param primary_action: [optional] Bind a primary server/client side action.
+	:param is_minimizable: [optional] Allow users to minimize the modal
+	:param wide: [optional] Show wide modal
 	"""
 	from frappe.utils import encode
 
@@ -366,6 +368,9 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None,
 	if primary_action:
 		out.primary_action = primary_action
 
+	if wide:
+		out.wide = wide
+
 	message_log.append(json.dumps(out))
 
 	if raise_exception and hasattr(raise_exception, '__name__'):
@@ -387,12 +392,12 @@ def clear_last_message():
 	if len(local.message_log) > 0:
 		local.message_log = local.message_log[:-1]
 
-def throw(msg, exc=ValidationError, title=None, is_minimizable=None):
+def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None):
 	"""Throw execption and show message (`msgprint`).
 
 	:param msg: Message.
 	:param exc: Exception class. Default `frappe.ValidationError`"""
-	msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable)
+	msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable, wide=wide)
 
 def emit_js(js, user=False, **kwargs):
 	if user == False:

From 435b522dbf34980c9af56b140bc5a6a76087f484 Mon Sep 17 00:00:00 2001
From: Saurabh <saurabh6790@gmail.com>
Date: Mon, 10 Aug 2020 11:53:39 +0530
Subject: [PATCH 211/361] fix: cache all apps in local and switch to
 enable/disable frappe logger

---
 frappe/__init__.py |  6 +++++-
 frappe/app.py      | 19 ++++++++++---------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index c6e762919f..58207607c3 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -153,6 +153,7 @@ def init(site, sites_path=None, new_site=False):
 	local.site = site
 	local.sites_path = sites_path
 	local.site_path = os.path.join(sites_path, site)
+	local.all_apps = None
 
 	local.request_ip = None
 	local.response = _dict({"docs":[]})
@@ -916,10 +917,13 @@ def get_installed_apps(sort=False, frappe_last=False):
 	if not db:
 		connect()
 
+	if not local.all_apps:
+		local.all_apps  = get_all_apps(True)
+
 	installed = json.loads(db.get_global("installed_apps") or "[]")
 
 	if sort:
-		installed = [app for app in get_all_apps(True) if app in installed]
+		installed = [app for app in local.all_apps if app in installed]
 
 	if frappe_last:
 		if 'frappe' in installed:
diff --git a/frappe/app.py b/frappe/app.py
index 725bec183a..39bff83122 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -99,15 +99,16 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-			"site": get_site_name(request.host),
-			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-			"base_url": getattr(request, "base_url", "NOTFOUND"),
-			"full_path": getattr(request, "full_path", "NOTFOUND"),
-			"method": getattr(request, "method", "NOTFOUND"),
-			"scheme": getattr(request, "scheme", "NOTFOUND"),
-			"http_status_code": getattr(response, "status_code", "NOTFOUND")
-		})
+		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+				"site": get_site_name(request.host),
+				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+				"base_url": getattr(request, "base_url", "NOTFOUND"),
+				"full_path": getattr(request, "full_path", "NOTFOUND"),
+				"method": getattr(request, "method", "NOTFOUND"),
+				"scheme": getattr(request, "scheme", "NOTFOUND"),
+				"http_status_code": getattr(response, "status_code", "NOTFOUND")
+			})
 
 		if response and hasattr(frappe.local, 'rate_limiter'):
 			response.headers.extend(frappe.local.rate_limiter.headers())

From b76a01f28d36d8d065f140c7e96234a26c1b84cd Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 12:50:37 +0530
Subject: [PATCH 212/361] refactor: add set admin to finally

---
 frappe/search/website_search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py
index ca6f0fe56b..de93fea3f5 100644
--- a/frappe/search/website_search.py
+++ b/frappe/search/website_search.py
@@ -56,11 +56,11 @@ class WebsiteSearch(FullTextSearch):
 			text_content = page_content.text if page_content else ""
 			title = soup.title.text.strip() if soup.title else route
 
-			frappe.set_user("Administrator")
-
 			return frappe._dict(title=title, content=text_content, path=route)
 		except Exception:
 			pass
+		finally:
+			frappe.set_user("Administrator")
 
 	def parse_result(self, result):
 		title_highlights = result.highlights("title")

From e1d9909b78da6dd2e15165fdf92b2f2f96b03280 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 12:50:49 +0530
Subject: [PATCH 213/361] feat: do not update search index in tests

---
 frappe/website/website_generator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index 427a264725..c406679e54 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -148,7 +148,7 @@ class WebsiteGenerator(Document):
 
 	def remove_old_route_from_index(self):
 		"""Remove page from the website index if the route has changed."""
-		if self.allow_website_search_indexing():
+		if self.allow_website_search_indexing() or frappe.flags.in_test:
 			return
 		old_doc = self.get_doc_before_save()
 		# Check if the route is changed
@@ -162,7 +162,7 @@ class WebsiteGenerator(Document):
 			- remove document from index if document is unpublished
 			- update index otherwise
 		"""
-		if not self.allow_website_search_indexing():
+		if not self.allow_website_search_indexing() or frappe.flags.in_test:
 			return
 
 		if self.is_website_published():

From c72a19f696218fd720083a7b13553453bc9242c4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 13:52:11 +0530
Subject: [PATCH 214/361] fix: reference errors in test

---
 frappe/search/test_full_text_search.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index feaa8ca84a..57e908da99 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -43,13 +43,13 @@ class TestFullTextSearch(unittest.TestCase):
 		self.assertTrue('os/gnu' in res)
 
 	def test_remove_document_from_index(self):
-		self.remove_document_from_index("os/gnu")
+		self.index.remove_document_from_index("os/gnu")
 		res = self.index.search("GNU")
 		self.assertEqual(len(res), 0)
 
 	def test_update_index(self):
 		# Update existing index
-		self.update_index({
+		self.index.update_index({
 			'name': "sw/erpnext",
 			'content': """AwesomeERPNext"""
 		})
@@ -61,7 +61,7 @@ class TestFullTextSearch(unittest.TestCase):
 		self.assertEqual(res[0], "sw/erpnext")
 
 		# Update new doc
-		self.update_index({
+		self.index.update_index({
 			'name': "sw/frappebooks",
 			'content': """DesktopAccounting"""
 		})

From 579f5dfc8bc5150812e244c777cf44f8e35078de Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 14:23:44 +0530
Subject: [PATCH 215/361] fix: test conditions

---
 frappe/search/test_full_text_search.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index 57e908da99..daa58265bb 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -13,13 +13,13 @@ class TestFullTextSearch(unittest.TestCase):
 	def test_search_term(self):
 		# Search Wikipedia
 		res = self.index.search("multilingual online encyclopedia")
-		self.assertEqual(res[0], 'wikipedia')
+		self.assertEqual(res[0], 'site/wikipedia')
 
 		res = self.index.search("Linux is a family")
-		self.assertEqual(res[0], 'linux')
+		self.assertEqual(res[0], 'os/linux')
 
 		res = self.index.search("Enterprise Resource Planning")
-		self.assertEqual(res[0], 'erpnext')
+		self.assertEqual(res[0], 'sw/erpnext')
 
 	def test_search_limit(self):
 		res = self.index.search("CommonSearchTerm")

From 8d71dad60bb8f5eb519fda8fdbb1886582e62a82 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 14:24:00 +0530
Subject: [PATCH 216/361] feat: delete doc only if valid name is provided

---
 frappe/search/full_text_search.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py
index 4dc405a5d2..dd6e69111d 100644
--- a/frappe/search/full_text_search.py
+++ b/frappe/search/full_text_search.py
@@ -55,6 +55,9 @@ class FullTextSearch:
 			self (object): FullTextSearch Instance
 			doc_name (str): name of the document to be removed
 		"""
+		if not doc_name:
+			return
+
 		ix = self.get_index()
 		with ix.searcher():
 			writer = ix.writer()

From 64f913fa36721f7477a03507f780ab6b03ff56be Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 16:51:37 +0530
Subject: [PATCH 217/361] refactor: catch exceptions

---
 frappe/public/js/frappe/form/formatters.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js
index a7adc9175a..39541757a5 100644
--- a/frappe/public/js/frappe/form/formatters.js
+++ b/frappe/public/js/frappe/form/formatters.js
@@ -230,9 +230,14 @@ frappe.form.formatters = {
 	TextEditor: function(value) {
 		let formatted_value = frappe.form.formatters.Text(value);
 		// to use ql-editor styles
-		if (!$(formatted_value).find('.ql-editor').length) {
+		try {
+			if (!$(formatted_value).find('.ql-editor').length) {
+				formatted_value = `<div class="ql-editor read-mode">${formatted_value}</div>`;
+			}
+		} catch(e) {
 			formatted_value = `<div class="ql-editor read-mode">${formatted_value}</div>`;
 		}
+
 		return formatted_value;
 	},
 	Code: function(value) {

From 2076a81d295bb661117d326c147c03e6319da98e Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Mon, 10 Aug 2020 17:52:26 +0530
Subject: [PATCH 218/361] fix: catch only expected error

---
 frappe/utils/data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index f36fa656b0..ab334372e8 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -96,7 +96,7 @@ def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, se
 			as_datetime = True
 		try:
 			date = parser.parse(date)
-		except:
+		except ParserError:
 			frappe.throw(frappe._("Please select a valid date filter"), title=frappe._("Invalid Date"))
 
 	date = date + relativedelta(years=years, months=months, weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)

From 24c5fbf743f9adc14c79a661bdb8d19d90c6a8e4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 10 Aug 2020 21:34:02 +0530
Subject: [PATCH 219/361] test: fix tests

---
 frappe/search/test_full_text_search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py
index daa58265bb..be9669d3c5 100644
--- a/frappe/search/test_full_text_search.py
+++ b/frappe/search/test_full_text_search.py
@@ -15,7 +15,7 @@ class TestFullTextSearch(unittest.TestCase):
 		res = self.index.search("multilingual online encyclopedia")
 		self.assertEqual(res[0], 'site/wikipedia')
 
-		res = self.index.search("Linux is a family")
+		res = self.index.search("Linux kernel")
 		self.assertEqual(res[0], 'os/linux')
 
 		res = self.index.search("Enterprise Resource Planning")

From 23975d5a70e24d3f90dd4c6e24231fa3d1752d10 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 11 Aug 2020 20:13:56 +0530
Subject: [PATCH 220/361] feat: catch exceptions on tokenizing values in
 get_value

---
 frappe/public/js/frappe/form/controls/text_editor.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/controls/text_editor.js b/frappe/public/js/frappe/form/controls/text_editor.js
index 3fe0b60ae0..5881e52d74 100644
--- a/frappe/public/js/frappe/form/controls/text_editor.js
+++ b/frappe/public/js/frappe/form/controls/text_editor.js
@@ -201,9 +201,14 @@ frappe.ui.form.ControlTextEditor = frappe.ui.form.ControlCode.extend({
 		// hack to retain space sequence.
 		value = value.replace(/(\s)(\s)/g, ' &nbsp;');
 
-		if (!$(value).find('.ql-editor').length) {
+		try {
+			if (!$(value).find('.ql-editor').length) {
+				value = `<div class="ql-editor read-mode">${value}</div>`;
+			}
+		} catch(e) {
 			value = `<div class="ql-editor read-mode">${value}</div>`;
 		}
+
 		return value;
 	},
 

From b7a5bcadfd3f56df10dae0001c085af97bd755fc Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Tue, 11 Aug 2020 22:41:45 +0530
Subject: [PATCH 221/361] chore: Update redis to avoid unexpected exception
 (#11235)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2995553541..46e853794e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -50,7 +50,7 @@ python-dateutil==2.8.1
 pytz==2019.3
 PyYAML==5.3.1
 rauth==0.7.3
-redis==3.5.1
+redis==3.5.3
 requests-oauthlib==1.3.0
 requests==2.23.0
 RestrictedPython==5.0

From a7f3956740670c6b7de76ee996376d3af97638ab Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Tue, 11 Aug 2020 23:04:05 +0530
Subject: [PATCH 222/361] fix: Pass entire navbar settings

---
 frappe/boot.py                                |  4 +--
 .../navbar_settings/navbar_settings.py        | 31 ++-----------------
 .../public/js/frappe/ui/toolbar/navbar.html   | 14 ++++-----
 frappe/public/js/frappe/ui/toolbar/toolbar.js |  2 +-
 4 files changed, 13 insertions(+), 38 deletions(-)

diff --git a/frappe/boot.py b/frappe/boot.py
index df870adc60..5b1a1bf573 100644
--- a/frappe/boot.py
+++ b/frappe/boot.py
@@ -21,7 +21,7 @@ from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabl
 from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points
 from frappe.model.base_document import get_controller
 from frappe.social.doctype.post.post import frequently_visited_links
-from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_items
+from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings
 
 def get_bootinfo():
 	"""build and return boot info"""
@@ -60,7 +60,7 @@ def get_bootinfo():
 	load_print(bootinfo, doclist)
 	doclist.extend(get_meta_bundle("Page"))
 	bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1})
-	bootinfo.navbar_items = get_navbar_items()
+	bootinfo.navbar_settings = get_navbar_settings()
 
 	# ipinfo
 	if frappe.session.data.get('ipinfo'):
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py
index b95482d6d9..f7c437bf00 100644
--- a/frappe/core/doctype/navbar_settings/navbar_settings.py
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.py
@@ -21,7 +21,7 @@ class NavbarSettings(Document):
 			self.help_dropdown + self.settings_dropdown if item.is_standard]
 
 		if not frappe.flags.in_patch and (len(before_save_items) > len(after_save_items)):
-			frappe.throw(_("Cannot delete standard navbar items"))
+			frappe.throw(_("Please hide the standard navbar items instead of deleting them"))
 
 @frappe.whitelist()
 def get_app_logo():
@@ -31,34 +31,9 @@ def get_app_logo():
 
 	return app_logo
 
-def get_navbar_items():
+def get_navbar_settings():
 	navbar_settings = frappe.get_single('Navbar Settings')
-
-	navbar_items = frappe._dict()
-	navbar_items.help_dropdown = []
-	navbar_items.settings_dropdown = []
-
-	for item in navbar_settings.settings_dropdown:
-		navbar_items.settings_dropdown.append({
-			'label': item.item_label,
-			'item_type': item.item_type,
-			'route': item.route,
-			'hidden': item.hidden,
-			'action': item.action
-		})
-
-	for item in navbar_settings.help_dropdown:
-		navbar_items.help_dropdown.append({
-			'label': item.item_label,
-			'item_type': item.item_type,
-			'route': item.route,
-			'hidden': item.hidden,
-			'action': item.action
-		})
-
-	navbar_items.logo_width = navbar_settings.logo_width
-
-	return navbar_items
+	return navbar_settings
 
 
 
diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html
index b66049183e..820902659e 100644
--- a/frappe/public/js/frappe/ui/toolbar/navbar.html
+++ b/frappe/public/js/frappe/ui/toolbar/navbar.html
@@ -5,7 +5,7 @@
 				<i class="octicon octicon-three-bars"></i>
 			</a>
 			<a class="navbar-brand navbar-home hidden-xs hidden-sm" href="#">
-				<img class="app-logo" style="width: {{navbar_items.logo_width || 24}}px" src="{{ frappe.app.logo_url }}">
+				<img class="app-logo" style="width: {{navbar_settings.logo_width || 24}}px" src="{{ frappe.app.logo_url }}">
 			</a>
 			<ul class="nav navbar-nav" id="navbar-breadcrumbs">
 			</ul>
@@ -23,18 +23,18 @@
 					{%= __("Settings") %}</span>
 				<b class="caret hidden-xs hidden-sm"></b></a>
 				<ul class="dropdown-menu" id="toolbar-user" role="menu">
-					{% for item in navbar_items.settings_dropdown %}
+					{% for item in navbar_settings.settings_dropdown %}
 						{% if (!item.hidden) { %}
 							{% if (item.route) { %}
 								<li>
 									<a href="{{ item.route }}">
-										{%= __(item.label) %}
+										{%= __(item.item_label) %}
 									</a>
 								</li>
 							{% } else if (item.action) { %}
 								<li>
 									<a onclick="return {{ item.action }}">
-										{%= __(item.label) %}
+										{%= __(item.item_label) %}
 									</a>
 								</li>
 							{% } else { %}
@@ -55,18 +55,18 @@
 					<!-- For custom links added via jquery -->
 					<li id="help-links"></li>
 					<li class="divider documentation-links"></li>
-					{% for item in navbar_items.help_dropdown %}
+					{% for item in navbar_settings.help_dropdown %}
 						{% if (!item.hidden) { %}
 							{% if (item.route) { %}
 								<li>
 									<a href="{{ item.route }}">
-										{%= __(item.label) %}
+										{%= __(item.item_label) %}
 									</a>
 								</li>
 							{% } else if (item.action) { %}
 								<li>
 									<a onclick="return {{ item.action }}">
-										{%= __(item.label) %}
+										{%= __(item.item_label) %}
 									</a>
 								</li>
 							{% } else { %}
diff --git a/frappe/public/js/frappe/ui/toolbar/toolbar.js b/frappe/public/js/frappe/ui/toolbar/toolbar.js
index a69edd7210..86da3abec2 100644
--- a/frappe/public/js/frappe/ui/toolbar/toolbar.js
+++ b/frappe/public/js/frappe/ui/toolbar/toolbar.js
@@ -8,7 +8,7 @@ frappe.ui.toolbar.Toolbar = Class.extend({
 	init: function() {
 		$('header').append(frappe.render_template("navbar", {
 			avatar: frappe.avatar(frappe.session.user),
-			navbar_items: frappe.boot.navbar_items
+			navbar_settings: frappe.boot.navbar_settings
 		}));
 		$('.dropdown-toggle').dropdown();
 

From a7b3afdff52cafbfc5b858ff93d066cd06cc2c7d Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 11 Aug 2020 23:28:34 +0530
Subject: [PATCH 223/361] fix: Add enable in_test flag for ui_tests

---
 frappe/commands/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index 28b6344b8e..b25aaa6b96 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -528,11 +528,11 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 @pass_context
 def run_ui_tests(context, app, headless=False):
 	"Run UI tests"
-
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
 	site_url = frappe.utils.get_site_url(site)
 	admin_password = frappe.get_conf(site).admin_password
+	frappe.flags.in_test = True
 
 	# override baseUrl using env variable
 	site_env = 'CYPRESS_baseUrl={}'.format(site_url)

From 2bd9ab4d984cd6f4984e655d307350564c534439 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 11 Aug 2020 23:29:14 +0530
Subject: [PATCH 224/361] style: Add spaces

JS change to trigger Cypress test
---
 frappe/public/js/frappe/desk.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js
index 2e80dbfd85..536c5838db 100644
--- a/frappe/public/js/frappe/desk.js
+++ b/frappe/public/js/frappe/desk.js
@@ -5,7 +5,7 @@
 // __('Modules') __('Domains') __('Places') __('Administration') # for translation, don't remove
 
 frappe.start_app = function() {
-	if(!frappe.Application)
+	if (!frappe.Application)
 		return;
 	frappe.assets.check();
 	frappe.provide('frappe.app');
@@ -14,7 +14,7 @@ frappe.start_app = function() {
 };
 
 $(document).ready(function() {
-	if(!frappe.utils.supportsES6) {
+	if (!frappe.utils.supportsES6) {
 		frappe.msgprint({
 			indicator: 'red',
 			title: __('Browser not supported'),

From b03513613f62a2a63a1ab9f4f68445cab21d9063 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Tue, 11 Aug 2020 23:39:55 +0530
Subject: [PATCH 225/361] fix: Init site

---
 frappe/commands/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index b25aaa6b96..d506fe3378 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -532,6 +532,7 @@ def run_ui_tests(context, app, headless=False):
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
 	site_url = frappe.utils.get_site_url(site)
 	admin_password = frappe.get_conf(site).admin_password
+	frappe.init(site=site)
 	frappe.flags.in_test = True
 
 	# override baseUrl using env variable

From e27c910f666bb0874361b3a9fc7aa744b7f51c07 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <surajshetty3416@gmail.com>
Date: Wed, 12 Aug 2020 00:02:39 +0530
Subject: [PATCH 226/361] fix: Make users while setting up class

---
 frappe/commands/utils.py                          | 2 --
 frappe/workflow/doctype/workflow/test_workflow.py | 5 ++++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index d506fe3378..721376016c 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -532,8 +532,6 @@ def run_ui_tests(context, app, headless=False):
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
 	site_url = frappe.utils.get_site_url(site)
 	admin_password = frappe.get_conf(site).admin_password
-	frappe.init(site=site)
-	frappe.flags.in_test = True
 
 	# override baseUrl using env variable
 	site_env = 'CYPRESS_baseUrl={}'.format(site_url)
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 84adc3a096..4ffbe6ff8d 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -8,9 +8,12 @@ from frappe.utils import random_string
 from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions
 from frappe.test_runner import make_test_records
 
-make_test_records("User")
 
 class TestWorkflow(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		make_test_records("User")
+
 	def setUp(self):
 		frappe.db.sql('DELETE FROM `tabToDo`')
 		frappe.db.sql("DELETE FROM `tabHas Role` WHERE `role`='Test Approver'")

From 1cd4cb48e9a4c7e6f0ff02f25865c59549c4e4f6 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:07:45 +0530
Subject: [PATCH 227/361] fix: error message

---
 frappe/public/js/frappe/views/treeview.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/views/treeview.js b/frappe/public/js/frappe/views/treeview.js
index 41a2c200cd..815e561f41 100644
--- a/frappe/public/js/frappe/views/treeview.js
+++ b/frappe/public/js/frappe/views/treeview.js
@@ -14,7 +14,7 @@ frappe.views.TreeFactory = class TreeFactory extends frappe.views.Factory {
 			};
 
 			if (!frappe.treeview_settings[route[1]] && !frappe.meta.get_docfield(route[1], "is_group")) {
-				frappe.msgprint(__("Tree view not available for {0}", [route[1]] ));
+				frappe.msgprint(__("Tree view is not available for {0}", [route[1]] ));
 				return false;
 			}
 			$.extend(options, frappe.treeview_settings[route[1]] || {});

From 79c1a5fe24915d75dac71889a0ab14414d60a1ba Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 14:08:04 +0530
Subject: [PATCH 228/361] fix: label for group by column

---
 frappe/core/doctype/report/report.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 3228fec13a..dd695d484e 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -229,7 +229,12 @@ class Report(Document):
 				if meta.get_field(fieldname):
 					field = meta.get_field(fieldname)
 				else:
-					field = frappe._dict(fieldname=fieldname, label=meta.get_label(fieldname))
+					if fieldname == '_aggregate_column':
+						label = get_group_by_column_label(group_by_args, meta)
+					else:
+						label = meta.get_label(fieldname)
+
+					field = frappe._dict(fieldname=fieldname, label=label)
 					# since name is the primary key for a document, it will always be a Link datatype
 					if fieldname == "name":
 						field.fieldtype = "Link"
@@ -282,4 +287,19 @@ def get_group_by_field(args, doctype):
 			args.aggregate_on
 		)
 
-	return group_by_field
\ No newline at end of file
+	return group_by_field
+
+def get_group_by_column_label(args, meta):
+	if args['aggregate_function'] == 'count':
+		label = 'Count'
+	else:
+		sql_fn_map = {
+			'avg': 'Average',
+			'sum': 'Sum'
+		}
+		aggregate_on_label = meta.get_label(args.aggregate_on)
+		label = _('{function} of {fieldlabel}').format(
+			function=sql_fn_map[args.aggregate_function],
+			fieldlabel = aggregate_on_label
+		)
+	return label

From 478ebc2e51436e2e42657652326e2f7a49e8e67c Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:08:09 +0530
Subject: [PATCH 229/361] feat: don't show views for single doctypes

---
 frappe/public/js/frappe/widgets/widget_dialog.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index 1728cbe863..e91f04fbb1 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -192,7 +192,13 @@ class ShortcutDialog extends WidgetDialog {
 				options: "\nList\nReport Builder\nDashboard\nTree\nNew",
 				description: "Which view of the associated DocType should this shortcut take you to?",
 				depends_on: (state) => {
-					return state.type == "DocType";
+					if (this.dialog) {
+						let doctype = this.dialog.get_value("link_to");
+						let is_single = frappe.boot.single_types.includes(doctype);
+						return state.type == "DocType" && !is_single;
+					}
+
+					return false
 				}
 			},
 			{

From ca46c24202c8e1cb3e4af59496242dfdfe351067 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:08:25 +0530
Subject: [PATCH 230/361] feat: conditionally show options for doc_view

---
 frappe/public/js/frappe/widgets/widget_dialog.js | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index e91f04fbb1..de3bcc785e 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -180,17 +180,25 @@ class ShortcutDialog extends WidgetDialog {
 							this.setup_filter(doctype);
 							this.show_filters();
 						}
+
+						const views = ["List", "Report Builder", "Dashboard", "New"]
+						if (frappe.boot.treeviews.includes(doctype)) views.push("Tree")
+						if (frappe.boot.calendars.includes(doctype)) views.push("Calendar")
+
+						this.dialog.set_df_property("doc_view", "options", views.join("\n"))
+
 					} else {
 						this.hide_filters();
 					}
-				},
+				}
 			},
 			{
 				fieldtype: "Select",
 				fieldname: "doc_view",
 				label: "DocType View",
-				options: "\nList\nReport Builder\nDashboard\nTree\nNew",
+				options: "List\nReport Builder\nDashboard\nTree\nNew\nCalendar",
 				description: "Which view of the associated DocType should this shortcut take you to?",
+				default: "List",
 				depends_on: (state) => {
 					if (this.dialog) {
 						let doctype = this.dialog.get_value("link_to");

From 8bd113212c916350e3cbd4b01ab742d40f572452 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:13:05 +0530
Subject: [PATCH 231/361] feat: add calendar option for doc_view

---
 frappe/desk/doctype/desk_shortcut/desk_shortcut.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/desk/doctype/desk_shortcut/desk_shortcut.json b/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
index faf0b55d4b..fbcc2dfb82 100644
--- a/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
+++ b/frappe/desk/doctype/desk_shortcut/desk_shortcut.json
@@ -42,7 +42,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "DocType View",
-   "options": "\nList\nReport Builder\nDashboard\nTree\nNew"
+   "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar"
   },
   {
    "fieldname": "stats_filter",
@@ -96,9 +96,10 @@
    "reqd": 1
   }
  ],
+ "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-07-30 12:35:48.095761",
+ "modified": "2020-08-12 14:11:55.080390",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Desk Shortcut",

From cf1bf9150847a7d6dcc330b8b5a7d30efabac0e4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:13:14 +0530
Subject: [PATCH 232/361] feat: add condition for calendar

---
 frappe/public/js/frappe/widgets/utils.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index 00a2cb5df6..2c2d36feb6 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -32,11 +32,14 @@ function generate_route(item) {
 					route = "List/" + item.doctype + "/Report";
 					break;
 				case "Dashboard":
-				        route = "List/" + item.doctype + "/Dashboard";
+				    route = "List/" + item.doctype + "/Dashboard";
 					break;
 				case "New":
 					route = "Form/" + item.doctype + "/New " + item.doctype;
 					break;
+				case "Calendar":
+					route = "List/" + item.doctype + "/Calendar/Default";
+					break;
 				default:
 					frappe.throw({message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View")});
 					route = "";

From dae8f30151024307fede77b7753d8adce9717930 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:14:17 +0530
Subject: [PATCH 233/361] style: fix indentation

---
 frappe/public/js/frappe/widgets/utils.js | 60 ++++++++++++------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index 2c2d36feb6..293ef29567 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -19,30 +19,30 @@ function generate_route(item) {
 					}
 				}
 				switch (item.doc_view) {
-				case "List":
-					if (item.filters) {
-						frappe.route_options = item.filters;
-					}
-					route = "List/" + item.doctype;
-					break;
-				case "Tree":
-					route = "Tree/" + item.doctype;
-					break;
-				case "Report Builder":
-					route = "List/" + item.doctype + "/Report";
-					break;
-				case "Dashboard":
-				    route = "List/" + item.doctype + "/Dashboard";
-					break;
-				case "New":
-					route = "Form/" + item.doctype + "/New " + item.doctype;
-					break;
-				case "Calendar":
-					route = "List/" + item.doctype + "/Calendar/Default";
-					break;
-				default:
-					frappe.throw({message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View")});
-					route = "";
+					case "List":
+						if (item.filters) {
+							frappe.route_options = item.filters;
+						}
+						route = "List/" + item.doctype;
+						break;
+					case "Tree":
+						route = "Tree/" + item.doctype;
+						break;
+					case "Report Builder":
+						route = "List/" + item.doctype + "/Report";
+						break;
+					case "Dashboard":
+						route = "List/" + item.doctype + "/Dashboard";
+						break;
+					case "New":
+						route = "Form/" + item.doctype + "/New " + item.doctype;
+						break;
+					case "Calendar":
+						route = "List/" + item.doctype + "/Calendar/Default";
+						break;
+					default:
+						frappe.throw({ message: __("Not a valid DocType view:") + item.doc_view, title: __("Unknown View") });
+						route = "";
 				}
 			}
 		} else if (type === "report" && item.is_query_report) {
@@ -63,7 +63,7 @@ function generate_route(item) {
 	if (item.route_options) {
 		route +=
 			"?" +
-			$.map(item.route_options, function(value, key) {
+			$.map(item.route_options, function (value, key) {
 				return (
 					encodeURIComponent(key) + "=" + encodeURIComponent(value)
 				);
@@ -127,20 +127,20 @@ function generate_grid(data) {
 }
 
 const build_summary_item = (summary) => {
-	let df = {fieldtype: summary.datatype};
+	let df = { fieldtype: summary.datatype };
 	let doc = null;
 
 	if (summary.datatype == "Currency") {
 		df.options = "currency";
-		doc = {currency: summary.currency};
+		doc = { currency: summary.currency };
 	}
 
 	let value = frappe.format(summary.value, df, null, doc);
-	let indicator = summary.indicator ? `indicator ${ summary.indicator.toLowerCase() }` : '';
+	let indicator = summary.indicator ? `indicator ${summary.indicator.toLowerCase()}` : '';
 
 	return $(`<div class="summary-item">
 		<span class="summary-label small text-muted ${indicator}">${summary.label}</span>
-		<h1 class="summary-value">${ value }</h1>
+		<h1 class="summary-value">${ value}</h1>
 	</div>`);
 };
 
@@ -150,7 +150,7 @@ function shorten_number(number, country) {
 	let x = Math.abs(Math.round(number));
 	for (const map of number_system) {
 		if (x >= map.divisor) {
-			return Math.round(number/map.divisor) +  ' ' + map.symbol;
+			return Math.round(number / map.divisor) + ' ' + map.symbol;
 		}
 	}
 	return number.toFixed();

From 5903a0c58229a353c1ba269cf88604231922da2c Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 12 Aug 2020 14:18:31 +0530
Subject: [PATCH 234/361] style: fix linting issues

---
 frappe/public/js/frappe/widgets/widget_dialog.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js
index de3bcc785e..d5b6df6021 100644
--- a/frappe/public/js/frappe/widgets/widget_dialog.js
+++ b/frappe/public/js/frappe/widgets/widget_dialog.js
@@ -181,11 +181,11 @@ class ShortcutDialog extends WidgetDialog {
 							this.show_filters();
 						}
 
-						const views = ["List", "Report Builder", "Dashboard", "New"]
-						if (frappe.boot.treeviews.includes(doctype)) views.push("Tree")
-						if (frappe.boot.calendars.includes(doctype)) views.push("Calendar")
+						const views = ["List", "Report Builder", "Dashboard", "New"];
+						if (frappe.boot.treeviews.includes(doctype)) views.push("Tree");
+						if (frappe.boot.calendars.includes(doctype)) views.push("Calendar");
 
-						this.dialog.set_df_property("doc_view", "options", views.join("\n"))
+						this.dialog.set_df_property("doc_view", "options", views.join("\n"));
 
 					} else {
 						this.hide_filters();
@@ -206,7 +206,7 @@ class ShortcutDialog extends WidgetDialog {
 						return state.type == "DocType" && !is_single;
 					}
 
-					return false
+					return false;
 				}
 			},
 			{

From 5ab919ed7565634abb36e80f3a30a2a593526b9b Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 14:39:17 +0530
Subject: [PATCH 235/361] refactor: enable auto deletion by default

---
 frappe/core/doctype/system_settings/system_settings.js   | 2 +-
 frappe/core/doctype/system_settings/system_settings.json | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js
index eebc984a2f..b6514dea9f 100644
--- a/frappe/core/doctype/system_settings/system_settings.js
+++ b/frappe/core/doctype/system_settings/system_settings.js
@@ -31,7 +31,7 @@ frappe.ui.form.on("System Settings", "enable_two_factor_auth", function(frm) {
 frappe.ui.form.on("System Settings", "enable_prepared_report_auto_deletion", function(frm) {
 	if (frm.doc.enable_prepared_report_auto_deletion) {
 		if (!frm.doc.prepared_report_expiry_period) {
-			frm.set_value('prepared_report_expiry_period', 30);
+			frm.set_value('prepared_report_expiry_period', 7);
 		}
 	}
 });
diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json
index 45ab84e62b..819bb89e72 100644
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@@ -442,7 +442,7 @@
    "label": "Prepared Report Expiry Period (Days)"
   },
   {
-   "default": "0",
+   "default": "1",
    "fieldname": "enable_prepared_report_auto_deletion",
    "fieldtype": "Check",
    "label": "Enable Auto-deletion of Prepared Reports"
@@ -457,7 +457,7 @@
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2020-07-26 17:58:34.197718",
+ "modified": "2020-08-12 14:35:45.214327",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

From 6038c49fc3a6cfd73579af3f286785f73de37fa6 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 15:03:47 +0530
Subject: [PATCH 236/361] fix: translation syntax

---
 frappe/public/js/frappe/views/reports/query_report.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 83926c18be..e0b9019ca3 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -732,12 +732,12 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		const route = `#List/Prepared Report/List?status=Queued&report_name=${this.report_name}`;
 		const no_of_reports_html = reports.length == 1
 			? `${__('There is ')}<a class="underline" href="${route}">${__('1 Report')}</a>`
-			: `${__('There are ')}<a class="underline" href="${route}">${__(`${reports.length} Reports`)}</a>`;
+			: `${__('There are ')}<a class="underline" href="${route}">${__(`{} Reports`, [reports.length])}</a>`;
 
 		let warning_message = `
 			<p>
 				${__(`Are you sure you want to generate a new report?
-					${no_of_reports_html} with the same filters already in the queue:`)}
+					{} with the same filters already in the queue:`, [no_of_reports_html])}
 			</p>`;
 
 		let get_item_html = item => `<a class="underline" href="#Form/Prepared Report/${item.name}">${item.name}</a>`;

From d7a648f7bfdaef01966df378fff2dc17ffefb842 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 12 Aug 2020 15:49:48 +0530
Subject: [PATCH 237/361] fix: Show message for invalid _sidebar.json

---
 frappe/website/context.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/frappe/website/context.py b/frappe/website/context.py
index c5f23ab21c..335d0c0643 100644
--- a/frappe/website/context.py
+++ b/frappe/website/context.py
@@ -129,8 +129,12 @@ def build_context(context):
 
 def load_sidebar(context, sidebar_json_path):
 	with open(sidebar_json_path, 'r') as sidebarfile:
-		context.sidebar_items = json.loads(sidebarfile.read())
-		context.show_sidebar = 1
+		try:
+			sidebar_json = sidebarfile.read()
+			context.sidebar_items = json.loads(sidebar_json)
+			context.show_sidebar = 1
+		except json.decoder.JSONDecodeError:
+			frappe.throw('Invalid Sidebar JSON at ' + sidebar_json_path)
 
 def get_sidebar_json_path(path, look_for=False):
 	'''

From f8de13071c3f096242104e1e240e55b7bf9ad587 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 12 Aug 2020 15:52:22 +0530
Subject: [PATCH 238/361] fix(bug): pass correct fieldname to refresh_field

---
 frappe/public/js/frappe/form/grid_row.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js
index f4bcecc68e..733c1bea5f 100644
--- a/frappe/public/js/frappe/form/grid_row.js
+++ b/frappe/public/js/frappe/form/grid_row.js
@@ -396,7 +396,7 @@ export default class GridRow {
 		var field_on_change_function = field.df.onchange;
 		field.df.onchange = function(e) {
 			field_on_change_function && field_on_change_function(e);
-			me.grid.grid_rows[this.doc.idx - 1].refresh_field(this.df.fieldname);
+			me.grid.grid_rows[this.doc.idx - 1].refresh_field(field.df.fieldname);
 		};
 		field.refresh();
 		if(field.$input) {

From 0bfba240cfbeaa5a80ed56b982917453f3ed4fc5 Mon Sep 17 00:00:00 2001
From: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
Date: Wed, 12 Aug 2020 12:39:59 +0200
Subject: [PATCH 239/361] feat: allow long passwords (#11065)

Co-authored-by: Chinmay Pai <chinmaydpai@gmail.com>
---
 frappe/database/mariadb/database.py           |  4 ++--
 frappe/database/mariadb/framework_mariadb.sql |  2 +-
 frappe/database/postgres/database.py          |  4 ++--
 .../database/postgres/framework_postgres.sql  |  2 +-
 frappe/patches.txt                            |  1 +
 .../patches/v13_0/increase_password_length.py |  7 ++++++
 frappe/utils/password.py                      | 22 +++++++++++--------
 7 files changed, 27 insertions(+), 15 deletions(-)
 create mode 100644 frappe/patches/v13_0/increase_password_length.py

diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py
index 4ec89c126d..3cbb2e4f0e 100644
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@@ -46,7 +46,7 @@ class MariaDBDatabase(Database):
 			'Data':			('varchar', self.VARCHAR_LEN),
 			'Link':			('varchar', self.VARCHAR_LEN),
 			'Dynamic Link':	('varchar', self.VARCHAR_LEN),
-			'Password':		('varchar', self.VARCHAR_LEN),
+			'Password':		('text', ''),
 			'Select':		('varchar', self.VARCHAR_LEN),
 			'Rating':		('int', '1'),
 			'Read Only':	('varchar', self.VARCHAR_LEN),
@@ -186,7 +186,7 @@ class MariaDBDatabase(Database):
 				`doctype` VARCHAR(140) NOT NULL,
 				`name` VARCHAR(255) NOT NULL,
 				`fieldname` VARCHAR(140) NOT NULL,
-				`password` VARCHAR(255) NOT NULL,
+				`password` TEXT NOT NULL,
 				`encrypted` INT(1) NOT NULL DEFAULT 0,
 				PRIMARY KEY (`doctype`, `name`, `fieldname`)
 			) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql
index af537e0612..1e3749e030 100644
--- a/frappe/database/mariadb/framework_mariadb.sql
+++ b/frappe/database/mariadb/framework_mariadb.sql
@@ -277,7 +277,7 @@ CREATE TABLE `__Auth` (
 	`doctype` VARCHAR(140) NOT NULL,
 	`name` VARCHAR(255) NOT NULL,
 	`fieldname` VARCHAR(140) NOT NULL,
-	`password` VARCHAR(255) NOT NULL,
+	`password` TEXT NOT NULL,
 	`encrypted` INT(1) NOT NULL DEFAULT 0,
 	PRIMARY KEY (`doctype`, `name`, `fieldname`)
 ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py
index e348916705..3d997864e4 100644
--- a/frappe/database/postgres/database.py
+++ b/frappe/database/postgres/database.py
@@ -51,7 +51,7 @@ class PostgresDatabase(Database):
 			'Data':			('varchar', self.VARCHAR_LEN),
 			'Link':			('varchar', self.VARCHAR_LEN),
 			'Dynamic Link':	('varchar', self.VARCHAR_LEN),
-			'Password':		('varchar', self.VARCHAR_LEN),
+			'Password':		('text', ''),
 			'Select':		('varchar', self.VARCHAR_LEN),
 			'Rating':		('smallint', None),
 			'Read Only':	('varchar', self.VARCHAR_LEN),
@@ -179,7 +179,7 @@ class PostgresDatabase(Database):
 				"doctype" VARCHAR(140) NOT NULL,
 				"name" VARCHAR(255) NOT NULL,
 				"fieldname" VARCHAR(140) NOT NULL,
-				"password" VARCHAR(255) NOT NULL,
+				"password" TEXT NOT NULL,
 				"encrypted" INT NOT NULL DEFAULT 0,
 				PRIMARY KEY ("doctype", "name", "fieldname")
 			)""")
diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql
index 8f77ed6230..a946a7ee5c 100644
--- a/frappe/database/postgres/framework_postgres.sql
+++ b/frappe/database/postgres/framework_postgres.sql
@@ -281,7 +281,7 @@ CREATE TABLE "__Auth" (
 	"doctype" VARCHAR(140) NOT NULL,
 	"name" VARCHAR(255) NOT NULL,
 	"fieldname" VARCHAR(140) NOT NULL,
-	"password" VARCHAR(255) NOT NULL,
+	"password" TEXT NOT NULL,
 	"encrypted" int NOT NULL DEFAULT 0,
 	PRIMARY KEY ("doctype", "name", "fieldname")
 );
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 75750ab59c..68b4e5d99c 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -297,3 +297,4 @@ frappe.patches.v13_0.replace_old_data_import # 2020-06-24
 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts
 frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
 frappe.patches.v13_0.generate_theme_files_in_public_folder
+frappe.patches.v13_0.increase_password_length
diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py
new file mode 100644
index 0000000000..1bb1979051
--- /dev/null
+++ b/frappe/patches/v13_0/increase_password_length.py
@@ -0,0 +1,7 @@
+import frappe
+
+def execute():
+	frappe.db.multisql({
+		"mariadb": "ALTER  TABLE `__Auth` MODIFY `password` TEXT NOT NULL",
+		"postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT'
+	})
diff --git a/frappe/utils/password.py b/frappe/utils/password.py
index b939607b19..b1461ce7fe 100644
--- a/frappe/utils/password.py
+++ b/frappe/utils/password.py
@@ -10,6 +10,8 @@ from cryptography.fernet import Fernet, InvalidToken
 from passlib.hash import pbkdf2_sha256, mysql41
 from passlib.registry import register_crypt_handler
 from passlib.context import CryptContext
+from pymysql.constants.ER import DATA_TOO_LONG
+from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION
 
 class LegacyPassword(pbkdf2_sha256):
 	name = "frappe_legacy"
@@ -49,11 +51,17 @@ def get_decrypted_password(doctype, name, fieldname='password', raise_exception=
 		frappe.throw(_('Password not found'), frappe.AuthenticationError)
 
 def set_encrypted_password(doctype, name, pwd, fieldname='password'):
-	frappe.db.sql("""insert into `__Auth` (doctype, name, fieldname, `password`, encrypted)
-		values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 1)
-		{on_duplicate_update} `password`=%(pwd)s, encrypted=1""".format(
-			on_duplicate_update=frappe.db.get_on_duplicate_update(['doctype', 'name', 'fieldname'])
-		), { 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) })
+	try:
+		frappe.db.sql("""insert into `__Auth` (doctype, name, fieldname, `password`, encrypted)
+			values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 1)
+			{on_duplicate_update} `password`=%(pwd)s, encrypted=1""".format(
+				on_duplicate_update=frappe.db.get_on_duplicate_update(['doctype', 'name', 'fieldname'])
+			), { 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) })
+	except frappe.db.DataError as e:
+		if ((frappe.db.db_type == 'mariadb' and e.args[0] == DATA_TOO_LONG) or
+			(frappe.db.db_type == 'postgres' and e.pgcode == STRING_DATA_RIGHT_TRUNCATION)):
+			frappe.throw("Most probably your password is too long.", exc=e)
+		raise e
 
 def check_password(user, pwd, doctype='User', fieldname='password'):
 	'''Checks if user and password are correct, else raises frappe.AuthenticationError'''
@@ -131,10 +139,6 @@ def create_auth_table():
 	frappe.db.create_auth_table()
 
 def encrypt(pwd):
-	if len(pwd) > 127:
-		# encrypting > 127 chars will lead to truncation
-		frappe.throw(_('Password cannot be more than 127 characters long'))
-
 	cipher_suite = Fernet(encode(get_encryption_key()))
 	cipher_text = cstr(cipher_suite.encrypt(encode(pwd)))
 	return cipher_text

From 5c5253e00d8f5f6adc78d1257255b7d288316638 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 16:35:44 +0530
Subject: [PATCH 240/361] feat(Report Builder): Ability to show child table row
 index number

---
 .../js/frappe/views/reports/report_view.js     | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js
index 95a3577698..60dd96064e 100644
--- a/frappe/public/js/frappe/views/reports/report_view.js
+++ b/frappe/public/js/frappe/views/reports/report_view.js
@@ -836,8 +836,15 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 			const child_table_fields = frappe.meta.get_docfields(cdt).filter(standard_fields_filter);
 
 			out[cdt] = child_table_fields;
-		});
 
+			// add index column for child tables
+			out[cdt].push({
+				label: __('Index'),
+				fieldname: 'idx',
+				fieldtype: 'Int',
+				parent: cdt
+			});
+		});
 		return out;
 	}
 
@@ -936,6 +943,15 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 			docfield = this.group_by_control.get_group_by_docfield();
 		}
 
+		// child table index column
+		if (fieldname === 'idx' && doctype !== this.doctype) {
+			docfield = {
+				label: "Index",
+				fieldtype: "Int",
+				parent: doctype,
+			}
+		}
+
 		if (!docfield) {
 			docfield = frappe.model.get_std_field(fieldname, true);
 

From 8f558b56db50cda266f9bafa6cc505e4cb228673 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 16:36:45 +0530
Subject: [PATCH 241/361] fix: check field only if doctype is same

---
 frappe/public/js/frappe/views/reports/report_view.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js
index 60dd96064e..e33746907d 100644
--- a/frappe/public/js/frappe/views/reports/report_view.js
+++ b/frappe/public/js/frappe/views/reports/report_view.js
@@ -864,7 +864,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 				.map(df => ({
 					label: __(df.label),
 					value: df.fieldname,
-					checked: this.fields.find(f => f[0] === df.fieldname)
+					checked: this.fields.find(f => f[0] === df.fieldname && f[1] === this.doctype)
 				}))
 		});
 

From 9717ba3907f8bde956ef5699072d8b48b5a014d9 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 16:45:28 +0530
Subject: [PATCH 242/361] style: missing semicolon

---
 frappe/public/js/frappe/views/reports/report_view.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js
index e33746907d..c7d001ed94 100644
--- a/frappe/public/js/frappe/views/reports/report_view.js
+++ b/frappe/public/js/frappe/views/reports/report_view.js
@@ -949,7 +949,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 				label: "Index",
 				fieldtype: "Int",
 				parent: doctype,
-			}
+			};
 		}
 
 		if (!docfield) {

From a4c5750bd83755db732575b8c40a6a4960d7486c Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 12 Aug 2020 19:26:41 +0530
Subject: [PATCH 243/361] fix: Move message handling to client side instead

---
 .../deleted_document/deleted_document.py      | 23 ++++------------
 .../deleted_document/deleted_document_list.js | 27 ++++++++++++-------
 2 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py
index d8f39ca139..116fc5caf5 100644
--- a/frappe/core/doctype/deleted_document/deleted_document.py
+++ b/frappe/core/doctype/deleted_document/deleted_document.py
@@ -62,21 +62,8 @@ def bulk_restore(docnames):
 			failed.append(d)
 			frappe.db.rollback()
 
-	if failed or invalid:
-		def body(docnames):
-			href = "<li><a href='/desk#Form/Deleted Document/{0}'>{0}</a></li>"
-			return "<br><ul>" + "".join([href.format(docname) for docname in docnames])
-
-		def message(title, docnames):
-			if docnames:
-				return _(title) + body(docnames) + "</ul>"
-			return ""
-
-		restored_summary = message("Documents restored successfully", restored)
-		invalid_summary = message("Documents that were already Restored", invalid)
-		failed_summary = message("Documents that Failed to Restore", failed)
-
-		summary = restored_summary + invalid_summary + failed_summary
-		frappe.msgprint(summary, title="Document Restoration Summary", indicator="orange", is_minimizable=True)
-
-	return restored
+	return {
+		"restored": restored,
+		"invalid": invalid,
+		"failed": failed
+	}
diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index ac21d0ff42..3ab347b34d 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -11,17 +11,24 @@ frappe.listview_settings["Deleted Document"] = {
 					args: { "docnames": docnames },
 					callback: function (r) {
 						if (r.message) {
-							let num = r.message.length;
-							let message;
-							if (num === 0) {
-								message = __("No Documents were Restored");
-							} else if (num === 1) {
-								message = __("Document was Restored");
-							} else {
-								message = __("{0} Documents were Restored", [num]);
+							function body(docnames) {
+								const href = "<li><a href='/desk#Form/Deleted Document/%(0)s'>%(0)s</a></li>";
+								const html = docnames.map(docname => { return repl(href, {'0': docname}) });
+								return "<br><ul>" + html.join("");
 							}
-							frappe.msgprint(message);
-							if (num > 0) {
+							function message(title, docnames) {
+								return (docnames.length > 0) ? __(title) + body(docnames) + "</ul>": "";
+							}
+
+							let { restored, invalid, failed } = r.message;
+							let restored_summary = message("Documents restored successfully", restored);
+							let invalid_summary = message("Documents that were already Restored", invalid);
+							let failed_summary = message("Documents that Failed to Restore", failed);
+							let summary = restored_summary + invalid_summary + failed_summary;
+
+							frappe.msgprint(summary, "Document Restoration Summary", true);
+
+							if (restored.length > 0) {
 								doclist.refresh();
 							}
 						}

From 89492d26ab8eee1de3dad9b4390d50af46a9a987 Mon Sep 17 00:00:00 2001
From: Abhishek Balam <abhishekbalam96@gmail.com>
Date: Wed, 12 Aug 2020 21:37:49 +0530
Subject: [PATCH 244/361] fix: add option for python in code fieldtype

---
 frappe/public/js/frappe/form/controls/code.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js
index 726a83db72..4feacd4a03 100644
--- a/frappe/public/js/frappe/form/controls/code.js
+++ b/frappe/public/js/frappe/form/controls/code.js
@@ -46,6 +46,8 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
 		const language_map = {
 			'Javascript': 'ace/mode/javascript',
 			'JS': 'ace/mode/javascript',
+			'Python': 'ace/mode/python',
+			'Py': 'ace/mode/python',
 			'HTML': 'ace/mode/html',
 			'CSS': 'ace/mode/css',
 			'Markdown': 'ace/mode/markdown',

From c9d89a33eae63c3b077a893c6c58ac999e1c38a6 Mon Sep 17 00:00:00 2001
From: prssanna <prssud@gmail.com>
Date: Wed, 12 Aug 2020 15:25:14 +0530
Subject: [PATCH 245/361] fix: set form indicator in disabled form if status is
 not saved

---
 frappe/public/js/frappe/form/toolbar.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js
index be69775b26..ab62723161 100644
--- a/frappe/public/js/frappe/form/toolbar.js
+++ b/frappe/public/js/frappe/form/toolbar.js
@@ -185,10 +185,10 @@ frappe.ui.form.Toolbar = Class.extend({
 		return this.page.add_dropdown(label);
 	},
 	set_indicator: function() {
-		// if(this.frm.save_disabled)
-		// 	return;
-
 		var indicator = frappe.get_indicator(this.frm.doc);
+		if (this.frm.save_disabled && [__('Saved'), __('Not Saved')].includes(indicator[0])) {
+			return;
+		}
 		if(indicator) {
 			this.page.set_indicator(indicator[0], indicator[1]);
 		} else {

From 46ac0735b3e4c0c71f4bcd9dd4a591f16777e525 Mon Sep 17 00:00:00 2001
From: Glen Whitney <glen@studioinfinity.org>
Date: Sun, 9 Aug 2020 03:12:25 +0000
Subject: [PATCH 246/361] fix(Data Import): Allow parent with differing child
 table lengths

  Prior to this commit, if a parent has multiple child tables and
  the child tables have different numbers of rows, then the
  data import will fail with an error when it detects missing
  mandatory fields in the (necessary) empty records in the
  shorter child table.

  This commit fixes the problem by ignoring child rows that have
  only INVALID_VALUES for fields relating to that child (e.g.,
  that are altogether empty in the fields for that child).

  Fixes #11222.
---
 frappe/core/doctype/data_import/importer.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 485f7caf08..161ab5d9a0 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -465,6 +465,7 @@ class ImportFile:
 
 				if doctype != self.doctype and table_df:
 					child_doc = row.parse_doc(doctype, parent_doc, table_df)
+					if child_doc is None: continue
 					parent_doc[table_df.fieldname] = parent_doc.get(table_df.fieldname, [])
 					parent_doc[table_df.fieldname].append(child_doc)
 
@@ -589,17 +590,22 @@ class Row:
 		for key in frappe.model.default_fields + ("__islocal",):
 			doc.pop(key, None)
 
+		record_is_empty = True
 		for col, value in zip(columns, values):
 			df = col.df
 			if value in INVALID_VALUES:
 				value = None
 
 			if value is not None:
+				record_is_empty = False
 				value = self.validate_value(value, col)
 
 			if value is not None:
 				doc[df.fieldname] = self.parse_value(value, col)
 
+		if record_is_empty:
+			return None
+
 		is_table = frappe.get_meta(doctype).istable
 		is_update = self.import_type == UPDATE
 		if is_table and is_update:

From 2ba30a91ce3c265d46d406348647af14921c2e5e Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 13 Aug 2020 12:12:08 +0530
Subject: [PATCH 247/361] feat: set samesite noen for mobile

---
 frappe/auth.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/frappe/auth.py b/frappe/auth.py
index 998e97fe24..7902469c14 100644
--- a/frappe/auth.py
+++ b/frappe/auth.py
@@ -340,6 +340,11 @@ class CookieManager:
 	def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"):
 		if not secure and hasattr(frappe.local, 'request'):
 			secure = frappe.local.request.scheme == "https"
+
+		# Cordova does not work with Lax
+		if frappe.local.session.data.device == "mobile":
+			samesite = None
+
 		self.cookies[key] = {
 			"value": value,
 			"expires": expires,

From 0e380510c5a36ea6a2075a98fd1f7a2711135384 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 13 Aug 2020 13:57:07 +0530
Subject: [PATCH 248/361] feat: ignore add total when syncing fixtures

---
 frappe/modules/import_file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py
index 27649b8da9..5970eae5ca 100644
--- a/frappe/modules/import_file.py
+++ b/frappe/modules/import_file.py
@@ -9,7 +9,7 @@ from frappe.utils import get_datetime_str
 from frappe.model.base_document import get_controller
 
 ignore_values = {
-	"Report": ["disabled", "prepared_report"],
+	"Report": ["disabled", "prepared_report", "add_total_row"],
 	"Print Format": ["disabled"],
 	"Notification": ["enabled"],
 	"Print Style": ["disabled"],

From e8691b4d545cf78fda08a61a47a33e0a13259911 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Thu, 13 Aug 2020 14:36:07 +0530
Subject: [PATCH 249/361] feat: allow get_print and attach print in server
 scripts

---
 frappe/utils/safe_exec.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index d97e770d88..5c3ee6e853 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -71,6 +71,8 @@ def get_safe_globals():
 			msgprint=frappe.msgprint,
 			throw=frappe.throw,
 			sendmail = frappe.sendmail,
+			get_print = frappe.get_print,
+			attach_print = frappe.attach_print,
 
 			user=user,
 			get_fullname=frappe.utils.get_fullname,

From 08d0552ef714adde01d48b6fd0c169a1c816d42b Mon Sep 17 00:00:00 2001
From: Abhishek Balam <abhishekbalam96@gmail.com>
Date: Thu, 13 Aug 2020 14:38:57 +0530
Subject: [PATCH 250/361] fix: show language error only when language is set in
 options

---
 frappe/public/js/frappe/form/controls/code.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js
index 4feacd4a03..a547cfcf32 100644
--- a/frappe/public/js/frappe/form/controls/code.js
+++ b/frappe/public/js/frappe/form/controls/code.js
@@ -59,7 +59,7 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
 		const language = this.df.options;
 
 		const valid_languages = Object.keys(language_map);
-		if (!valid_languages.includes(language)) {
+		if (language && !valid_languages.includes(language)) {
 			// eslint-disable-next-line
 			console.warn(`Invalid language option provided for field "${this.df.label}". Valid options are ${valid_languages.join(', ')}.`);
 		}

From 2699160853383a4c2e9c01b0fbe1b7cd93927dec Mon Sep 17 00:00:00 2001
From: gavin <gavin18d@gmail.com>
Date: Thu, 13 Aug 2020 16:17:21 +0530
Subject: [PATCH 251/361] style: few word do trick

why waste time say lot word when few word do trick

Co-authored-by: Faris Ansari <netchampfaris@users.noreply.github.com>
---
 .../core/doctype/deleted_document/deleted_document_list.js  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index 3ab347b34d..542228eeac 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -3,12 +3,10 @@ frappe.listview_settings["Deleted Document"] = {
 		const action = () => {
 			const selected_docs = doclist.get_checked_items();
 			if (selected_docs.length > 0) {
-				let docnames = selected_docs.map((doc) => {
-					return doc.name;
-				});
+				let docnames = selected_docs.map(doc => doc.name);
 				frappe.call({
 					method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore",
-					args: { "docnames": docnames },
+					args: { docnames },
 					callback: function (r) {
 						if (r.message) {
 							function body(docnames) {

From f06dc0f7d9f4f79c601c1c783d6fb36cc9f1e45d Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 13 Aug 2020 16:29:31 +0530
Subject: [PATCH 252/361] fix: Literal strings for translation

* Used const instead of let
* Used template strings instead of repl
---
 .../deleted_document/deleted_document_list.js   | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index 542228eeac..d639ecb631 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -10,19 +10,20 @@ frappe.listview_settings["Deleted Document"] = {
 					callback: function (r) {
 						if (r.message) {
 							function body(docnames) {
-								const href = "<li><a href='/desk#Form/Deleted Document/%(0)s'>%(0)s</a></li>";
-								const html = docnames.map(docname => { return repl(href, {'0': docname}) });
+								const html = docnames.map(docname => {
+									return `<li><a href='/desk#Form/Deleted Document/${docname}'>${docname}</a></li>`;
+								});
 								return "<br><ul>" + html.join("");
 							}
 							function message(title, docnames) {
-								return (docnames.length > 0) ? __(title) + body(docnames) + "</ul>": "";
+								return (docnames.length > 0) ? title + body(docnames) + "</ul>": "";
 							}
 
-							let { restored, invalid, failed } = r.message;
-							let restored_summary = message("Documents restored successfully", restored);
-							let invalid_summary = message("Documents that were already Restored", invalid);
-							let failed_summary = message("Documents that Failed to Restore", failed);
-							let summary = restored_summary + invalid_summary + failed_summary;
+							const { restored, invalid, failed } = r.message;
+							const restored_summary = message(__("Documents restored successfully"), restored);
+							const invalid_summary = message(__("Documents that were already Restored"), invalid);
+							const failed_summary = message(__("Documents that Failed to Restore"), failed);
+							const summary = restored_summary + invalid_summary + failed_summary;
 
 							frappe.msgprint(summary, "Document Restoration Summary", true);
 

From b7e6f85a85a823900002821f04a6b024d28ebb44 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchampfaris@users.noreply.github.com>
Date: Thu, 13 Aug 2020 16:44:56 +0530
Subject: [PATCH 253/361] fix: convert to sentence case

---
 .../core/doctype/deleted_document/deleted_document_list.js  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js
index d639ecb631..f5e1147dfb 100644
--- a/frappe/core/doctype/deleted_document/deleted_document_list.js
+++ b/frappe/core/doctype/deleted_document/deleted_document_list.js
@@ -21,11 +21,11 @@ frappe.listview_settings["Deleted Document"] = {
 
 							const { restored, invalid, failed } = r.message;
 							const restored_summary = message(__("Documents restored successfully"), restored);
-							const invalid_summary = message(__("Documents that were already Restored"), invalid);
-							const failed_summary = message(__("Documents that Failed to Restore"), failed);
+							const invalid_summary = message(__("Documents that were already restored"), invalid);
+							const failed_summary = message(__("Documents that failed to restore"), failed);
 							const summary = restored_summary + invalid_summary + failed_summary;
 
-							frappe.msgprint(summary, "Document Restoration Summary", true);
+							frappe.msgprint(summary, __("Document Restoration Summary"), true);
 
 							if (restored.length > 0) {
 								doclist.refresh();

From b49571f9b354451815f17605c16709aa004785a5 Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Thu, 13 Aug 2020 17:53:09 +0530
Subject: [PATCH 254/361] feat(notification): Add WhatsApp/SMS channels (#9623)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/core/doctype/role/role.py              |  22 +-
 .../core/doctype/sms_settings/sms_settings.py |   3 +
 .../doctype/notification/notification.js      | 163 +++++++++---
 .../doctype/notification/notification.json    |  32 ++-
 .../doctype/notification/notification.py      |  67 ++++-
 .../doctype/notification/test_notification.py |   4 +-
 .../doctype/notification/test_records.json    |  12 +-
 .../notification_recipient.json               | 246 ++++--------------
 .../doctype/twilio_number_group/__init__.py   |   0
 .../twilio_number_group.json                  |  32 +++
 .../twilio_number_group.py                    |  10 +
 .../doctype/twilio_settings/__init__.py       |   0
 .../twilio_settings/test_twilio_settings.py   |  10 +
 .../twilio_settings/twilio_settings.js        |   8 +
 .../twilio_settings/twilio_settings.json      |  57 ++++
 .../twilio_settings/twilio_settings.py        |  51 ++++
 frappe/patches.txt                            |   1 +
 .../v13_0/rename_notification_fields.py       |  16 ++
 requirements.txt                              |   3 +-
 19 files changed, 458 insertions(+), 279 deletions(-)
 create mode 100644 frappe/integrations/doctype/twilio_number_group/__init__.py
 create mode 100644 frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
 create mode 100644 frappe/integrations/doctype/twilio_number_group/twilio_number_group.py
 create mode 100644 frappe/integrations/doctype/twilio_settings/__init__.py
 create mode 100644 frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
 create mode 100644 frappe/integrations/doctype/twilio_settings/twilio_settings.js
 create mode 100644 frappe/integrations/doctype/twilio_settings/twilio_settings.json
 create mode 100644 frappe/integrations/doctype/twilio_settings/twilio_settings.py
 create mode 100644 frappe/patches/v13_0/rename_notification_fields.py

diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py
index 657340ec24..e458b401e4 100644
--- a/frappe/core/doctype/role/role.py
+++ b/frappe/core/doctype/role/role.py
@@ -33,16 +33,22 @@ class Role(Document):
 				if user_type != user.user_type:
 					user.save()
 
-# Get email addresses of all users that have been assigned this role
-def get_emails_from_role(role):
-	emails = []
 
-	for user in get_users(role):
-		user_email, enabled = frappe.db.get_value("User", user, ["email", "enabled"])
-		if enabled and user_email not in ["admin@example.com", "guest@example.com"]:
-			emails.append(user_email)
+def get_info_based_on_role(role, field='email'):
+	''' Get information of all users that have been assigned this role '''
+	users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"},
+		fields=["parent"])
 
-	return emails
+	return get_user_info(users, field)
+
+def get_user_info(users, field='email'):
+	''' Fetch details about users for the specified field '''
+	info_list = []
+	for user in users:
+		user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"])
+		if enabled and user_info not in ["admin@example.com", "guest@example.com"]:
+			info_list.append(user_info)
+	return info_list
 
 def get_users(role):
 	return [d.parent for d in frappe.get_all("Has Role", filters={"role": role, "parenttype": "User"},
diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py
index f6134e045a..ac835108c1 100644
--- a/frappe/core/doctype/sms_settings/sms_settings.py
+++ b/frappe/core/doctype/sms_settings/sms_settings.py
@@ -18,6 +18,9 @@ class SMSSettings(Document):
 def validate_receiver_nos(receiver_list):
 	validated_receiver_list = []
 	for d in receiver_list:
+		if not d:
+			break
+
 		# remove invalid character
 		for x in [' ','-', '(', ')']:
 			d = d.replace(x, '')
diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js
index 02fc8512ca..454514f922 100644
--- a/frappe/email/doctype/notification/notification.js
+++ b/frappe/email/doctype/notification/notification.js
@@ -3,104 +3,175 @@
 
 this.frm.add_fetch('sender', 'email_id', 'sender_email');
 
-this.frm.fields_dict.sender.get_query = function(){
+this.frm.fields_dict.sender.get_query = function() {
 	return {
 		filters: {
-			'enable_outgoing': 1
+			enable_outgoing: 1
 		}
-	}
+	};
 };
 
 frappe.notification = {
 	setup_fieldname_select: function(frm) {
 		// get the doctype to update fields
-		if(!frm.doc.document_type) {
+		if (!frm.doc.document_type) {
 			return;
 		}
 
 		frappe.model.with_doctype(frm.doc.document_type, function() {
 			let get_select_options = function(df) {
-				return {value: df.fieldname, label: df.fieldname + " (" + __(df.label) + ")"};
-			}
+				return {
+					value: df.fieldname,
+					label: df.fieldname + ' (' + __(df.label) + ')'
+				};
+			};
 
 			let get_date_change_options = function() {
 				let date_options = $.map(fields, function(d) {
-					return (d.fieldtype=="Date" || d.fieldtype=="Datetime")?
-						get_select_options(d) : null;
+					return d.fieldtype == 'Date' || d.fieldtype == 'Datetime'
+						? get_select_options(d)
+						: null;
 				});
 				// append creation and modified date to Date Change field
 				return date_options.concat([
-					{ value: "creation", label: `creation (${__('Created On')})` },
-					{ value: "modified", label: `modified (${__('Last Modified Date')})` }
+					{ value: 'creation', label: `creation (${__('Created On')})` },
+					{ value: 'modified', label: `modified (${__('Last Modified Date')})` }
 				]);
-			}
+			};
 
-			let fields = frappe.get_doc("DocType", frm.doc.document_type).fields;
-			let options = $.map(fields,
-				function(d) { return in_list(frappe.model.no_value_type, d.fieldtype) ?
-					null : get_select_options(d); });
+			let fields = frappe.get_doc('DocType', frm.doc.document_type).fields;
+			let options = $.map(fields, function(d) {
+				return in_list(frappe.model.no_value_type, d.fieldtype)
+					? null : get_select_options(d);
+			});
 
 			// set value changed options
-			frm.set_df_property("value_changed", "options", [""].concat(options));
-			frm.set_df_property("set_property_after_alert", "options", [""].concat(options));
+			frm.set_df_property('value_changed', 'options', [''].concat(options));
+			frm.set_df_property(
+				'set_property_after_alert',
+				'options',
+				[''].concat(options)
+			);
 
 			// set date changed options
-			frm.set_df_property("date_changed", "options", get_date_change_options());
+			frm.set_df_property('date_changed', 'options', get_date_change_options());
 
-			let email_fields = $.map(fields,
-				function(d) { return (d.options == "Email" ||
-					(d.options=='User' && d.fieldtype=='Link')) ?
-					get_select_options(d) : null; });
+			let receiver_fields = [];
+			if (frm.doc.channel === 'Email') {
+				receiver_fields = $.map(fields, function(d) {
+					return d.options == 'Email' ||
+						(d.options == 'User' && d.fieldtype == 'Link')
+						? get_select_options(d) : null;
+				});
+			} else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) {
+				receiver_fields = $.map(fields, function(d) {
+					return d.options == 'Phone' ? get_select_options(d) : null;
+				});
+			}
 
 			// set email recipient options
-			frappe.meta.get_docfield("Notification Recipient", "email_by_document_field",
+			frappe.meta.get_docfield(
+				'Notification Recipient',
+				'receiver_by_document_field',
 				// set first option as blank to allow notification not to be defaulted to the owner
-				frm.doc.name).options = [""].concat(["owner"].concat(email_fields));
+				frm.doc.name
+			).options = [''].concat(["owner"]).concat(receiver_fields);
 
 			frm.fields_dict.recipients.grid.refresh();
 		});
-	}
-}
+	},
+	setup_example_message: function(frm) {
+		let template = '';
+		if (frm.doc.channel === 'WhatsApp') {
+			template = `<h5 style='display: inline-block'>Warning:</h5> Only Use Pre-Approved WhatsApp for Business Template
+<h5>Message Example</h5>
 
-frappe.ui.form.on("Notification", {
+<pre>
+Your {{ doc.name }} order of {{ doc.total }} has shipped and should be delivered on {{ doc.date }}. Details : {{doc.customer}}
+</pre>`;
+		} else if (frm.doc.channel === 'Email') {
+			template = `<h5>Message Example</h5>
+
+<pre>&lt;h3&gt;Order Overdue&lt;/h3&gt;
+
+&lt;p&gt;Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.&lt;/p&gt;
+
+&lt;!-- show last comment --&gt;
+{% if comments %}
+Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
+{% endif %}
+
+&lt;h4&gt;Details&lt;/h4&gt;
+
+&lt;ul&gt;
+&lt;li&gt;Customer: {{ doc.customer }}
+&lt;li&gt;Amount: {{ doc.grand_total }}
+&lt;/ul&gt;
+</pre>
+			`;
+		} else {
+			template = `<h5>Message Example</h5>
+
+<pre>*Order Overdue*
+
+Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.
+
+<!-- show last comment -->
+{% if comments %}
+Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
+{% endif %}
+
+*Details*
+
+• Customer: {{ doc.customer }}
+• Amount: {{ doc.grand_total }}
+</pre>`;
+		}
+		frm.set_df_property('message_examples', 'options', template);
+
+	}
+};
+
+frappe.ui.form.on('Notification', {
 	onload: function(frm) {
-		frm.set_query("document_type", function() {
+		frm.set_query('document_type', function() {
 			return {
-				"filters": {
-					"istable": 0
+				filters: {
+					istable: 0
 				}
-			}
+			};
 		});
-		frm.set_query("print_format", function() {
+		frm.set_query('print_format', function() {
 			return {
-				"filters": {
-					"doc_type": frm.doc.document_type
+				filters: {
+					doc_type: frm.doc.document_type
 				}
-			}
+			};
 		});
 	},
 	refresh: function(frm) {
 		frappe.notification.setup_fieldname_select(frm);
-		frm.get_field("is_standard").toggle(frappe.boot.developer_mode);
+		frm.get_field('is_standard').toggle(frappe.boot.developer_mode);
 		frm.trigger('event');
 	},
 	document_type: function(frm) {
 		frappe.notification.setup_fieldname_select(frm);
 	},
 	view_properties: function(frm) {
-		frappe.route_options = {doc_type:frm.doc.document_type};
-		frappe.set_route("Form", "Customize Form");
+		frappe.route_options = { doc_type: frm.doc.document_type };
+		frappe.set_route('Form', 'Customize Form');
 	},
 	event: function(frm) {
-		if(in_list(['Days Before', 'Days After'], frm.doc.event)) {
+		if (in_list(['Days Before', 'Days After'], frm.doc.event)) {
 			frm.add_custom_button(__('Get Alerts for Today'), function() {
 				frappe.call({
-					method: 'frappe.email.doctype.notification.notification.get_documents_for_today',
+					method:
+						'frappe.email.doctype.notification.notification.get_documents_for_today',
 					args: {
 						notification: frm.doc.name
 					},
 					callback: function(r) {
-						if(r.message) {
+						if (r.message) {
 							frappe.msgprint(r.message);
 						} else {
 							frappe.msgprint(__('No alerts for today'));
@@ -111,6 +182,14 @@ frappe.ui.form.on("Notification", {
 		}
 	},
 	channel: function(frm) {
-		frm.toggle_reqd("recipients", frm.doc.channel=="Email");
+		frm.toggle_reqd('recipients', frm.doc.channel == 'Email');
+		frappe.notification.setup_fieldname_select(frm);
+		frappe.notification.setup_example_message(frm);
+		if (frm.doc.channel === 'SMS' && frm.doc.__islocal) {
+			frm.set_df_property('channel',
+				'description', `To use SMS Channel, initialize <a href="#Form/SMS Settings">SMS Settings</a>.`);
+		} else {
+			frm.set_df_property('channel', 'description', ` `);
+		}
 	}
 });
diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json
index 932f0491a9..95f218ad73 100644
--- a/frappe/email/doctype/notification/notification.json
+++ b/frappe/email/doctype/notification/notification.json
@@ -10,6 +10,7 @@
   "enabled",
   "column_break_2",
   "channel",
+  "twilio_number",
   "slack_webhook_url",
   "filters",
   "subject",
@@ -37,7 +38,6 @@
   "message_sb",
   "message",
   "message_examples",
-  "slack_message_examples",
   "view_properties",
   "column_break_25",
   "attach_print",
@@ -60,11 +60,13 @@
    "fieldname": "channel",
    "fieldtype": "Select",
    "label": "Channel",
-   "options": "Email\nSlack\nSystem Notification",
-   "reqd": 1
+   "options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS",
+   "reqd": 1,
+   "set_only_once": 1
   },
   {
    "depends_on": "eval:doc.channel=='Slack'",
+   "description": "To use Slack Channel, add a <a href=\"\\#Form/Slack Webhook URL\">Slack Webhook URL</a>.",
    "fieldname": "slack_webhook_url",
    "fieldtype": "Link",
    "label": "Slack Channel",
@@ -77,13 +79,14 @@
    "label": "Filters"
   },
   {
+   "depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)",
    "description": "To add dynamic subject, use jinja tags like\n\n<div><pre><code>{{ doc.name }} Delivered</code></pre></div>",
    "fieldname": "subject",
    "fieldtype": "Data",
    "ignore_xss_filter": 1,
    "in_list_view": 1,
    "label": "Subject",
-   "reqd": 1
+   "mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)"
   },
   {
    "fieldname": "document_type",
@@ -153,6 +156,7 @@
    "label": "Value Changed"
   },
   {
+   "depends_on": "eval: doc.channel == 'Email'",
    "fieldname": "sender",
    "fieldtype": "Link",
    "label": "Sender",
@@ -203,7 +207,7 @@
    "label": "Value To Be Set"
   },
   {
-   "depends_on": "eval:doc.channel!=='Slack'",
+   "depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)",
    "fieldname": "column_break_5",
    "fieldtype": "Section Break",
    "label": "Recipients"
@@ -228,19 +232,11 @@
    "label": "Message"
   },
   {
-   "depends_on": "eval:doc.channel=='Email'",
    "fieldname": "message_examples",
    "fieldtype": "HTML",
    "label": "Message Examples",
    "options": "<h5>Message Example</h5>\n\n<pre>&lt;h3&gt;Order Overdue&lt;/h3&gt;\n\n&lt;p&gt;Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.&lt;/p&gt;\n\n&lt;!-- show last comment --&gt;\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n&lt;h4&gt;Details&lt;/h4&gt;\n\n&lt;ul&gt;\n&lt;li&gt;Customer: {{ doc.customer }}\n&lt;li&gt;Amount: {{ doc.grand_total }}\n&lt;/ul&gt;\n</pre>"
   },
-  {
-   "depends_on": "eval:doc.channel=='Slack'",
-   "fieldname": "slack_message_examples",
-   "fieldtype": "HTML",
-   "label": "Message Examples",
-   "options": "<h5>Message Example</h5>\n\n<pre>*Order Overdue*\n\nTransaction {{ doc.name }} has exceeded Due Date. Please take necessary action.\n\n<!-- show last comment -->\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n*Details*\n\n\u2022 Customer: {{ doc.customer }}\n\u2022 Amount: {{ doc.grand_total }}\n</pre>"
-  },
   {
    "fieldname": "view_properties",
    "fieldtype": "Button",
@@ -266,6 +262,14 @@
    "label": "Print Format",
    "options": "Print Format"
   },
+  {
+   "depends_on": "eval: doc.channel==='WhatsApp'",
+   "description": "To use WhatsApp for Business, initialize <a href=\"#Form/Twilio Settings\">Twilio Settings</a>.",
+   "fieldname": "twilio_number",
+   "fieldtype": "Link",
+   "label": "Twilio Number",
+   "options": "Twilio Number Group"
+  },
   {
    "default": "0",
    "depends_on": "eval: doc.channel !== 'System Notification'",
@@ -277,7 +281,7 @@
  ],
  "icon": "fa fa-envelope",
  "links": [],
- "modified": "2020-06-23 14:01:25.462544",
+ "modified": "2020-08-11 19:24:35.479373",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",
diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py
index 81670756f6..2ec208c89d 100644
--- a/frappe/email/doctype/notification/notification.py
+++ b/frappe/email/doctype/notification/notification.py
@@ -7,12 +7,14 @@ import frappe
 import json, os
 from frappe import _
 from frappe.model.document import Document
-from frappe.core.doctype.role.role import get_emails_from_role
+from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info
 from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date
 from frappe.utils.jinja import validate_template
 from frappe.modules.utils import export_module_json, get_doc_module
 from six import string_types
 from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
+from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message
+from frappe.core.doctype.sms_settings.sms_settings import send_sms
 from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
 
 class Notification(Document):
@@ -26,7 +28,9 @@ class Notification(Document):
 			self.name = self.subject
 
 	def validate(self):
-		validate_template(self.subject)
+		if self.channel not in ('WhatsApp', 'SMS'):
+			validate_template(self.subject)
+
 		validate_template(self.message)
 
 		if self.event in ("Days Before", "Days After") and not self.date_changed:
@@ -126,8 +130,15 @@ def get_context(context):
 			if self.channel == 'Slack':
 				self.send_a_slack_msg(doc, context)
 
+			if self.channel == 'WhatsApp':
+				self.send_whatsapp_msg(doc, context)
+
+			if self.channel == 'SMS':
+				self.send_sms(doc, context)
+
 			if self.channel == 'System Notification' or self.send_system_notification:
 				self.create_system_notification(doc, context)
+
 		except:
 			frappe.log_error(title='Failed to send notification', message=frappe.get_traceback())
 
@@ -195,11 +206,24 @@ def get_context(context):
 				and attachments[0].get('print_letterhead')) or False))
 
 	def send_a_slack_msg(self, doc, context):
-			send_slack_message(
-				webhook_url=self.slack_webhook_url,
-				message=frappe.render_template(self.message, context),
-				reference_doctype = doc.doctype,
-				reference_name = doc.name)
+		send_slack_message(
+			webhook_url=self.slack_webhook_url,
+			message=frappe.render_template(self.message, context),
+			reference_doctype=doc.doctype,
+			reference_name=doc.name)
+
+	def send_whatsapp_msg(self, doc, context):
+		send_whatsapp_message(
+			sender=self.twilio_number,
+			receiver_list=self.get_receiver_list(doc, context),
+			message=frappe.render_template(self.message, context),
+		)
+
+	def send_sms(self, doc, context):
+		send_sms(
+			receiver_list=self.get_receiver_list(doc, context),
+			msg=frappe.render_template(self.message, context)
+		)
 
 	def get_list_of_recipients(self, doc, context):
 		recipients = []
@@ -209,8 +233,8 @@ def get_context(context):
 			if recipient.condition:
 				if not frappe.safe_eval(recipient.condition, None, context):
 					continue
-			if recipient.email_by_document_field:
-				email_ids_value = doc.get(recipient.email_by_document_field)
+			if recipient.receiver_by_document_field:
+				email_ids_value = doc.get(recipient.receiver_by_document_field)
 				if validate_email_address(email_ids_value):
 					email_ids = email_ids_value.replace(",", "\n")
 					recipients = recipients + email_ids.split("\n")
@@ -232,8 +256,8 @@ def get_context(context):
 				bcc = bcc + recipient.bcc.split("\n")
 
 			#For sending emails to specified role
-			if recipient.email_by_role:
-				emails = get_emails_from_role(recipient.email_by_role)
+			if recipient.receiver_by_role:
+				emails = get_info_based_on_role(recipient.receiver_by_role, 'email')
 
 				for email in emails:
 					recipients = recipients + email.split("\n")
@@ -242,6 +266,27 @@ def get_context(context):
 			return None, None, None
 		return list(set(recipients)), list(set(cc)), list(set(bcc))
 
+	def get_receiver_list(self, doc, context):
+		''' return receiver list based on the doc field and role specified '''
+		receiver_list = []
+		for recipient in self.recipients:
+			if recipient.condition:
+				if not frappe.safe_eval(recipient.condition, None, context):
+					continue
+
+			# For sending messages to the owner's mobile phone number
+			if recipient.receiver_by_document_field == 'owner':
+				receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no'))
+			# For sending messages to the number specified in the receiver field
+			elif recipient.receiver_by_document_field:
+				receiver_list.append(doc.get(recipient.receiver_by_document_field))
+
+			#For sending messages to specified role
+			if recipient.receiver_by_role:
+				receiver_list += get_info_based_on_role(recipient.receiver_by_role, 'mobile_no')
+
+		return receiver_list
+
 	def get_attachment(self, doc):
 		""" check print settings are attach the pdf """
 		if not self.attach_print:
diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py
index b9bbde172d..9bdf09375d 100644
--- a/frappe/email/doctype/notification/test_notification.py
+++ b/frappe/email/doctype/notification/test_notification.py
@@ -63,7 +63,7 @@ class TestNotification(unittest.TestCase):
 		notification.message = "test"
 
 		recipent = frappe.new_doc("Notification Recipient")
-		recipent.email_by_document_field = "owner"
+		recipent.receiver_by_document_field = "owner"
 
 		notification.recipents = recipent
 		notification.condition = "test"
@@ -105,7 +105,7 @@ class TestNotification(unittest.TestCase):
 			"value_changed": "description1",
 			"message": "Description changed",
 			"recipients": [
-				{ "email_by_document_field": "owner" }
+				{ "receiver_by_document_field": "owner" }
 			]
 		}).insert()
 		frappe.db.commit()
diff --git a/frappe/email/doctype/notification/test_records.json b/frappe/email/doctype/notification/test_records.json
index 865b2ac021..665f800c0f 100644
--- a/frappe/email/doctype/notification/test_records.json
+++ b/frappe/email/doctype/notification/test_records.json
@@ -8,7 +8,7 @@
 		"message": "New comment {{ doc.content }} created",
 		"condition": "doc.communication_type=='Comment'",
 		"recipients": [
-			{ "email_by_document_field": "owner" }
+			{ "receiver_by_document_field": "owner" }
 		]
 	},
 	{
@@ -20,7 +20,7 @@
 		"message": "New comment {{ doc.content }} saved",
 		"condition": "doc.communication_type=='Comment'",
 		"recipients": [
-			{ "email_by_document_field": "owner" }
+			{ "receiver_by_document_field": "owner" }
 		],
 		"set_property_after_alert": "subject",
 		"property_value": "__testing__"
@@ -34,7 +34,7 @@
 		"condition": "doc.event_type=='Public'",
 		"message": "A new public event {{ doc.subject }} on {{ doc.starts_on }} is created",
 		"recipients": [
-			{ "email_by_document_field": "owner" }
+			{ "receiver_by_document_field": "owner" }
 		]
 	},
 	{
@@ -46,7 +46,7 @@
 		"value_changed": "description",
 		"message": "Description changed",
 		"recipients": [
-			{ "email_by_document_field": "owner" }
+			{ "receiver_by_document_field": "owner" }
 		]
 	},
 	{
@@ -59,7 +59,7 @@
 		"days_in_advance": 2,
 		"message": "Description changed",
 		"recipients": [
-			{ "email_by_document_field": "owner" }
+			{ "receiver_by_document_field": "owner" }
 		]
 	},
 	{
@@ -70,7 +70,7 @@
 		"attach_print": 0,
 		"message": "New user {{ doc.name }} created",
 		"recipients": [
-			{ "email_by_document_field": "owner", "cc": "{{ doc.email }}" }
+			{ "receiver_by_document_field": "owner", "cc": "{{ doc.email }}" }
 		]
 	}
 ]
diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.json b/frappe/email/doctype/notification_recipient/notification_recipient.json
index ec35dccc63..201899cd57 100644
--- a/frappe/email/doctype/notification_recipient/notification_recipient.json
+++ b/frappe/email/doctype/notification_recipient/notification_recipient.json
@@ -1,204 +1,60 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2014-07-11 17:19:37.037109", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2014-07-11 17:19:37.037109",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "receiver_by_document_field",
+  "receiver_by_role",
+  "cc",
+  "bcc",
+  "condition"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "", 
-   "fieldname": "email_by_document_field", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email By Document Field", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
+   "depends_on": "eval:parent.channel=='Email'",
+   "description": "Optional: Always send to these ids. Each Email Address on a new row",
+   "fieldname": "cc",
+   "fieldtype": "Code",
+   "label": "CC"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_by_role", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email By Role", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Role", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
+   "depends_on": "eval:parent.channel=='Email'",
+   "fieldname": "bcc",
+   "fieldtype": "Code",
+   "label": "BCC"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Optional: Always send to these ids. Each Email Address on a new row", 
-   "fieldname": "cc", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "CC", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
+   "description": "Expression, Optional",
+   "fieldname": "condition",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Condition"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "bcc", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "BCC", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "receiver_by_document_field",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Receiver By Document Field"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Expression, Optional", 
-   "fieldname": "condition", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Condition", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
+   "fieldname": "receiver_by_role",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Receiver By Role",
+   "options": "Role"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2018-09-03 18:37:57.043251", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Notification Recipient", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0, 
- "track_views": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-02-21 11:18:40.125233",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Notification Recipient",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC"
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_number_group/__init__.py b/frappe/integrations/doctype/twilio_number_group/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
new file mode 100644
index 0000000000..1790581ca7
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
@@ -0,0 +1,32 @@
+{
+ "actions": [],
+ "autoname": "field:phone_number",
+ "creation": "2020-02-24 13:58:58.036914",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "phone_number"
+ ],
+ "fields": [
+  {
+   "fieldname": "phone_number",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Phone Number",
+   "unique": 1
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-03-02 14:54:34.396254",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Twilio Number Group",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py
new file mode 100644
index 0000000000..04cb9ae146
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class TwilioNumberGroup(Document):
+	pass
diff --git a/frappe/integrations/doctype/twilio_settings/__init__.py b/frappe/integrations/doctype/twilio_settings/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
new file mode 100644
index 0000000000..bcb1368d68
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestTwilioSettings(unittest.TestCase):
+	pass
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.js b/frappe/integrations/doctype/twilio_settings/twilio_settings.js
new file mode 100644
index 0000000000..59ebcf2e7d
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Twilio Settings', {
+	refresh: function(frm) {
+		frm.dashboard.set_headline(__("For more information, {0}.", [`<a href='https://docs.erpnext.com/docs/user/manual/en/setting-up/notifications'>${__('Click here')}</a>`]));
+	}
+});
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.json b/frappe/integrations/doctype/twilio_settings/twilio_settings.json
new file mode 100644
index 0000000000..e54500fd5d
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.json
@@ -0,0 +1,57 @@
+{
+ "actions": [],
+ "creation": "2020-01-28 15:21:44.457163",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "account_sid",
+  "auth_token",
+  "column_break_2",
+  "twilio_number"
+ ],
+ "fields": [
+  {
+   "fieldname": "account_sid",
+   "fieldtype": "Data",
+   "label": "Account SID"
+  },
+  {
+   "fieldname": "auth_token",
+   "fieldtype": "Password",
+   "label": "Auth Token"
+  },
+  {
+   "fieldname": "column_break_2",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "twilio_number",
+   "fieldtype": "Table",
+   "label": "Twilio Number",
+   "options": "Twilio Number Group"
+  }
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-08-11 15:28:57.860554",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Twilio Settings",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.py b/frappe/integrations/doctype/twilio_settings/twilio_settings.py
new file mode 100644
index 0000000000..ba0565b3af
--- /dev/null
+++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+from twilio.rest import Client
+from frappe import _
+from frappe.utils.password import get_decrypted_password
+from six import string_types
+
+class TwilioSettings(Document):
+	pass
+
+def send_whatsapp_message(sender, receiver_list, message):
+	import json
+	if isinstance(receiver_list, string_types):
+		receiver_list = json.loads(receiver_list)
+		if not isinstance(receiver_list, list):
+			receiver_list = [receiver_list]
+
+
+	twilio_settings = frappe.get_doc("Twilio Settings")
+	auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
+	client = Client(twilio_settings.account_sid, auth_token)
+	args = {
+		"from_": 'whatsapp:+{}'.format(sender),
+		"body": message
+	}
+
+	failed_delivery = []
+
+	for rec in receiver_list:
+		args.update({"to": 'whatsapp:{}'.format(rec)})
+		resp = _send_whatsapp(args, client)
+		if not resp or resp.error_message:
+			failed_delivery.append(rec)
+
+	if failed_delivery:
+		frappe.log_error(_("The message wasn't correctly delivered to: {}".format(", ".join(failed_delivery))), _('Delivery Failed'))
+
+
+def _send_whatsapp(message_dict, client):
+	response = frappe._dict()
+	try:
+		response = client.messages.create(**message_dict)
+	except Exception as e:
+		frappe.log_error(e, title = _('Twilio WhatsApp Message Error'))
+
+	return response
\ No newline at end of file
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 8f100c79f1..8ea6d8ae29 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -299,3 +299,4 @@ frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
 frappe.patches.v13_0.add_standard_navbar_items
 frappe.patches.v13_0.generate_theme_files_in_public_folder
 frappe.patches.v13_0.increase_password_length
+frappe.patches.v13_0.rename_notification_fields
diff --git a/frappe/patches/v13_0/rename_notification_fields.py b/frappe/patches/v13_0/rename_notification_fields.py
new file mode 100644
index 0000000000..2984e6503c
--- /dev/null
+++ b/frappe/patches/v13_0/rename_notification_fields.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.utils.rename_field import rename_field
+
+def execute():
+	"""
+		Change notification recipient fields from email to receiver fields
+	"""
+	frappe.reload_doc("Email", "doctype", "Notification Recipient")
+	frappe.reload_doc("Email", "doctype", "Notification")
+
+	rename_field("Notification Recipient", "email_by_document_field", "receiver_by_document_field")
+	rename_field("Notification Recipient", "email_by_role", "receiver_by_role")
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 46e853794e..a88d0e3c1d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -70,4 +70,5 @@ xlrd==1.2.0
 zxcvbn-python==4.4.24
 pycryptodome==3.9.8
 paytmchecksum==1.7.0
-wrapt==1.10.11
\ No newline at end of file
+wrapt==1.10.11
+twilio==6.44.2
\ No newline at end of file

From 6bfd07be51f4c0117a8f9100cced951018628ade Mon Sep 17 00:00:00 2001
From: Anurag Mishra <32095923+Anurag810@users.noreply.github.com>
Date: Fri, 14 Aug 2020 16:12:02 +0530
Subject: [PATCH 255/361] feat: Ignore doctypes for cancel_all (#11034)

---
 frappe/core/doctype/doctype/test_doctype.py | 93 +++++++++++++++++++++
 frappe/desk/form/linked_with.py             | 15 ++--
 frappe/public/js/frappe/form/form.js        | 42 ++++++----
 3 files changed, 130 insertions(+), 20 deletions(-)

diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py
index fe9f88b9b3..00e80ce4e7 100644
--- a/frappe/core/doctype/doctype/test_doctype.py
+++ b/frappe/core/doctype/doctype/test_doctype.py
@@ -376,3 +376,96 @@ class TestDocType(unittest.TestCase):
 		link_doc.delete()
 		doc.delete()
 		frappe.db.commit()
+
+	def test_ignore_cancelation_of_linked_doctype_during_cancell(self):
+		import json
+		from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs
+
+		#create linked doctype
+		link_doc = self.new_doctype('Test Linked Doctype 1')
+		link_doc.is_submittable = 1
+		for data in link_doc.get('permissions'):
+			data.submit = 1
+			data.cancel = 1
+		link_doc.insert()
+
+		#create first parent doctype
+		test_doc_1 = self.new_doctype('Test Doctype 1')
+		test_doc_1.is_submittable = 1
+
+		field_2 = test_doc_1.append('fields', {})
+		field_2.label = 'Test Linked Doctype 1'
+		field_2.fieldname  = 'test_linked_doctype_a'
+		field_2.fieldtype = 'Link'
+		field_2.options = 'Test Linked Doctype 1'
+
+		for data in test_doc_1.get('permissions'):
+			data.submit = 1
+			data.cancel = 1
+		test_doc_1.insert()
+
+		#crete second parent doctype
+		doc = self.new_doctype('Test Doctype 2')
+		doc.is_submittable = 1
+
+		field_2 = doc.append('fields', {})
+		field_2.label = 'Test Linked Doctype 1'
+		field_2.fieldname  = 'test_linked_doctype_a'
+		field_2.fieldtype = 'Link'
+		field_2.options = 'Test Linked Doctype 1'
+
+		for data in link_doc.get('permissions'):
+			data.submit = 1
+			data.cancel = 1
+		doc.insert()
+
+		# create doctype data
+		data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1')
+		data_link_doc_1.some_fieldname = 'Data1'
+		data_link_doc_1.insert()
+		data_link_doc_1.save()
+		data_link_doc_1.submit()
+
+		data_doc_2 = frappe.new_doc('Test Doctype 1')
+		data_doc_2.some_fieldname = 'Data1'
+		data_doc_2.test_linked_doctype_a = data_link_doc_1.name
+		data_doc_2.insert()
+		data_doc_2.save()
+		data_doc_2.submit()
+
+		data_doc = frappe.new_doc('Test Doctype 2')
+		data_doc.some_fieldname = 'Data1'
+		data_doc.test_linked_doctype_a = data_link_doc_1.name
+		data_doc.insert()
+		data_doc.save()
+		data_doc.submit()
+
+		docs = get_submitted_linked_docs(link_doc.name, data_link_doc_1.name)
+		dump_docs = json.dumps(docs.get('docs'))
+
+		cancel_all_linked_docs(dump_docs, ignore_doctypes_on_cancel_all=["Test Doctype 2"])
+
+		# checking that doc for Test Doctype 2 is not canceled
+		self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel)
+
+		data_doc.load_from_db()
+		data_doc_2.load_from_db()
+		self.assertEqual(data_link_doc_1.docstatus, 2)
+
+		#linked doc is canceled
+		self.assertEqual(data_doc_2.docstatus, 2)
+
+		#ignored doctype 2 during cancel
+		self.assertEqual(data_doc.docstatus, 1)
+
+		# delete doctype record
+		data_doc.cancel()
+		data_doc.delete()
+		data_doc_2.delete()
+		data_link_doc_1.delete()
+
+		# delete doctype
+		link_doc.delete()
+		doc.delete()
+		test_doc_1.delete()
+		frappe.db.commit()
diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py
index 5bae49ea95..733ee1774c 100644
--- a/frappe/desk/form/linked_with.py
+++ b/frappe/desk/form/linked_with.py
@@ -11,7 +11,6 @@ from frappe import _
 from frappe.model.meta import is_single
 from frappe.modules import load_doctype_module
 
-
 @frappe.whitelist()
 def get_submitted_linked_docs(doctype, name, docs=None, visited=None):
 	"""
@@ -78,7 +77,7 @@ def get_submitted_linked_docs(doctype, name, docs=None, visited=None):
 
 
 @frappe.whitelist()
-def cancel_all_linked_docs(docs):
+def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]):
 	"""
 	Cancel all linked doctype
 
@@ -87,14 +86,16 @@ def cancel_all_linked_docs(docs):
 	"""
 
 	docs = json.loads(docs)
+	if isinstance(ignore_doctypes_on_cancel_all, string_types):
+		ignore_doctypes_on_cancel_all = json.loads(ignore_doctypes_on_cancel_all)
 	for i, doc in enumerate(docs, 1):
-		if validate_linked_doc(doc) is True:
-			frappe.publish_progress(percent=i * 100 / len(docs), title=_("Cancelling documents"))
+		if validate_linked_doc(doc, ignore_doctypes_on_cancel_all) is True:
+			frappe.publish_progress(percent=i * 100 / ((len(docs) - len(ignore_doctypes_on_cancel_all))), title=_("Cancelling documents"))
 			linked_doc = frappe.get_doc(doc.get("doctype"), doc.get("name"))
 			linked_doc.cancel()
 
 
-def validate_linked_doc(docinfo):
+def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]):
 	"""
 	Validate a document to be submitted and non-exempted from auto-cancel.
 
@@ -105,6 +106,10 @@ def validate_linked_doc(docinfo):
 		bool: True if linked document passes all validations, else False
 	"""
 
+	#ignore doctype to cancel
+	if docinfo.get("doctype") in ignore_doctypes_on_cancel_all:
+		return False
+
 	# skip non-submittable doctypes since they don't need to be cancelled
 	if not frappe.get_meta(docinfo.get('doctype')).is_submittable:
 		return False
diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js
index c0b76ee94d..b4bc758399 100644
--- a/frappe/public/js/frappe/form/form.js
+++ b/frappe/public/js/frappe/form/form.js
@@ -667,22 +667,29 @@ frappe.ui.form.Form = class FrappeForm {
 	savecancel(btn, callback, on_error) {
 		const me = this;
 		this.validate_form_action('Cancel');
-
+		me.ignore_doctypes_on_cancel_all = me.ignore_doctypes_on_cancel_all || [];
 		frappe.call({
 			method: "frappe.desk.form.linked_with.get_submitted_linked_docs",
 			args: {
 				doctype: me.doc.doctype,
 				name: me.doc.name
 			},
-			freeze: true,
-			callback: (r) => {
-				if (!r.exc && r.message.count > 0) {
-					me._cancel_all(r, btn, callback, on_error);
-				} else {
-					me._cancel(btn, callback, on_error, false);
+			freeze: true
+		}).then(r => {
+			if (!r.exc) {
+				let doctypes_to_cancel = (r.message.docs || []).map(value => {
+					return value.doctype;
+				}).filter(value => {
+					return !me.ignore_doctypes_on_cancel_all.includes(value);
+				});
+
+				if (doctypes_to_cancel.length) {
+					return me._cancel_all(r, btn, callback, on_error);
 				}
 			}
-		});
+			return me._cancel(btn, callback, on_error, false);
+		}
+		);
 	}
 
 	_cancel_all(r, btn, callback, on_error) {
@@ -693,12 +700,16 @@ frappe.ui.form.Form = class FrappeForm {
 		let links = r.message.docs;
 		const doctypes = Array.from(new Set(links.map(link => link.doctype)));
 
+		me.ignore_doctypes_on_cancel_all = me.ignore_doctypes_on_cancel_all || [];
+
 		for (let doctype of doctypes) {
-			let docnames = links
-				.filter((link) => link.doctype == doctype)
-				.map((link) => frappe.utils.get_form_link(link.doctype, link.name, true))
-				.join(", ");
-			links_text += `<li><strong>${doctype}</strong>: ${docnames}</li>`;
+			if (!me.ignore_doctypes_on_cancel_all.includes(doctype)) {
+				let docnames = links
+					.filter((link) => link.doctype == doctype)
+					.map((link) => frappe.utils.get_form_link(link.doctype, link.name, true))
+					.join(", ");
+				links_text += `<li><strong>${doctype}</strong>: ${docnames}</li>`;
+			}
 		}
 		links_text = `<ul>${links_text}</ul>`;
 
@@ -728,7 +739,8 @@ frappe.ui.form.Form = class FrappeForm {
 				frappe.call({
 					method: "frappe.desk.form.linked_with.cancel_all_linked_docs",
 					args: {
-						docs: links
+						docs: links,
+						ignore_doctypes_on_cancel_all: me.ignore_doctypes_on_cancel_all || []
 					},
 					freeze: true,
 					callback: (resp) => {
@@ -742,7 +754,7 @@ frappe.ui.form.Form = class FrappeForm {
 		}
 
 		d.show();
-	};
+	}
 
 	_cancel(btn, callback, on_error, skip_confirm) {
 		const me = this;

From 14400fe1d34ce23499028987da63fe63167b2327 Mon Sep 17 00:00:00 2001
From: Mohammad Hasnain Mohsin Rajan <hasnain2808@gmail.com>
Date: Fri, 14 Aug 2020 17:04:09 +0530
Subject: [PATCH 256/361] style: fix typo tood to todo (#11272)

---
 frappe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index a68c32fe03..6b50f8ab28 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -763,7 +763,7 @@ def get_doc(*args, **kwargs):
 
 		# insert a new document
 		todo = frappe.get_doc({"doctype":"ToDo", "description": "test"})
-		tood.insert()
+		todo.insert()
 
 		# open an existing document
 		todo = frappe.get_doc("ToDo", "TD0001")

From c2ea55b0c0c47f9e17359dbfd075db9a20a418df Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 7 Aug 2020 21:58:23 +0530
Subject: [PATCH 257/361] style: Black

---
 frappe/utils/backups.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index 71906f540f..a64ba8c3a7 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -102,7 +102,7 @@ class BackupGenerator:
 			"database": "*-{}-database.sql.gz",
 			"public": "*-{}-files.tar",
 			"private": "*-{}-private-files.tar",
-			"config": "*-{}-site_config_backup.json"
+			"config": "*-{}-site_config_backup.json",
 		}
 
 		def backup_time(file_path):
@@ -111,7 +111,8 @@ class BackupGenerator:
 			return timegm(datetime.strptime(file_timestamp, "%Y%m%d_%H%M%S").utctimetuple())
 
 		def get_latest(file_pattern):
-			file_list = glob(os.path.join(backup_path, file_pattern))
+			file_pattern = os.path.join(backup_path, file_pattern.format(self.site_slug))
+			file_list = glob(file_pattern)
 			if file_list:
 				return max(file_list, key=backup_time)
 
@@ -122,14 +123,20 @@ class BackupGenerator:
 				return file_path
 
 		latest_backups = {
-			file_type: get_latest(pattern.format(self.site_slug)) for file_type, pattern in file_type_slugs.items()
+			file_type: get_latest(pattern)
+			for file_type, pattern in file_type_slugs.items()
 		}
 
 		recent_backups = {
 			file_type: old_enough(file_name) for file_type, file_name in latest_backups.items()
 		}
 
-		return recent_backups.get("database"), recent_backups.get("public"), recent_backups.get("private"), recent_backups.get("config")
+		return (
+			recent_backups.get("database"),
+			recent_backups.get("public"),
+			recent_backups.get("private"),
+			recent_backups.get("config"),
+		)
 
 	def zip_files(self):
 		for folder in ("public", "private"):
@@ -232,7 +239,14 @@ def fetch_latest_backups():
 		dict: relative Backup Paths
 	"""
 	frappe.only_for("System Manager")
-	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name, frappe.conf.db_password, db_host=frappe.db.host, db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
+	odb = BackupGenerator(
+		frappe.conf.db_name,
+		frappe.conf.db_name,
+		frappe.conf.db_password,
+		db_host=frappe.db.host,
+		db_type=frappe.conf.db_type,
+		db_port=frappe.conf.db_port,
+	)
 	database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
 
 	return {

From 9e000393512b1125bb1e2ee9a069e5c177150795 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 14 Aug 2020 17:25:18 +0530
Subject: [PATCH 258/361] style: Black

---
 frappe/integrations/offsite_backup_utils.py | 30 +++++++++++++++------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index 6e64bc5428..a551d8edf1 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -12,8 +12,10 @@ from frappe.utils import split_emails, get_backups_path
 def send_email(success, service_name, doctype, email_field, error_status=None):
 	recipients = get_recipients(doctype, email_field)
 	if not recipients:
-		frappe.log_error("No Email Recipient found for {0}".format(service_name),
-				"{0}: Failed to send backup status email".format(service_name))
+		frappe.log_error(
+			"No Email Recipient found for {0}".format(service_name),
+			"{0}: Failed to send backup status email".format(service_name),
+		)
 		return
 
 	if success:
@@ -23,7 +25,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 		subject = "Backup Upload Successful"
 		message = """
 <h3>Backup Uploaded Successfully!</h3>
-<p>Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!</p>""".format(service_name)
+<p>Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!</p>""".format(
+			service_name
+		)
 
 	else:
 		subject = "[Warning] Backup Upload Failed"
@@ -31,7 +35,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 <h3>Backup Upload Failed!</h3>
 <p>Oops, your automated backup to {0} failed.</p>
 <p>Error message: {1}</p>
-<p>Please contact your system manager for more information.</p>""".format(service_name, error_status)
+<p>Please contact your system manager for more information.</p>""".format(
+			service_name, error_status
+		)
 
 	frappe.sendmail(recipients=recipients, subject=subject, message=message)
 
@@ -45,7 +51,15 @@ def get_recipients(doctype, email_field):
 
 def get_latest_backup_file(with_files=False):
 	from frappe.utils.backups import BackupGenerator
-	odb = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name, frappe.conf.db_password, db_host=frappe.db.host, db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
+
+	odb = BackupGenerator(
+		frappe.conf.db_name,
+		frappe.conf.db_name,
+		frappe.conf.db_password,
+		db_host=frappe.db.host,
+		db_type=frappe.conf.db_type,
+		db_port=frappe.conf.db_port,
+	)
 	database, public, private, config = odb.get_recent_backup(older_than=24 * 30)
 
 	if with_files:
@@ -56,11 +70,11 @@ def get_latest_backup_file(with_files=False):
 
 def get_file_size(file_path, unit):
 	if not unit:
-		unit = 'MB'
+		unit = "MB"
 
 	file_size = os.path.getsize(file_path)
 
-	memory_size_unit_mapper = {'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4}
+	memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4}
 	i = 0
 	while i < memory_size_unit_mapper[unit]:
 		file_size = file_size / 1000.0
@@ -72,7 +86,7 @@ def get_file_size(file_path, unit):
 def validate_file_size():
 	frappe.flags.create_new_backup = True
 	latest_file, site_config = get_latest_backup_file()
-	file_size = get_file_size(latest_file, unit='GB')
+	file_size = get_file_size(latest_file, unit="GB")
 
 	if file_size > 1:
 		frappe.flags.create_new_backup = False

From beaaf16758000bec232ec295ffa0b7548b2a66ca Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 14 Aug 2020 18:30:27 +0530
Subject: [PATCH 259/361] test: show traceback on smtp error

---
 frappe/email/smtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index aa025465e5..2fb9534f61 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -244,7 +244,7 @@ class SMTPServer:
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.msgprint(_('Invalid Outgoing Mail Server or Port'))
+			frappe.msgprint(_('Invalid Outgoing Mail Server or Port <br> {}').format(frappe.get_traceback()))
 			traceback = sys.exc_info()[2]
 			raise_(frappe.ValidationError, e, traceback)
 

From 88d28d7757049ae8ba7b78734f4eb57e78f538d4 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Fri, 14 Aug 2020 18:47:10 +0530
Subject: [PATCH 260/361] chore(revert): show traceback on smtp error

This reverts commit beaaf16758000bec232ec295ffa0b7548b2a66ca.
---
 frappe/email/smtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index 2fb9534f61..aa025465e5 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -244,7 +244,7 @@ class SMTPServer:
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.msgprint(_('Invalid Outgoing Mail Server or Port <br> {}').format(frappe.get_traceback()))
+			frappe.msgprint(_('Invalid Outgoing Mail Server or Port'))
 			traceback = sys.exc_info()[2]
 			raise_(frappe.ValidationError, e, traceback)
 

From ecacb4cc4ec6032d18a2050e4fa41d0ae106fe07 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Sat, 15 Aug 2020 12:57:30 +0530
Subject: [PATCH 261/361] fix: Add missing Toggle Full Width action in Navbar
 Settings

---
 frappe/patches.txt                             |  1 +
 .../add_toggle_width_in_navbar_settings.py     | 18 ++++++++++++++++++
 frappe/utils/install.py                        |  6 ++++++
 3 files changed, 25 insertions(+)
 create mode 100644 frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py

diff --git a/frappe/patches.txt b/frappe/patches.txt
index 8f100c79f1..e3a8180357 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -299,3 +299,4 @@ frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart
 frappe.patches.v13_0.add_standard_navbar_items
 frappe.patches.v13_0.generate_theme_files_in_public_folder
 frappe.patches.v13_0.increase_password_length
+frappe.patches.v13_0.add_toggle_width_in_navbar_settings
diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
new file mode 100644
index 0000000000..98dc009197
--- /dev/null
+++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
@@ -0,0 +1,18 @@
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+	navbar_settings = navbar_settings = frappe.get_single("Navbar Settings")
+
+	for navbar_item in navbar_settings.settings_dropdown[4:]:
+		navbar_item.idx = navbar_item.idx + 1
+
+	navbar_settings.append('settings_dropdown', {
+		'item_label': 'Toggle Full Width',
+		'item_type': 'Action',
+		'action': 'frappe.ui.toolbar.toggle_full_width()',
+		'is_standard': 1,
+		'idx': 6
+	})
+
+	navbar_settings.save()
\ No newline at end of file
diff --git a/frappe/utils/install.py b/frappe/utils/install.py
index dbb77abd66..714e45a5b2 100644
--- a/frappe/utils/install.py
+++ b/frappe/utils/install.py
@@ -201,6 +201,12 @@ def add_standard_navbar_items():
 			'action': 'frappe.ui.toolbar.view_website()',
 			'is_standard': 1
 		},
+		{
+			'item_label': 'Toggle Full Width',
+			'item_type': 'Action',
+			'action': 'frappe.ui.toolbar.toggle_full_width()',
+			'is_standard': 1
+		},
 		{
 			'item_label': 'Background Jobs',
 			'item_type': 'Route',

From d207f8c29ca0b6888e0ecaaf87fa7d06daad9788 Mon Sep 17 00:00:00 2001
From: Syed Mujeer Hashmi <mujeerhashmi@4csolutions.in>
Date: Sat, 15 Aug 2020 18:04:36 +0530
Subject: [PATCH 262/361] fix: Numcard shows a incorrect number after >1.0e+5

Signed-off-by: Syed Mujeer Hashmi <mujeerhashmi@4csolutions.in>
---
 frappe/public/js/frappe/widgets/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index 4599b4adc8..efba24b4fb 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -124,7 +124,7 @@ function shorten_number(number, country) {
 	for (const map of number_system) {
 		const condition = map.condition ? map.condition(x) : x >= map.divisor;
 		if (condition) {
-			return Math.round(number/map.divisor) +  ' ' + map.symbol;
+			return (x/map.divisor) +  ' ' + map.symbol;
 		}
 	}
 	return number.toFixed();

From 74f43ab2282b24901adcc75bad37db58d7245d5b Mon Sep 17 00:00:00 2001
From: Syed Mujeer Hashmi <mujeerhashmi@4csolutions.in>
Date: Sun, 16 Aug 2020 13:08:53 +0530
Subject: [PATCH 263/361] fix: Number can be negative as well don't use 'x'.

Co-authored-by: Prssanna Desai <prssud@gmail.com>
---
 frappe/public/js/frappe/widgets/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js
index efba24b4fb..f36793996a 100644
--- a/frappe/public/js/frappe/widgets/utils.js
+++ b/frappe/public/js/frappe/widgets/utils.js
@@ -124,7 +124,7 @@ function shorten_number(number, country) {
 	for (const map of number_system) {
 		const condition = map.condition ? map.condition(x) : x >= map.divisor;
 		if (condition) {
-			return (x/map.divisor) +  ' ' + map.symbol;
+			return (number/map.divisor).toFixed(2) + ' ' + map.symbol;
 		}
 	}
 	return number.toFixed();

From 1635597f67686c266efec33529e2862c61055de4 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Sun, 16 Aug 2020 18:28:45 +0530
Subject: [PATCH 264/361] fix: Assignment

---
 frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
index 98dc009197..cab171f166 100644
--- a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
+++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 import frappe
 
 def execute():
-	navbar_settings = navbar_settings = frappe.get_single("Navbar Settings")
+	navbar_settings = frappe.get_single("Navbar Settings")
 
 	for navbar_item in navbar_settings.settings_dropdown[4:]:
 		navbar_item.idx = navbar_item.idx + 1

From f549c5118598f779ae5b2424af105934dcf06b79 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Sun, 16 Aug 2020 19:26:29 +0530
Subject: [PATCH 265/361] fix: Idx for help dropdown

---
 frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
index cab171f166..4d241cd8c7 100644
--- a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
+++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
@@ -4,7 +4,7 @@ import frappe
 def execute():
 	navbar_settings = frappe.get_single("Navbar Settings")
 
-	for navbar_item in navbar_settings.settings_dropdown[4:]:
+	for navbar_item in navbar_settings.settings_dropdown[5:]:
 		navbar_item.idx = navbar_item.idx + 1
 
 	navbar_settings.append('settings_dropdown', {

From 77018fc9d6ab714a3e5be292386c2077c3537ebc Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 10 Aug 2020 11:07:51 +0530
Subject: [PATCH 266/361] fix(cleanup): cleanup layout of doctype, user, role,
 add home_page to Portal Settings, Role

---
 frappe/auth.py                                |  13 +-
 frappe/core/doctype/doctype/doctype.json      |  84 +++++-
 frappe/core/doctype/module_def/module_def.js  |   4 +-
 .../core/doctype/module_def/module_def.json   | 236 +++++----------
 frappe/core/doctype/module_def/module_def.py  |  10 +-
 frappe/core/doctype/role/role.json            | 280 +++++------------
 .../doctype/server_script/server_script.js    |  45 ++-
 .../doctype/server_script/server_script.json  |  15 +-
 frappe/core/doctype/user/user.json            |  67 ++++-
 .../onboarding_step/onboarding_step.json      |   2 +-
 .../generate_theme_files_in_public_folder.py  |   7 +-
 frappe/public/less/page.less                  |   4 +-
 frappe/utils/boilerplate.py                   |   5 +-
 .../portal_settings/portal_settings.json      | 282 ++++--------------
 .../portal_settings/test_portal_settings.py   |  10 +
 frappe/website/doctype/web_form/web_form.json |   4 +-
 frappe/website/doctype/web_page/web_page.json |   4 +-
 .../website_settings/website_settings.json    |  10 +-
 frappe/website/utils.py                       |  52 +++-
 frappe/www/login.py                           |   6 +-
 20 files changed, 507 insertions(+), 633 deletions(-)
 create mode 100644 frappe/website/doctype/portal_settings/test_portal_settings.py

diff --git a/frappe/auth.py b/frappe/auth.py
index 7902469c14..6d51629c58 100644
--- a/frappe/auth.py
+++ b/frappe/auth.py
@@ -18,6 +18,7 @@ from frappe.utils.password import check_password, delete_login_failed_cache
 from frappe.core.doctype.activity_log.activity_log import add_authentication_log
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,
 	confirm_otp_token, get_cached_user_pass)
+from frappe.website.utils import get_home_page
 
 from six.moves.urllib.parse import quote
 
@@ -167,7 +168,7 @@ class LoginManager:
 			frappe.local.cookie_manager.set_cookie("system_user", "no")
 			if not resume:
 				frappe.local.response["message"] = "No App"
-				frappe.local.response["home_page"] = get_website_user_home_page(self.user)
+				frappe.local.response["home_page"] = '/' + get_home_page()
 		else:
 			frappe.local.cookie_manager.set_cookie("system_user", "yes")
 			if not resume:
@@ -382,16 +383,6 @@ def clear_cookies():
 		frappe.session.sid = ""
 	frappe.local.cookie_manager.delete_cookie(["full_name", "user_id", "sid", "user_image", "system_user"])
 
-def get_website_user_home_page(user):
-	home_page_method = frappe.get_hooks('get_website_user_home_page')
-	if home_page_method:
-		home_page = frappe.get_attr(home_page_method[-1])(user)
-		return '/' + home_page.strip('/')
-	elif frappe.get_hooks('website_user_home_page'):
-		return '/' + frappe.get_hooks('website_user_home_page')[-1].strip('/')
-	else:
-		return '/me'
-
 def get_last_tried_login_data(user, get_last_login=False):
 	locked_account_time = frappe.cache().hget('locked_account_time', user)
 	if get_last_login and locked_account_time:
diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json
index 698289140e..b890f743cc 100644
--- a/frappe/core/doctype/doctype/doctype.json
+++ b/frappe/core/doctype/doctype/doctype.json
@@ -473,6 +473,8 @@
    "label": "Documentation Link"
   },
   {
+   "collapsible": 1,
+   "collapsible_depends_on": "actions",
    "fieldname": "actions_section",
    "fieldtype": "Section Break",
    "label": "Actions"
@@ -484,6 +486,8 @@
    "options": "DocType Action"
   },
   {
+   "collapsible": 1,
+   "collapsible_depends_on": "links",
    "fieldname": "links_section",
    "fieldtype": "Section Break",
    "label": "Links Section"
@@ -528,8 +532,84 @@
  ],
  "icon": "fa fa-bolt",
  "idx": 6,
- "links": [],
- "modified": "2020-07-21 16:20:57.028802",
+ "links": [
+  {
+   "group": "Views",
+   "link_doctype": "Report",
+   "link_fieldname": "ref_doctype"
+  },
+  {
+   "group": "Workflow",
+   "link_doctype": "Workflow",
+   "link_fieldname": "document_type"
+  },
+  {
+   "group": "Workflow",
+   "link_doctype": "Notification",
+   "link_fieldname": "document_type"
+  },
+  {
+   "group": "Customization",
+   "link_doctype": "Custom Field",
+   "link_fieldname": "dt"
+  },
+  {
+   "group": "Customization",
+   "link_doctype": "Custom Script",
+   "link_fieldname": "dt"
+  },
+  {
+   "group": "Customization",
+   "link_doctype": "Server Script",
+   "link_fieldname": "reference_doctype"
+  },
+  {
+   "group": "Workflow",
+   "link_doctype": "Webhook",
+   "link_fieldname": "webhook_doctype"
+  },
+  {
+   "group": "Views",
+   "link_doctype": "Print Format",
+   "link_fieldname": "doc_type"
+  },
+  {
+   "group": "Views",
+   "link_doctype": "Web Form",
+   "link_fieldname": "doc_type"
+  },
+  {
+   "group": "Views",
+   "link_doctype": "Calendar View",
+   "link_fieldname": "reference_doctype"
+  },
+  {
+   "group": "Views",
+   "link_doctype": "Kanban Board",
+   "link_fieldname": "reference_doctype"
+  },
+  {
+   "group": "Workflow",
+   "link_doctype": "Onboarding Step",
+   "link_fieldname": "reference_document"
+  },
+  {
+   "group": "Rules",
+   "link_doctype": "Auto Repeat",
+   "link_fieldname": "reference_doctype"
+  },
+  {
+   "group": "Rules",
+   "link_doctype": "Assignment Rule",
+   "link_fieldname": "document_type"
+  },
+  {
+   "group": "Rules",
+   "link_doctype": "Energy Point Rule",
+   "link_fieldname": "reference_doctype"
+  }
+ ],
+ "modified": "2020-08-06 12:59:32.369093",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",
diff --git a/frappe/core/doctype/module_def/module_def.js b/frappe/core/doctype/module_def/module_def.js
index 5bb8db6d5c..c7a6cf85f9 100644
--- a/frappe/core/doctype/module_def/module_def.js
+++ b/frappe/core/doctype/module_def/module_def.js
@@ -3,6 +3,8 @@
 
 frappe.ui.form.on('Module Def', {
 	refresh: function(frm) {
-
+		frappe.xcall('frappe.core.doctype.module_def.module_def.get_installed_apps').then(r => {
+			frm.set_df_property('app_name', 'options', JSON.parse(r));
+		});
 	}
 });
diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json
index 4ff7a40877..b841eb1f0b 100644
--- a/frappe/core/doctype/module_def/module_def.json
+++ b/frappe/core/doctype/module_def/module_def.json
@@ -1,172 +1,88 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:module_name", 
- "beta": 0, 
- "creation": "2013-01-10 16:34:03", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "editable_grid": 0, 
- "engine": "InnoDB", 
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "field:module_name",
+ "creation": "2013-01-10 16:34:03",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "module_name",
+  "custom",
+  "app_name",
+  "restrict_to_domain"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "module_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Module Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "oldfieldname": "module_name", 
-   "oldfieldtype": "Data", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "module_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Module Name",
+   "oldfieldname": "module_name",
+   "oldfieldtype": "Data",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "app_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 1, 
-   "label": "App Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "app_name",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "App Name",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "restrict_to_domain", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Restrict To Domain", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Domain", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "restrict_to_domain",
+   "fieldtype": "Link",
+   "label": "Restrict To Domain",
+   "options": "Domain"
+  },
+  {
+   "default": "0",
+   "fieldname": "custom",
+   "fieldtype": "Check",
+   "label": "Custom"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-sitemap", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-07-13 03:05:28.213656", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Module Def", 
- "owner": "Administrator", 
+ ],
+ "icon": "fa fa-sitemap",
+ "idx": 1,
+ "links": [
+  {
+   "link_doctype": "DocType",
+   "link_fieldname": "module"
+  },
+  {
+   "link_doctype": "Desk Page",
+   "link_fieldname": "module"
+  }
+ ],
+ "modified": "2020-08-06 12:39:30.740379",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Module Def",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Administrator", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Administrator",
+   "share": 1,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 0, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "create": 1,
+   "delete": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 1, 
- "sort_order": "ASC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "show_name_in_global_search": 1,
+ "sort_field": "modified",
+ "sort_order": "ASC",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py
index 755cb86dbe..930c46e60b 100644
--- a/frappe/core/doctype/module_def/module_def.py
+++ b/frappe/core/doctype/module_def/module_def.py
@@ -2,7 +2,7 @@
 # MIT License. See license.txt
 
 from __future__ import unicode_literals
-import frappe, os
+import frappe, os, json
 
 from frappe.model.document import Document
 
@@ -11,7 +11,7 @@ class ModuleDef(Document):
 		"""If in `developer_mode`, create folder for module and
 			add in `modules.txt` of app if missing."""
 		frappe.clear_cache()
-		if frappe.conf.get("developer_mode"):
+		if not self.custom and frappe.conf.get("developer_mode"):
 			self.create_modules_folder()
 			self.add_to_modules_txt()
 
@@ -43,7 +43,7 @@ class ModuleDef(Document):
 	def on_trash(self):
 		"""Delete module name from modules.txt"""
 
-		if frappe.flags.in_uninstall:
+		if frappe.flags.in_uninstall or self.custom:
 			return
 
 		modules = None
@@ -60,3 +60,7 @@ class ModuleDef(Document):
 
 				frappe.clear_cache()
 				frappe.setup_module_map()
+
+@frappe.whitelist()
+def get_installed_apps():
+	return json.dumps(frappe.get_installed_apps())
\ No newline at end of file
diff --git a/frappe/core/doctype/role/role.json b/frappe/core/doctype/role/role.json
index 1eebb71a36..0ad15ba10b 100644
--- a/frappe/core/doctype/role/role.json
+++ b/frappe/core/doctype/role/role.json
@@ -1,216 +1,90 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:role_name", 
- "beta": 0, 
- "creation": "2013-01-08 15:50:01", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Document", 
- "editable_grid": 0, 
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "field:role_name",
+ "creation": "2013-01-08 15:50:01",
+ "doctype": "DocType",
+ "document_type": "Document",
+ "engine": "InnoDB",
+ "field_order": [
+  "role_name",
+  "home_page",
+  "restrict_to_domain",
+  "column_break_4",
+  "disabled",
+  "desk_access",
+  "two_factor_auth"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "role_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Role Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "oldfieldname": "role_name", 
-   "oldfieldtype": "Data", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "role_name",
+   "fieldtype": "Data",
+   "label": "Role Name",
+   "oldfieldname": "role_name",
+   "oldfieldtype": "Data",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "If disabled, this role will be removed from all users.", 
-   "fieldname": "disabled", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Disabled", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "description": "If disabled, this role will be removed from all users.",
+   "fieldname": "disabled",
+   "fieldtype": "Check",
+   "label": "Disabled"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "1", 
-   "fieldname": "desk_access", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Desk Access", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "1",
+   "fieldname": "desk_access",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Desk Access"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "0", 
-   "fieldname": "two_factor_auth", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Two Factor Authentication", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "two_factor_auth",
+   "fieldtype": "Check",
+   "label": "Two Factor Authentication"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "restrict_to_domain", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Restrict To Domain", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Domain", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "restrict_to_domain",
+   "fieldtype": "Link",
+   "label": "Restrict To Domain",
+   "options": "Domain"
+  },
+  {
+   "description": "Route: Example \"/desk\"",
+   "fieldname": "home_page",
+   "fieldtype": "Data",
+   "label": "Home Page"
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-bookmark", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-07-06 12:42:57.097914", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Role", 
- "owner": "Administrator", 
+ ],
+ "icon": "fa fa-bookmark",
+ "idx": 1,
+ "links": [],
+ "modified": "2020-08-06 15:42:59.036960",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Role",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_order": "ASC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "ASC",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/core/doctype/server_script/server_script.js b/frappe/core/doctype/server_script/server_script.js
index d7f4c3e536..78ef2d0509 100644
--- a/frappe/core/doctype/server_script/server_script.js
+++ b/frappe/core/doctype/server_script/server_script.js
@@ -2,8 +2,11 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Server Script', {
+	setup: function(frm) {
+		frm.trigger('setup_help');
+	},
 	refresh: function(frm) {
-		if(frm.doc.script_type === 'Scheduler Event' && !frm.doc.disabled){
+		if (frm.doc.script_type === 'Scheduler Event' && !frm.doc.disabled) {
 			frm.add_custom_button('Schedule Script', function() {
 				var d = new frappe.ui.Dialog({
 					title: "Schedule Script Execution",
@@ -33,14 +36,50 @@ frappe.ui.form.on('Server Script', {
 		}
 	},
 
-	schedule_script(frm, data){
+	schedule_script(frm, data) {
 		frm.call({
 			method: "frappe.core.doctype.server_script.server_script.setup_scheduler_events",
 			args: {
 				'script_name': frm.doc.name,
 				'frequency': data.event_type
 			}
-		})
+		});
+	},
+
+	setup_help(frm) {
+		frm.get_field('help_html').html(`
+<h3>Examples</h3>
+<h4>DocType Event</h4>
+<pre><code>
+# set property
+if "test" in doc.description:
+    doc.status = 'Closed'
+
+
+# validate
+if "validate" in doc.description:
+    raise frappe.ValidationError
+
+# auto create another document
+if doc.allocted_to:
+    frappe.get_doc(dict(
+        doctype = 'ToDo'
+        owner = doc.allocated_to,
+        description = doc.subject
+    )).insert()
+</code></pre>
+<hr>
+
+<h4>API Call</h4>
+<pre><code>
+# respond to API
+
+if frappe.form_dict.message == "ping":
+	frappe.response['message'] = "pong"
+else:
+	frappe.response['message'] = "ok"
+</code></pre>
+`);
 	}
 
 });
diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json
index bef3dfc60c..3ed4076430 100644
--- a/frappe/core/doctype/server_script/server_script.json
+++ b/frappe/core/doctype/server_script/server_script.json
@@ -14,7 +14,9 @@
   "api_method",
   "allow_guest",
   "section_break_8",
-  "script"
+  "script",
+  "help_section",
+  "help_html"
  ],
  "fields": [
   {
@@ -72,10 +74,19 @@
   {
    "fieldname": "section_break_8",
    "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "help_section",
+   "fieldtype": "Section Break",
+   "label": "Help"
+  },
+  {
+   "fieldname": "help_html",
+   "fieldtype": "HTML"
   }
  ],
  "links": [],
- "modified": "2020-04-06 11:24:38.161555",
+ "modified": "2020-08-07 13:13:02.483963",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Server Script",
diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json
index 7ed14e094c..9e6781ba64 100644
--- a/frappe/core/doctype/user/user.json
+++ b/frappe/core/doctype/user/user.json
@@ -588,9 +588,70 @@
  "icon": "fa fa-user",
  "idx": 413,
  "image_field": "user_image",
- "links": [],
+ "links": [
+  {
+   "group": "Profile",
+   "link_doctype": "Contact",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Profile",
+   "link_doctype": "Chat Profile",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Profile",
+   "link_doctype": "Blogger",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Logs",
+   "link_doctype": "Access Log",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Logs",
+   "link_doctype": "Activity Log",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Logs",
+   "link_doctype": "Energy Point Log",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Logs",
+   "link_doctype": "Route History",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Settings",
+   "link_doctype": "User Permission",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Settings",
+   "link_doctype": "Assignment Rule",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Settings",
+   "link_doctype": "Document Follow",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Activity",
+   "link_doctype": "Communication",
+   "link_fieldname": "user"
+  },
+  {
+   "group": "Activity",
+   "link_doctype": "ToDo",
+   "link_fieldname": "owner"
+  }
+ ],
  "max_attachments": 5,
- "modified": "2020-04-08 12:27:36.716490",
+ "modified": "2020-08-06 19:48:49.677800",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User",
@@ -606,13 +667,13 @@
    "read": 1,
    "report": 1,
    "role": "System Manager",
+   "set_user_permissions": 1,
    "share": 1,
    "write": 1
   },
   {
    "permlevel": 1,
    "read": 1,
-   "report": 1,
    "role": "System Manager",
    "write": 1
   }
diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.json b/frappe/desk/doctype/onboarding_step/onboarding_step.json
index 365a1c7d21..79d659b1ed 100644
--- a/frappe/desk/doctype/onboarding_step/onboarding_step.json
+++ b/frappe/desk/doctype/onboarding_step/onboarding_step.json
@@ -184,7 +184,7 @@
   }
  ],
  "links": [],
- "modified": "2020-05-18 19:42:30.435604",
+ "modified": "2020-08-06 12:55:20.377679",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Onboarding Step",
diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index c5a64780cd..b2021d2b52 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -11,5 +11,8 @@ def execute():
 	)
 	for theme in themes:
 		doc = frappe.get_doc("Website Theme", theme.name)
-		doc.generate_bootstrap_theme()
-		doc.save()
+		try:
+			doc.generate_bootstrap_theme()
+			doc.save()
+		except:
+			pass
diff --git a/frappe/public/less/page.less b/frappe/public/less/page.less
index 2b4fb7f495..77d12688a4 100644
--- a/frappe/public/less/page.less
+++ b/frappe/public/less/page.less
@@ -20,7 +20,7 @@
 .sub-heading {
 	display: inline-block;
 	margin-right: 10px;
-	max-width: 50%;
+	max-width: 40%;
 	vertical-align: middle;
 	overflow: visible;
 }
@@ -83,7 +83,7 @@
 	padding-bottom: 17px;
 
 	.sub-heading {
-		max-width: 50%;
+		max-width: 40%;
 		overflow: hidden;
 	}
 }
diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py
index 7dfa2ca1b0..675ceb3f57 100755
--- a/frappe/utils/boilerplate.py
+++ b/frappe/utils/boilerplate.py
@@ -173,7 +173,7 @@ app_license = "{app_license}"
 # Home Pages
 # ----------
 
-# application home page (will override Website Settings)
+# application home page (will not override Website Settings)
 # home_page = "login"
 
 # website user home page (by Role)
@@ -181,9 +181,6 @@ app_license = "{app_license}"
 #	"Role": "home_page"
 # }}
 
-# Website user home page (by function)
-# get_website_user_home_page = "{app_name}.utils.get_home_page"
-
 # Generators
 # ----------
 
diff --git a/frappe/website/doctype/portal_settings/portal_settings.json b/frappe/website/doctype/portal_settings/portal_settings.json
index 81b2a94c08..48321d0512 100644
--- a/frappe/website/doctype/portal_settings/portal_settings.json
+++ b/frappe/website/doctype/portal_settings/portal_settings.json
@@ -1,231 +1,79 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-03-30 01:40:20.001775", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 0, 
+ "actions": [],
+ "creation": "2016-03-30 01:40:20.001775",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "default_role",
+  "default_portal_home",
+  "standard_menu_items",
+  "hide_standard_menu",
+  "menu",
+  "custom_sidebar_menu",
+  "custom_menu"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "default_role", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Default Role at Time of Signup", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Role", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "default_role",
+   "fieldtype": "Link",
+   "label": "Default Role at Time of Signup",
+   "options": "Role"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "standard_menu_items", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Standard Sidebar Menu", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "standard_menu_items",
+   "fieldtype": "Section Break",
+   "label": "Standard Sidebar Menu"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "hide_standard_menu", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Hide Standard Menu", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "hide_standard_menu",
+   "fieldtype": "Check",
+   "label": "Hide Standard Menu"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "menu", 
-   "fieldtype": "Table", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Portal Menu", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Portal Menu Item", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "menu",
+   "fieldtype": "Table",
+   "label": "Portal Menu",
+   "options": "Portal Menu Item"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "custom_sidebar_menu", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Custom Sidebar Menu", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "custom_sidebar_menu",
+   "fieldtype": "Section Break",
+   "label": "Custom Sidebar Menu"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "custom_menu", 
-   "fieldtype": "Table", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Custom Menu Items", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Portal Menu Item", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "custom_menu",
+   "fieldtype": "Table",
+   "label": "Custom Menu Items",
+   "options": "Portal Menu Item"
+  },
+  {
+   "description": "Example: \"/desk\"",
+   "fieldname": "default_portal_home",
+   "fieldtype": "Data",
+   "label": "Default Portal Home"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:31.476181", 
- "modified_by": "Administrator", 
- "module": "Website", 
- "name": "Portal Settings", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-08-06 15:43:36.387050",
+ "modified_by": "Administrator",
+ "module": "Website",
+ "name": "Portal Settings",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "Website Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Website Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/website/doctype/portal_settings/test_portal_settings.py b/frappe/website/doctype/portal_settings/test_portal_settings.py
new file mode 100644
index 0000000000..3dd4aa0c35
--- /dev/null
+++ b/frappe/website/doctype/portal_settings/test_portal_settings.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestPortalSettings(unittest.TestCase):
+	pass
diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json
index da5f8bee83..77dbbb9734 100644
--- a/frappe/website/doctype/web_form/web_form.json
+++ b/frappe/website/doctype/web_form/web_form.json
@@ -203,6 +203,7 @@
   },
   {
    "collapsible": 1,
+   "collapsible_depends_on": "client_script",
    "fieldname": "client_script_section",
    "fieldtype": "Section Break",
    "label": "Client Script"
@@ -344,6 +345,7 @@
   },
   {
    "collapsible": 1,
+   "collapsible_depends_on": "custom_css",
    "fieldname": "custom_css_section",
    "fieldtype": "Section Break",
    "label": "Custom CSS"
@@ -359,7 +361,7 @@
  "icon": "icon-edit",
  "is_published_field": "published",
  "links": [],
- "modified": "2020-07-21 16:25:37.028459",
+ "modified": "2020-08-07 13:12:03.945686",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Form",
diff --git a/frappe/website/doctype/web_page/web_page.json b/frappe/website/doctype/web_page/web_page.json
index 8f03237363..955b34f44e 100644
--- a/frappe/website/doctype/web_page/web_page.json
+++ b/frappe/website/doctype/web_page/web_page.json
@@ -137,6 +137,7 @@
   },
   {
    "collapsible": 1,
+   "collapsible_depends_on": "insert_code",
    "fieldname": "custom_javascript",
    "fieldtype": "Section Break",
    "label": "Script"
@@ -157,6 +158,7 @@
   },
   {
    "collapsible": 1,
+   "collapsible_depends_on": "insert_style",
    "fieldname": "custom_css",
    "fieldtype": "Section Break",
    "label": "Style"
@@ -292,7 +294,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 20,
- "modified": "2020-07-21 16:25:17.899069",
+ "modified": "2020-08-07 10:55:54.885448",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Page",
diff --git a/frappe/website/doctype/website_settings/website_settings.json b/frappe/website/doctype/website_settings/website_settings.json
index c91366c103..65a45c1796 100644
--- a/frappe/website/doctype/website_settings/website_settings.json
+++ b/frappe/website/doctype/website_settings/website_settings.json
@@ -1,5 +1,11 @@
 {
- "actions": [],
+ "actions": [
+  {
+   "action": "frappe.sessions.clear",
+   "action_type": "Server Action",
+   "label": "Clear Cache"
+  }
+ ],
  "creation": "2013-04-30 12:58:46",
  "doctype": "DocType",
  "document_type": "Other",
@@ -358,7 +364,7 @@
  "issingle": 1,
  "links": [],
  "max_attachments": 10,
- "modified": "2020-05-15 14:12:32.907352",
+ "modified": "2020-08-07 13:34:19.181561",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Website Settings",
diff --git a/frappe/website/utils.py b/frappe/website/utils.py
index dc981fe30a..36e603f5e5 100644
--- a/frappe/website/utils.py
+++ b/frappe/website/utils.py
@@ -73,23 +73,22 @@ def get_home_page():
 	def _get_home_page():
 		home_page = None
 
-		get_website_user_home_page = frappe.get_hooks('get_website_user_home_page')
-		if get_website_user_home_page:
-			home_page = frappe.get_attr(get_website_user_home_page[-1])(frappe.session.user)
+		# for user
+		if frappe.session.user != 'Guest':
+			# by role
+			for role in frappe.get_roles():
+				home_page = frappe.db.get_value('Role', role, 'home_page')
+				if home_page: break
 
+			# portal default
+			if not home_page:
+				home_page = frappe.db.get_value("Portal Settings", None, "default_portal_home")
+
+		# by hooks
 		if not home_page:
-			role_home_page = frappe.get_hooks("role_home_page")
-			if role_home_page:
-				for role in frappe.get_roles():
-					if role in role_home_page:
-						home_page = role_home_page[role][-1]
-						break
-
-		if not home_page:
-			home_page = frappe.get_hooks("home_page")
-			if home_page:
-				home_page = home_page[-1]
+			home_page = get_home_page_via_hooks()
 
+		# global
 		if not home_page:
 			home_page = frappe.db.get_value("Website Settings", None, "home_page") or "login"
 
@@ -99,6 +98,31 @@ def get_home_page():
 
 	return frappe.cache().hget("home_page", frappe.session.user, _get_home_page)
 
+def get_home_page_via_hooks():
+	home_page = None
+
+	home_page_method = frappe.get_hooks('get_website_user_home_page')
+	if home_page_method:
+		home_page = frappe.get_attr(home_page_method[-1])(user).strip('/')
+	elif frappe.get_hooks('website_user_home_page'):
+		home_page = frappe.get_hooks('website_user_home_page')[-1].strip('/')
+
+	if not home_page:
+		role_home_page = frappe.get_hooks("role_home_page")
+		if role_home_page:
+			for role in frappe.get_roles():
+				if role in role_home_page:
+					home_page = role_home_page[role][-1]
+					break
+
+	if not home_page:
+		home_page = frappe.get_hooks("home_page")
+		if home_page:
+			home_page = home_page[-1]
+
+	return home_page
+
+
 def is_signup_enabled():
 	if getattr(frappe.local, "is_signup_enabled", None) is None:
 		frappe.local.is_signup_enabled = True
diff --git a/frappe/www/login.py b/frappe/www/login.py
index 65952f0154..f2d9c9d768 100644
--- a/frappe/www/login.py
+++ b/frappe/www/login.py
@@ -12,6 +12,7 @@ from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings
 from frappe.utils.password import get_decrypted_password
 from frappe.utils.html_utils import get_icon_html
 from frappe.integrations.oauth2_logins import decoder_compat
+from frappe.website.utils import get_home_page
 
 no_cache = True
 
@@ -20,7 +21,10 @@ def get_context(context):
 
 	if frappe.session.user != "Guest":
 		if not redirect_to:
-			redirect_to = "/me" if frappe.session.data.user_type=="Website User" else "/desk"
+			if frappe.session.data.user_type=="Website User":
+				redirect_to = get_home_page()
+			else:
+				redirect_to = "/desk"
 		frappe.local.flags.redirect_location = redirect_to
 		raise frappe.Redirect
 

From b0b5a565666d0c3bfc0c600d31aed54c3cdc7fba Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 10 Aug 2020 11:19:31 +0530
Subject: [PATCH 267/361] fix(minor): in install, load doctype_action and
 doctype_link before doctype

---
 frappe/utils/install.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/utils/install.py b/frappe/utils/install.py
index 714e45a5b2..d20abdb11d 100644
--- a/frappe/utils/install.py
+++ b/frappe/utils/install.py
@@ -10,6 +10,8 @@ from frappe.utils.password import update_password
 def before_install():
 	frappe.reload_doc("core", "doctype", "docfield")
 	frappe.reload_doc("core", "doctype", "docperm")
+	frappe.reload_doc("core", "doctype", "doctype_action")
+	frappe.reload_doc("core", "doctype", "doctype_link")
 	frappe.reload_doc("core", "doctype", "doctype")
 
 def after_install():

From b0f2af0ac5f5f91b7d0ed235123800b01fda227f Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 10 Aug 2020 13:19:39 +0530
Subject: [PATCH 268/361] fix(tests): add tests for home page

---
 frappe/tests/test_website.py | 45 ++++++++++++++++++++++++++++++++++++
 frappe/website/utils.py      |  2 +-
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py
index d114f7d1b2..601c98accc 100644
--- a/frappe/tests/test_website.py
+++ b/frappe/tests/test_website.py
@@ -4,10 +4,55 @@ import unittest
 
 import frappe
 from frappe.website import render
+from frappe.website.utils import get_home_page
 from frappe.utils import set_request
 
 
 class TestWebsite(unittest.TestCase):
+	def test_home_page_for_role(self):
+		frappe.delete_doc_if_exists('User', 'test-user-for-home-page@example.com')
+		frappe.delete_doc_if_exists('Role', 'home-page-test')
+		frappe.delete_doc_if_exists('Web Page', 'home-page-test')
+		user = frappe.get_doc(dict(
+			doctype='User',
+			email='test-user-for-home-page@example.com',
+			first_name='test')).insert()
+
+		role = frappe.get_doc(dict(
+			doctype = 'Role',
+			role_name = 'home-page-test',
+			desk_access = 0,
+			home_page = '/home-page-test'
+		)).insert()
+
+		user.add_roles(role.name)
+		user.save()
+
+		web_page = frappe.get_doc(dict(
+			doctype = 'Web Page',
+			title = 'Home Page Test',
+			route = 'home-page-test',
+			published = 1,
+			content_type = 'Markdown',
+			main_section_md = '# h1\n\home page test',
+		)).insert()
+
+		frappe.set_user('test-user-for-home-page@example.com')
+		self.assertEqual(get_home_page(), 'home-page-test')
+
+		frappe.set_user('Administrator')
+		role.home_page = ''
+		role.save()
+
+		# home page via portal settings
+		frappe.db.set_value('Portal Settings', None, 'default_portal_home', 'test-portal-home')
+
+		frappe.set_user('test-user-for-home-page@example.com')
+		frappe.cache().hdel('home_page', frappe.session.user)
+		self.assertEqual(get_home_page(), 'test-portal-home')
+
+
+
 	def test_page_load(self):
 		frappe.set_user('Guest')
 		set_request(method='POST', path='login')
diff --git a/frappe/website/utils.py b/frappe/website/utils.py
index 36e603f5e5..1100db8c56 100644
--- a/frappe/website/utils.py
+++ b/frappe/website/utils.py
@@ -67,7 +67,7 @@ def get_comment_list(doctype, name):
 
 
 def get_home_page():
-	if frappe.local.flags.home_page:
+	if frappe.local.flags.home_page and not frappe.flags.in_test:
 		return frappe.local.flags.home_page
 
 	def _get_home_page():

From 137df8290b1bbf3f37f8011de5643afcef1b787c Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 11 Aug 2020 16:45:18 +0530
Subject: [PATCH 269/361] fix(sider): fix sider issues

---
 .../v13_0/generate_theme_files_in_public_folder.py       | 3 ++-
 frappe/tests/test_website.py                             | 9 ---------
 frappe/utils/boilerplate.py                              | 2 +-
 frappe/website/utils.py                                  | 2 +-
 4 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index b2021d2b52..5f80540ffe 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -15,4 +15,5 @@ def execute():
 			doc.generate_bootstrap_theme()
 			doc.save()
 		except:
-			pass
+			print('Ignoring....')
+			print(frappe.get_traceback())
diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py
index 601c98accc..c708f670c1 100644
--- a/frappe/tests/test_website.py
+++ b/frappe/tests/test_website.py
@@ -28,15 +28,6 @@ class TestWebsite(unittest.TestCase):
 		user.add_roles(role.name)
 		user.save()
 
-		web_page = frappe.get_doc(dict(
-			doctype = 'Web Page',
-			title = 'Home Page Test',
-			route = 'home-page-test',
-			published = 1,
-			content_type = 'Markdown',
-			main_section_md = '# h1\n\home page test',
-		)).insert()
-
 		frappe.set_user('test-user-for-home-page@example.com')
 		self.assertEqual(get_home_page(), 'home-page-test')
 
diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py
index 675ceb3f57..d3b206559f 100755
--- a/frappe/utils/boilerplate.py
+++ b/frappe/utils/boilerplate.py
@@ -173,7 +173,7 @@ app_license = "{app_license}"
 # Home Pages
 # ----------
 
-# application home page (will not override Website Settings)
+# application home page (will override Website Settings)
 # home_page = "login"
 
 # website user home page (by Role)
diff --git a/frappe/website/utils.py b/frappe/website/utils.py
index 1100db8c56..239883f655 100644
--- a/frappe/website/utils.py
+++ b/frappe/website/utils.py
@@ -103,7 +103,7 @@ def get_home_page_via_hooks():
 
 	home_page_method = frappe.get_hooks('get_website_user_home_page')
 	if home_page_method:
-		home_page = frappe.get_attr(home_page_method[-1])(user).strip('/')
+		home_page = frappe.get_attr(home_page_method[-1])(frappe.session.user).strip('/')
 	elif frappe.get_hooks('website_user_home_page'):
 		home_page = frappe.get_hooks('website_user_home_page')[-1].strip('/')
 

From e213e50e6cd8b52655774bd7ce4ce2b4538c8f8d Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 11 Aug 2020 20:13:42 +0530
Subject: [PATCH 270/361] fix(sider): ignore except for patch

---
 frappe/patches/v13_0/generate_theme_files_in_public_folder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index 5f80540ffe..44f6fc92ae 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -14,6 +14,6 @@ def execute():
 		try:
 			doc.generate_bootstrap_theme()
 			doc.save()
-		except:
+		except: # pylint: disable=E722
 			print('Ignoring....')
 			print(frappe.get_traceback())

From 8cf288a657e69ed93979829bde20e69621705842 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 11 Aug 2020 20:20:07 +0530
Subject: [PATCH 271/361] fix(sider): use flake8 noqa?

---
 frappe/patches/v13_0/generate_theme_files_in_public_folder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index 44f6fc92ae..4315f06ebe 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -14,6 +14,6 @@ def execute():
 		try:
 			doc.generate_bootstrap_theme()
 			doc.save()
-		except: # pylint: disable=E722
+		except: # noqa: E722
 			print('Ignoring....')
 			print(frappe.get_traceback())

From bb074aa90de5dd01cb053ed3af6fca8473f9af1d Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 14:22:21 +0530
Subject: [PATCH 272/361] fix(test): retry test_event_producer

---
 .../event_producer/test_event_producer.py     | 68 +++++++++++--------
 1 file changed, 39 insertions(+), 29 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 0d414f5d63..9f7b7d8d34 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -205,35 +205,12 @@ class TestEventProducer(unittest.TestCase):
 
 	def test_inner_mapping(self):
 		producer = get_remote_site()
-		event_producer = frappe.get_doc('Event Producer', producer_url)
-		event_producer.producer_doctypes = []
-		inner_mapping = [
-			{
-				'local_fieldname':'role_name',
-				'remote_fieldname':'title'
-			}
-		]
-		inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
-		mapping = [
-			{
-				'local_fieldname':'description',
-				'remote_fieldname':'content',
-			},
-			{
-				'local_fieldname': 'role',
-				'remote_fieldname': 'title',
-				'mapping_type': 'Document',
-				'mapping': inner_map,
-				'remote_value_filters': json.dumps({'title': 'title'})
-			}
-		]
-		event_producer.append('producer_doctypes', {
-			'ref_doctype': 'ToDo',
-			'use_same_name': 1,
-			'has_mapping': 1,
-			'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
-		})
-		event_producer.save()
+
+		try:
+			setup_event_producer_for_inner_mapping()
+		except frappe.TimestampMismatchError:
+			# retry - event_producer keeps updating via log so try again
+			setup_event_producer_for_inner_mapping()
 
 		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
 		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
@@ -248,6 +225,39 @@ class TestEventProducer(unittest.TestCase):
 		reset_configuration(producer_url)
 
 
+def setup_event_producer_for_inner_mapping():
+	event_producer = frappe.get_doc('Event Producer', producer_url)
+	event_producer.producer_doctypes = []
+	inner_mapping = [
+		{
+			'local_fieldname':'role_name',
+			'remote_fieldname':'title'
+		}
+	]
+	inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping)
+	mapping = [
+		{
+			'local_fieldname':'description',
+			'remote_fieldname':'content',
+		},
+		{
+			'local_fieldname': 'role',
+			'remote_fieldname': 'title',
+			'mapping_type': 'Document',
+			'mapping': inner_map,
+			'remote_value_filters': json.dumps({'title': 'title'})
+		}
+	]
+	event_producer.append('producer_doctypes', {
+		'ref_doctype': 'ToDo',
+		'use_same_name': 1,
+		'has_mapping': 1,
+		'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping)
+	})
+	event_producer.save()
+	return event_producer
+
+
 def insert_into_producer(producer, description):
 	#create and insert todo on remote site
 	todo = dict(doctype='ToDo', description=description, assigned_by='Administrator')

From 77cc95a6737f71644351c480743ed7db4428f067 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 14:42:30 +0530
Subject: [PATCH 273/361] fix(test): retry test_event_producer

---
 .../doctype/event_producer/test_event_producer.py              | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 9f7b7d8d34..f7a6b54399 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -209,7 +209,8 @@ class TestEventProducer(unittest.TestCase):
 		try:
 			setup_event_producer_for_inner_mapping()
 		except frappe.TimestampMismatchError:
-			# retry - event_producer keeps updating via log so try again
+			# retry - event_producer keeps updating last_updated
+			# so retry if it fails the first time due to a background event
 			setup_event_producer_for_inner_mapping()
 
 		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')

From 72e266f908eb8197a0d166aecae75c859f69d53f Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 16:15:06 +0530
Subject: [PATCH 274/361] fix(tests): cleanup workflow test and commits

---
 .travis.yml                                       |  4 ++--
 frappe/model/workflow.py                          |  6 ++++--
 frappe/workflow/doctype/workflow/test_workflow.py | 13 ++++++-------
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 63895675ea..fde65e44d9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,12 +31,12 @@ matrix:
   - name: "Python 3.7 MariaDB"
     python: 3.7
     env: DB=mariadb TYPE=server
-    script: bench --site test_site run-tests --coverage
+    script: bench --site test_site run-tests --coverage --verbose
 
   - name: "Python 3.7 PostgreSQL"
     python: 3.7
     env: DB=postgres TYPE=server
-    script: bench --site test_site run-tests --coverage
+    script: bench --site test_site run-tests --coverage --verbose
 
   - name: "Cypress"
     python: 3.7
diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py
index 32919b3333..7239b202bd 100644
--- a/frappe/model/workflow.py
+++ b/frappe/model/workflow.py
@@ -272,9 +272,11 @@ def get_common_transition_actions(docs, doctype):
 				doc['doctype'] = doctype
 			actions = [t.get('action') for t in get_transitions(doc, raise_exception=True) \
 				if has_approval_access(frappe.session.user, doc, t)]
-			if not actions: return []
+			if not actions:
+				return []
 			common_actions = actions if i == 1 else set(common_actions).intersection(actions)
-			if not common_actions: return []
+			if not common_actions:
+				return []
 	except WorkflowStateError:
 		pass
 
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 4ffbe6ff8d..bd63af8118 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -15,8 +15,6 @@ class TestWorkflow(unittest.TestCase):
 		make_test_records("User")
 
 	def setUp(self):
-		frappe.db.sql('DELETE FROM `tabToDo`')
-		frappe.db.sql("DELETE FROM `tabHas Role` WHERE `role`='Test Approver'")
 		if not getattr(self, 'workflow', None):
 			self.workflow = create_todo_workflow()
 		frappe.set_user('Administrator')
@@ -129,8 +127,6 @@ def create_todo_workflow():
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)
-		frappe.db.commit()
-		frappe.cache().hdel('roles', frappe.session.user)
 		workflow = frappe.new_doc('Workflow')
 		workflow.workflow_name = 'Test ToDo'
 		workflow.document_type = 'ToDo'
@@ -148,13 +144,16 @@ def create_todo_workflow():
 			state = 'Rejected', allow_edit = 'Test Approver'
 		))
 		workflow.append('transitions', dict(
-			state = 'Pending', action='Approve', next_state = 'Approved', allowed='Test Approver', allow_self_approval= 1
+			state = 'Pending', action='Approve', next_state = 'Approved',
+			allowed='Test Approver', allow_self_approval= 1
 		))
 		workflow.append('transitions', dict(
-			state = 'Pending', action='Reject', next_state = 'Rejected', allowed='Test Approver', allow_self_approval= 1
+			state = 'Pending', action='Reject', next_state = 'Rejected',
+			allowed='Test Approver', allow_self_approval= 1
 		))
 		workflow.append('transitions', dict(
-			state = 'Rejected', action='Review', next_state = 'Pending', allowed='All', allow_self_approval= 1
+			state = 'Rejected', action='Review', next_state = 'Pending',
+			allowed='All', allow_self_approval= 1
 		))
 		workflow.insert(ignore_permissions=True)
 

From 541f7c016dff060a040be1c1f266bfa4ffae22f0 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 16:30:45 +0530
Subject: [PATCH 275/361] fix(tests): cleanup workflow test and commits

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fde65e44d9..2331217363 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,12 +31,12 @@ matrix:
   - name: "Python 3.7 MariaDB"
     python: 3.7
     env: DB=mariadb TYPE=server
-    script: bench --site test_site run-tests --coverage --verbose
+    script: bench --verbose --site test_site run-tests --coverage
 
   - name: "Python 3.7 PostgreSQL"
     python: 3.7
     env: DB=postgres TYPE=server
-    script: bench --site test_site run-tests --coverage --verbose
+    script: bench --verbose --site test_site run-tests --coverage
 
   - name: "Cypress"
     python: 3.7

From 14d9c28eb78ac7654f50cc034b7e0e2b3811b676 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 18:06:10 +0530
Subject: [PATCH 276/361] fix(test): try with print_sql

---
 frappe/__init__.py                            |  5 +-
 frappe/database/database.py                   | 46 +++++++++----------
 .../event_producer/test_event_producer.py     |  2 +
 3 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index 6b50f8ab28..c22899788e 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -267,7 +267,7 @@ def destroy():
 # memcache
 redis_server = None
 def cache():
-	"""Returns memcache connection."""
+	"""Returns redis connection."""
 	global redis_server
 	if not redis_server:
 		from frappe.utils.redis_wrapper import RedisWrapper
@@ -290,6 +290,9 @@ def errprint(msg):
 
 	error_log.append({"exc": msg})
 
+def print_sql(enable=True):
+	return cache().set_value('print_sql', enable)
+
 def log(msg):
 	"""Add to `debug_log`.
 
diff --git a/frappe/database/database.py b/frappe/database/database.py
index 101b97c915..374b61db16 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -134,6 +134,8 @@ class Database(object):
 			if debug:
 				time_start = time()
 
+			self.log_query(query, values, debug, explain)
+
 			if values!=():
 				if isinstance(values, dict):
 					values = dict(values)
@@ -142,41 +144,18 @@ class Database(object):
 				if not isinstance(values, (dict, tuple, list)):
 					values = (values,)
 
-				if debug and query.strip().lower().startswith('select'):
-					try:
-						if explain:
-							self.explain_query(query, values)
-						frappe.errprint(query % values)
-					except TypeError:
-						frappe.errprint([query, values])
-				if (frappe.conf.get("logging") or False)==2:
-					frappe.log("<<<< query")
-					frappe.log(query)
-					frappe.log("with values:")
-					frappe.log(values)
-					frappe.log(">>>>")
 				self._cursor.execute(query, values)
 
 				if frappe.flags.in_migrate:
 					self.log_touched_tables(query, values)
 
 			else:
-				if debug:
-					if explain:
-						self.explain_query(query)
-					frappe.errprint(query)
-				if (frappe.conf.get("logging") or False)==2:
-					frappe.log("<<<< query")
-					frappe.log(query)
-					frappe.log(">>>>")
-
 				self._cursor.execute(query)
 
 				if frappe.flags.in_migrate:
 					self.log_touched_tables(query)
 
 			if debug:
-				frappe.errprint(self._cursor.mogrify(query, values))
 				time_end = time()
 				frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2)))
 
@@ -213,6 +192,27 @@ class Database(object):
 		else:
 			return self._cursor.fetchall()
 
+	def log_query(self, query, values, debug, explain):
+		# for debugging in tests
+		if frappe.flags.in_test and frappe.cache().get_value('print_sql'):
+			print(self._cursor.mogrify(query, values))
+
+		# debug
+		if debug:
+			if explain and query.strip().lower().startswith('select'):
+				self.explain_query(query, values)
+			frappe.errprint(self._cursor.mogrify(query, values))
+
+		# info
+		if (frappe.conf.get("logging") or False)==2:
+			frappe.log("<<<< query")
+			frappe.log(query)
+			if values:
+				frappe.log("with values:")
+				frappe.log(values)
+			frappe.log(">>>>")
+
+
 	def explain_query(self, query, values=None):
 		"""Print `EXPLAIN` in error log."""
 		try:
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index f7a6b54399..441d0d492d 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -206,8 +206,10 @@ class TestEventProducer(unittest.TestCase):
 	def test_inner_mapping(self):
 		producer = get_remote_site()
 
+		frappe.print_sql(True)
 		try:
 			setup_event_producer_for_inner_mapping()
+			frappe.print_sql(False)
 		except frappe.TimestampMismatchError:
 			# retry - event_producer keeps updating last_updated
 			# so retry if it fails the first time due to a background event

From 22c6e08b1b3b53508194421b559716fcff54cc70 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 18:28:23 +0530
Subject: [PATCH 277/361] fix(test): try with print_sql

---
 .../doctype/event_producer/test_event_producer.py             | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 441d0d492d..ea6cbd35f0 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -209,11 +209,13 @@ class TestEventProducer(unittest.TestCase):
 		frappe.print_sql(True)
 		try:
 			setup_event_producer_for_inner_mapping()
-			frappe.print_sql(False)
 		except frappe.TimestampMismatchError:
 			# retry - event_producer keeps updating last_updated
 			# so retry if it fails the first time due to a background event
 			setup_event_producer_for_inner_mapping()
+		finally:
+			frappe.print_sql(False)
+
 
 		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
 		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})

From 4d97635ab4faca182acf5e6c1e492ba0f7292d99 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 19:09:37 +0530
Subject: [PATCH 278/361] fix(major): frappe.db.set_value will lock the row

---
 frappe/database/database.py                   | 31 ++++++++++++-------
 .../doctype/event_producer/event_producer.py  |  3 +-
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 374b61db16..325a1bbbd8 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -378,7 +378,7 @@ class Database(object):
 		return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache)
 
 	def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False,
-		debug=False, order_by=None, cache=False):
+		debug=False, order_by=None, cache=False, for_update=False):
 		"""Returns a document property or list of properties.
 
 		:param doctype: DocType name.
@@ -405,12 +405,12 @@ class Database(object):
 		"""
 
 		ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug,
-			order_by, cache=cache)
+			order_by, cache=cache, for_update=for_update)
 
 		return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None
 
 	def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False,
-		debug=False, order_by=None, update=None, cache=False):
+		debug=False, order_by=None, update=None, cache=False, for_update=False):
 		"""Returns multiple document properties.
 
 		:param doctype: DocType name.
@@ -449,7 +449,7 @@ class Database(object):
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:
-					out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update)
+					out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update, for_update)
 				except Exception as e:
 					if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)):
 						# table or column not found, return None
@@ -576,7 +576,7 @@ class Database(object):
 		"""Alias for get_single_value"""
 		return self.get_single_value(*args, **kwargs)
 
-	def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, update=None):
+	def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, update=None, for_update=False):
 		fl = []
 		if isinstance(fields, (list, tuple)):
 			for f in fields:
@@ -594,9 +594,15 @@ class Database(object):
 
 		order_by = ("order by " + order_by) if order_by else ""
 
-		r = self.sql("select {0} from `tab{1}` {2} {3} {4}"
-			.format(fl, doctype, "where" if conditions else "", conditions, order_by), values,
-			as_dict=as_dict, debug=debug, update=update)
+		r = self.sql("select {for_update} {fields} from `tab{doctype}` {where} {conditions} {order_by}"
+			.format(dict(
+				for_update = 'for update' if for_update else '',
+				fields = fl,
+				doctype = doctype,
+				where = "where" if conditions else "",
+				conditions = conditions,
+				order_by = order_by),
+			values, as_dict=as_dict, debug=debug, update=update)
 
 		return r
 
@@ -616,7 +622,7 @@ class Database(object):
 		return self.set_value(*args, **kwargs)
 
 	def set_value(self, dt, dn, field, val=None, modified=None, modified_by=None,
-		update_modified=True, debug=False):
+		update_modified=True, debug=False, for_update=True):
 		"""Set a single value in the database, do not call the ORM triggers
 		but update the modified timestamp (unless specified not to).
 
@@ -630,6 +636,7 @@ class Database(object):
 		:param modified_by: Set this user as `modified_by`.
 		:param update_modified: default True. Set as false, if you don't want to update the timestamp.
 		:param debug: Print the query in the developer / js console.
+		:param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit.
 		"""
 		if not modified:
 			modified = now()
@@ -647,7 +654,9 @@ class Database(object):
 
 		if dn and dt!=dn:
 			# with table
-			conditions, values = self.build_conditions(dn)
+			values = dict(
+				name=self.get_value(dt, dn, 'name', for_update=for_update)
+			)
 
 			values.update(to_update)
 
@@ -656,7 +665,7 @@ class Database(object):
 				set_values.append('`{0}`=%({0})s'.format(key))
 
 			self.sql("""update `tab{0}`
-				set {1} where {2}""".format(dt, ', '.join(set_values), conditions),
+				set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
 				values, debug=debug)
 
 		else:
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index d4aa0914d8..dccacce4f3 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -216,8 +216,7 @@ def sync(update, producer_site, event_producer, in_retry=False):
 			return 'Failed'
 		log_event_sync(update, event_producer.name, 'Failed', frappe.get_traceback())
 
-	frappe.db.set_value('Event Producer', event_producer.name, 'last_update', update.creation)
-	event_producer.reload()
+	event_producer.db_set('last_update', update.creation)
 	frappe.db.commit()
 
 

From e421bc19f6cd2ed827ebe60ad4057d98211c78f6 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 20:00:24 +0530
Subject: [PATCH 279/361] fix(major): frappe.db.set_value will lock the row

---
 frappe/database/database.py                               | 2 +-
 .../doctype/event_producer/test_event_producer.py         | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 325a1bbbd8..4469623c5a 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -601,7 +601,7 @@ class Database(object):
 				doctype = doctype,
 				where = "where" if conditions else "",
 				conditions = conditions,
-				order_by = order_by),
+				order_by = order_by)),
 			values, as_dict=as_dict, debug=debug, update=update)
 
 		return r
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index ea6cbd35f0..2ebc821075 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -13,8 +13,12 @@ producer_url = 'http://test_site_producer:8000'
 
 class TestEventProducer(unittest.TestCase):
 	def setUp(self):
+		frappe.print_sql(True)
 		create_event_producer(producer_url)
 
+	def tearDown(self):
+		frappe.print_sql(False)
+
 	def test_insert(self):
 		producer = get_remote_site()
 		producer_doc = insert_into_producer(producer, 'test creation 1 sync')
@@ -206,16 +210,12 @@ class TestEventProducer(unittest.TestCase):
 	def test_inner_mapping(self):
 		producer = get_remote_site()
 
-		frappe.print_sql(True)
 		try:
 			setup_event_producer_for_inner_mapping()
 		except frappe.TimestampMismatchError:
 			# retry - event_producer keeps updating last_updated
 			# so retry if it fails the first time due to a background event
 			setup_event_producer_for_inner_mapping()
-		finally:
-			frappe.print_sql(False)
-
 
 		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
 		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})

From eba6dfa12a1a7b316b782e37ca41a831fba8e19f Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 20:10:50 +0530
Subject: [PATCH 280/361] fix(major): frappe.db.set_value will lock the row

---
 frappe/database/database.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 4469623c5a..f5368df98f 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -449,7 +449,7 @@ class Database(object):
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:
-					out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update, for_update)
+					out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update)
 				except Exception as e:
 					if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)):
 						# table or column not found, return None
@@ -594,14 +594,14 @@ class Database(object):
 
 		order_by = ("order by " + order_by) if order_by else ""
 
-		r = self.sql("select {for_update} {fields} from `tab{doctype}` {where} {conditions} {order_by}"
-			.format(dict(
+		r = self.sql("select {fields} from `tab{doctype}` {where} {conditions} {order_by} {for_update}"
+			.format(
 				for_update = 'for update' if for_update else '',
 				fields = fl,
 				doctype = doctype,
 				where = "where" if conditions else "",
 				conditions = conditions,
-				order_by = order_by)),
+				order_by = order_by),
 			values, as_dict=as_dict, debug=debug, update=update)
 
 		return r

From c83a9a142b9cafb77df3fbf241d4f66ba53789b0 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 21:08:58 +0530
Subject: [PATCH 281/361] fix(minor): mogrify

---
 frappe/__init__.py          |  2 +-
 frappe/database/database.py | 17 ++++++++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index c22899788e..36a8b48ecd 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -291,7 +291,7 @@ def errprint(msg):
 	error_log.append({"exc": msg})
 
 def print_sql(enable=True):
-	return cache().set_value('print_sql', enable)
+	return cache().set_value('flag_print_sql', enable)
 
 def log(msg):
 	"""Add to `debug_log`.
diff --git a/frappe/database/database.py b/frappe/database/database.py
index f5368df98f..97e86a03e7 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -193,23 +193,26 @@ class Database(object):
 			return self._cursor.fetchall()
 
 	def log_query(self, query, values, debug, explain):
+		def mogirfy():
+			try:
+				return self._cursor.mogrify(query, values)
+			except TypeError:
+				return [query, values]
+
 		# for debugging in tests
-		if frappe.flags.in_test and frappe.cache().get_value('print_sql'):
-			print(self._cursor.mogrify(query, values))
+		if frappe.flags.in_test and frappe.cache().get_value('flag_print_sql'):
+			print(mogrify())
 
 		# debug
 		if debug:
 			if explain and query.strip().lower().startswith('select'):
 				self.explain_query(query, values)
-			frappe.errprint(self._cursor.mogrify(query, values))
+			frappe.errprint(mogrify())
 
 		# info
 		if (frappe.conf.get("logging") or False)==2:
 			frappe.log("<<<< query")
-			frappe.log(query)
-			if values:
-				frappe.log("with values:")
-				frappe.log(values)
+			frappe.log(mogrify())
 			frappe.log(">>>>")
 
 

From ef1d1b0dc4a8a8a4d021bd4daeae7412bd630be8 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 21:35:10 +0530
Subject: [PATCH 282/361] fix(minor): typo

---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 97e86a03e7..07b063c241 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -193,7 +193,7 @@ class Database(object):
 			return self._cursor.fetchall()
 
 	def log_query(self, query, values, debug, explain):
-		def mogirfy():
+		def mogrify():
 			try:
 				return self._cursor.mogrify(query, values)
 			except TypeError:

From b1971d1f654d548a6ba89b6e64061762d541a1e9 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 22:07:00 +0530
Subject: [PATCH 283/361] fix(minor): add for_update parameter in get_doc

---
 .../doctype/event_producer/test_event_producer.py      | 10 ++--------
 frappe/model/document.py                               |  5 ++++-
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 2ebc821075..8e2ccb4380 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -210,13 +210,7 @@ class TestEventProducer(unittest.TestCase):
 	def test_inner_mapping(self):
 		producer = get_remote_site()
 
-		try:
-			setup_event_producer_for_inner_mapping()
-		except frappe.TimestampMismatchError:
-			# retry - event_producer keeps updating last_updated
-			# so retry if it fails the first time due to a background event
-			setup_event_producer_for_inner_mapping()
-
+		setup_event_producer_for_inner_mapping()
 		producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping')
 		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title})
 		producer_note = producer.insert(producer_note)
@@ -231,7 +225,7 @@ class TestEventProducer(unittest.TestCase):
 
 
 def setup_event_producer_for_inner_mapping():
-	event_producer = frappe.get_doc('Event Producer', producer_url)
+	event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 	event_producer.producer_doctypes = []
 	inner_mapping = [
 		{
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 316c576f55..60c8198eb6 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -103,6 +103,9 @@ class Document(BaseDocument):
 				else:
 					self.name = args[1]
 
+				if 'for_update' in kwargs:
+					self.flags.for_update = True
+
 			self.load_from_db()
 			return
 
@@ -144,7 +147,7 @@ class Document(BaseDocument):
 			self._fix_numeric_types()
 
 		else:
-			d = frappe.db.get_value(self.doctype, self.name, "*", as_dict=1)
+			d = frappe.db.get_value(self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update)
 			if not d:
 				frappe.throw(_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError)
 

From fb7bc2659afb3895d757e7ccae6b4772bee90264 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 22:34:08 +0530
Subject: [PATCH 284/361] fix(minor): allow for_update as a key in get_doc

---
 frappe/model/document.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 60c8198eb6..4531e8ebd0 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -28,6 +28,7 @@ def get_doc(*args, **kwargs):
 
 	:param arg1: Document dict or DocType name.
 	:param arg2: [optional] document name.
+	:param for_update: [optional] select document for update.
 
 	There are multiple ways to call `get_doc`
 
@@ -45,6 +46,9 @@ def get_doc(*args, **kwargs):
 
 		# create new object with keyword arguments
 		user = get_doc(doctype='User', email_id='test@example.com')
+
+		# select a document for update
+		user = get_doc("User", "test@example.com", for_update=True)
 	"""
 	if args:
 		if isinstance(args[0], BaseDocument):
@@ -60,7 +64,7 @@ def get_doc(*args, **kwargs):
 		else:
 			raise ValueError('First non keyword argument must be a string or dict')
 
-	if kwargs:
+	if len(args) < 3 and kwargs:
 		if 'doctype' in kwargs:
 			doctype = kwargs['doctype']
 		else:

From fe9f1e803e31d56a3b2dad454ac1e70c192e941e Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 22:54:53 +0530
Subject: [PATCH 285/361] fix(minor): allow for_update as a key in get_doc

---
 frappe/model/document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 4531e8ebd0..e8da57fc86 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -64,7 +64,7 @@ def get_doc(*args, **kwargs):
 		else:
 			raise ValueError('First non keyword argument must be a string or dict')
 
-	if len(args) < 3 and kwargs:
+	if len(args) < 2 and kwargs:
 		if 'doctype' in kwargs:
 			doctype = kwargs['doctype']
 		else:

From e262ef6add62990bf945e14fecf9c6280fee2920 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 23:16:07 +0530
Subject: [PATCH 286/361] fix(minor): logging fix in document.py

---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 07b063c241..0385065a13 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -196,7 +196,7 @@ class Database(object):
 		def mogrify():
 			try:
 				return self._cursor.mogrify(query, values)
-			except TypeError:
+			except:
 				return [query, values]
 
 		# for debugging in tests

From 73c98d70824377807925b687960c2ae9f9bc963d Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 12 Aug 2020 23:53:51 +0530
Subject: [PATCH 287/361] fix(minor): stop printing

---
 .../doctype/event_producer/test_event_producer.py          | 7 ++++---
 frappe/model/document.py                                   | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 8e2ccb4380..53f5719f84 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -13,11 +13,12 @@ producer_url = 'http://test_site_producer:8000'
 
 class TestEventProducer(unittest.TestCase):
 	def setUp(self):
-		frappe.print_sql(True)
+		# frappe.print_sql(True)
 		create_event_producer(producer_url)
 
-	def tearDown(self):
-		frappe.print_sql(False)
+	# for debugging
+	# def tearDown(self):
+	# 	frappe.print_sql(False)
 
 	def test_insert(self):
 		producer = get_remote_site()
diff --git a/frappe/model/document.py b/frappe/model/document.py
index e8da57fc86..f9b73ffc6f 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -108,7 +108,7 @@ class Document(BaseDocument):
 					self.name = args[1]
 
 				if 'for_update' in kwargs:
-					self.flags.for_update = True
+					self.flags.for_update = kwargs.get('for_update')
 
 			self.load_from_db()
 			return

From 26a9230fddfb5cf326447ac97ae43d1879e23096 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 09:27:02 +0530
Subject: [PATCH 288/361] fix(tests): more debugging

---
 frappe/database/database.py                    | 18 +++++++++---------
 .../event_producer/test_event_producer.py      | 12 ++++++++----
 .../workflow/doctype/workflow/test_workflow.py |  5 +++++
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 0385065a13..c1439ad254 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -193,28 +193,28 @@ class Database(object):
 			return self._cursor.fetchall()
 
 	def log_query(self, query, values, debug, explain):
-		def mogrify():
-			try:
-				return self._cursor.mogrify(query, values)
-			except:
-				return [query, values]
-
 		# for debugging in tests
 		if frappe.flags.in_test and frappe.cache().get_value('flag_print_sql'):
-			print(mogrify())
+			print(self.mogrify(query, values))
 
 		# debug
 		if debug:
 			if explain and query.strip().lower().startswith('select'):
 				self.explain_query(query, values)
-			frappe.errprint(mogrify())
+			frappe.errprint(self.mogrify(query, values))
 
 		# info
 		if (frappe.conf.get("logging") or False)==2:
 			frappe.log("<<<< query")
-			frappe.log(mogrify())
+			frappe.log(self.mogrify(query, values))
 			frappe.log(">>>>")
 
+	def mogrify(self, query, values):
+		'''build the query string with values'''
+		if not values:
+			return query
+		else:
+			return self._cursor.mogrify(query, values)
 
 	def explain_query(self, query, values=None):
 		"""Print `EXPLAIN` in error log."""
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 53f5719f84..f10dc906ba 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -12,13 +12,17 @@ from frappe.event_streaming.doctype.event_producer.event_producer import pull_fr
 producer_url = 'http://test_site_producer:8000'
 
 class TestEventProducer(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		frappe.print_sql(True)
+
+	@classmethod
+	def tearDownClass(cls):
+		frappe.print_sql(False)
+
 	def setUp(self):
-		# frappe.print_sql(True)
 		create_event_producer(producer_url)
 
-	# for debugging
-	# def tearDown(self):
-	# 	frappe.print_sql(False)
 
 	def test_insert(self):
 		producer = get_remote_site()
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index bd63af8118..f23bdc5652 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -13,6 +13,11 @@ class TestWorkflow(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
 		make_test_records("User")
+		frappe.print_sql(True)
+
+	@classmethod
+	def tearDownClass(cls):
+		frappe.print_sql(False)
 
 	def setUp(self):
 		if not getattr(self, 'workflow', None):

From aad2375cf38db0cc5f984787e9a116063b6a8b04 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 10:52:07 +0530
Subject: [PATCH 289/361] fix(major): make document events more efficient, fix
 bug in cache_manager.get_doctype_map

---
 .../assignment_rule/assignment_rule.py        |   9 +-
 .../milestone_tracker/milestone_tracker.py    |   6 +-
 frappe/cache_manager.py                       |   6 +-
 frappe/database/database.py                   |   2 +-
 frappe/desk/form/document_follow.py           |  35 ++---
 frappe/model/__init__.py                      | 120 ++++++++++++++----
 frappe/model/document.py                      |  45 ++++---
 7 files changed, 156 insertions(+), 67 deletions(-)

diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py
index 78f05e7fe9..c64b9af221 100644
--- a/frappe/automation/doctype/assignment_rule/assignment_rule.py
+++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py
@@ -9,6 +9,7 @@ from frappe.model.document import Document
 from frappe.desk.form import assign_to
 import frappe.cache_manager
 from frappe import _
+from frappe.model import log_types
 
 class AssignmentRule(Document):
 
@@ -165,7 +166,13 @@ def reopen_closed_assignment(doc):
 	return True
 
 def apply(doc, method=None, doctype=None, name=None):
-	if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_setup_wizard:
+	if not doctype:
+		doctype = doc.doctype
+
+	if (frappe.flags.in_patch
+		or frappe.flags.in_install
+		or frappe.flags.in_setup_wizard
+		or doctype in log_types):
 		return
 
 	if not doc and doctype and name:
diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
index 154cb599e1..11109fd248 100644
--- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 import frappe.cache_manager
+from frappe.model import log_types
 
 class MilestoneTracker(Document):
 	def on_update(self):
@@ -32,8 +33,11 @@ class MilestoneTracker(Document):
 def evaluate_milestone(doc, event):
 	if (frappe.flags.in_install
 		or frappe.flags.in_migrate
-		or frappe.flags.in_setup_wizard):
+		or frappe.flags.in_setup_wizard
+		or doc.doctype in log_types):
 		return
+
+	# track milestones related to this doctype
 	for d in frappe.cache_manager.get_doctype_map('Milestone Tracker', doc.doctype,
 		dict(document_type = doc.doctype, disabled=0)):
 		frappe.get_doc('Milestone Tracker', d.name).apply(doc)
diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py
index 97b6c235b5..a3ac2b2841 100644
--- a/frappe/cache_manager.py
+++ b/frappe/cache_manager.py
@@ -107,14 +107,14 @@ def get_doctype_map(doctype, name, filters, order_by=None):
 	cache_key = frappe.scrub(doctype) + '_map'
 	doctype_map = cache.hget(cache_key, name)
 
-	if doctype_map:
+	if doctype_map is not None:
 		# cached, return
 		items = json.loads(doctype_map)
 	else:
 		# non cached, build cache
 		try:
 			items = frappe.get_all(doctype, filters=filters, order_by = order_by)
-			cache.hset(cache_key, doctype, json.dumps(items))
+			cache.hset(cache_key, name, json.dumps(items))
 		except frappe.db.TableMissingError:
 			# executed from inside patch, ignore
 			items = []
@@ -122,7 +122,7 @@ def get_doctype_map(doctype, name, filters, order_by=None):
 	return items
 
 def clear_doctype_map(doctype, name):
-	cache_key = frappe.scrub(doctype) + '_map'
+	frappe.scrub(doctype) + '_map'
 	frappe.cache().hdel(cache_key, name)
 
 def build_table_count_cache():
diff --git a/frappe/database/database.py b/frappe/database/database.py
index c1439ad254..8068ad877e 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -194,7 +194,7 @@ class Database(object):
 
 	def log_query(self, query, values, debug, explain):
 		# for debugging in tests
-		if frappe.flags.in_test and frappe.cache().get_value('flag_print_sql'):
+		if frappe.cache().get_value('flag_print_sql'):
 			print(self.mogrify(query, values))
 
 		# debug
diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py
index 333e9e1333..80f614b5b6 100644
--- a/frappe/desk/form/document_follow.py
+++ b/frappe/desk/form/document_follow.py
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 import frappe
 import frappe.utils
 from frappe.utils import get_url_to_form
+from frappe.model import log_types
 from frappe import _
 from itertools import groupby
 
@@ -20,22 +21,26 @@ def follow_document(doctype, doc_name, user, force=False):
 		avoided for some doctype
 		follow only if track changes are set to 1
 	'''
-	avoid_follow = ["Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log",
-		"File", "Version", "View Log", "Document Follow", "Comment"]
+	if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment")
+		or doctype in log_types):
+		return
 
-	track_changes = frappe.get_meta(doctype).track_changes
-	exists = is_document_followed(doctype, doc_name, user)
-	if exists == 0:
-		user_can_follow = frappe.db.get_value("User", user, "document_follow_notify", ignore=True)
-		if user != "Administrator" and user_can_follow and track_changes and (doctype not in avoid_follow or force):
-			doc = frappe.new_doc("Document Follow")
-			doc.update({
-				"ref_doctype": doctype,
-				"ref_docname": doc_name,
-				"user": user
-			})
-			doc.save()
-			return doc
+	if ((not frappe.get_meta(doctype).track_changes)
+		or user == "Administrator"):
+		return
+
+	if not frappe.db.get_value("User", user, "document_follow_notify", ignore=True, cache=True):
+		return
+
+	if not is_document_followed(doctype, doc_name, user):
+		doc = frappe.new_doc("Document Follow")
+		doc.update({
+			"ref_doctype": doctype,
+			"ref_docname": doc_name,
+			"user": user
+		})
+		doc.save()
+		return doc
 
 @frappe.whitelist()
 def unfollow_document(doctype, doc_name, user):
diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py
index 3c5d996439..e59d325c9a 100644
--- a/frappe/model/__init__.py
+++ b/frappe/model/__init__.py
@@ -38,36 +38,104 @@ data_fieldtypes = (
 	'Duration'
 )
 
-no_value_fields = ('Section Break', 'Column Break', 'HTML', 'Table', 'Table MultiSelect', 'Button', 'Image',
-	'Fold', 'Heading')
-display_fieldtypes = ('Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading')
-numeric_fieldtypes = ('Currency', 'Int', 'Long Int', 'Float', 'Percent', 'Check')
-default_fields = ('doctype','name','owner','creation','modified','modified_by',
-	'parent','parentfield','parenttype','idx','docstatus')
-optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen")
-table_fields = ('Table', 'Table MultiSelect')
-core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link', 'User', 'Role', 'Has Role',
-	'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',
-	'Customize Form Field', 'Property Setter', 'Custom Field', 'Custom Script')
-data_field_options = ('Email', 'Name', 'Phone')
+no_value_fields = (
+	'Section Break',
+	'Column Break',
+	'HTML',
+	'Table',
+	'Table MultiSelect',
+	'Button',
+	'Image',
+	'Fold',
+	'Heading'
+)
 
-def copytables(srctype, src, srcfield, tartype, tar, tarfield, srcfields, tarfields=[]):
-	if not tarfields:
-		tarfields = srcfields
-	l = []
-	data = src.get(srcfield)
-	for d in data:
-		newrow = tar.append(tarfield)
-		newrow.idx = d.idx
+display_fieldtypes = (
+	'Section Break',
+	'Column Break',
+	'HTML',
+	'Button',
+	'Image',
+	'Fold',
+	'Heading')
 
-		for i in range(len(srcfields)):
-			newrow.set(tarfields[i], d.get(srcfields[i]))
+numeric_fieldtypes = (
+	'Currency',
+	'Int',
+	'Long Int',
+	'Float',
+	'Percent',
+	'Check'
+)
 
-		l.append(newrow)
-	return l
+data_field_options = (
+	'Email',
+	'Name',
+	'Phone'
+)
 
-def db_exists(dt, dn):
-	return frappe.db.exists(dt, dn)
+default_fields = (
+	'doctype',
+	'name',
+	'owner',
+	'creation',
+	'modified',
+	'modified_by',
+	'parent',
+	'parentfield',
+	'parenttype',
+	'idx',
+	'docstatus'
+)
+
+optional_fields = (
+	"_user_tags",
+	"_comments",
+	"_assign",
+	"_liked_by",
+	"_seen"
+)
+
+table_fields = (
+	'Table',
+	'Table MultiSelect'
+)
+
+core_doctypes_list = (
+	'DocType',
+	'DocField',
+	'DocPerm',
+	'DocType Action',
+	'DocType Link',
+	'User',
+	'Role',
+	'Has Role',
+	'Page',
+	'Module Def',
+	'Print Format',
+	'Report',
+	'Customize Form',
+	'Customize Form Field',
+	'Property Setter',
+	'Custom Field',
+	'Custom Script'
+)
+
+log_types = (
+	'Version',
+	'Error Log',
+	'Scheduled Job Log',
+	'Event Sync Log',
+	'Event Update Log',
+	'Access Log',
+	'View Log',
+	'Activity Log',
+	'Energy Point Log',
+	'Notification Log',
+	'Email Queue',
+	'DocShare',
+	'Document Follow'
+)
 
 def delete_fields(args_dict, delete=0):
 	"""
diff --git a/frappe/model/document.py b/frappe/model/document.py
index f9b73ffc6f..556f43fd4f 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -240,9 +240,6 @@ class Document(BaseDocument):
 		self.set_docstatus()
 		self.flags.in_insert = False
 
-		# follow document on document creation
-
-
 		# run validate, on update etc.
 
 		# parent
@@ -980,28 +977,14 @@ class Document(BaseDocument):
 
 		update_global_search(self)
 
-		if getattr(self.meta, 'track_changes', False) and not self.flags.ignore_version \
-			and not self.doctype == 'Version' and not frappe.flags.in_install:
-			self.save_version()
+		self.save_version()
 
 		self.run_method('on_change')
 
 		if (self.doctype, self.name) in frappe.flags.currently_saving:
 			frappe.flags.currently_saving.remove((self.doctype, self.name))
 
-		# make event update log for doctypes having event consumers
-		if not frappe.flags.in_install and not frappe.flags.in_migrate and check_doctype_has_consumers(self.doctype):
-			if self.flags.update_log_for_doc_creation:
-				make_event_update_log(self, update_type='Create')
-				self.flags.update_log_for_doc_creation = False
-			else:
-				from frappe.event_streaming.doctype.event_update_log.event_update_log import get_update
-				diff = get_update(doc_before_save, self)
-				if diff:
-					doc = self
-					doc.diff = diff
-					make_event_update_log(doc, update_type='Update')
-
+		self.notify_consumers()
 		self.latest = None
 
 	def clear_cache(self):
@@ -1026,6 +1009,20 @@ class Document(BaseDocument):
 			}
 			frappe.publish_realtime("list_update", data, after_commit=True)
 
+	def notify_consumers(self):
+		# make event update log for doctypes having event consumers
+		if (not frappe.flags.in_install and not frappe.flags.in_migrate
+			and check_doctype_has_consumers(self.doctype)):
+			if self.flags.update_log_for_doc_creation:
+				make_event_update_log(self, update_type='Create')
+				self.flags.update_log_for_doc_creation = False
+			else:
+				from frappe.event_streaming.doctype.event_update_log.event_update_log import get_update
+				diff = get_update(doc_before_save, self)
+				if diff:
+					self.diff = diff
+					make_event_update_log(self, update_type='Update')
+
 	def db_set(self, fieldname, value=None, update_modified=True, notify=False, commit=False):
 		"""Set a value in the document object, update the timestamp and update the database.
 
@@ -1078,7 +1075,14 @@ class Document(BaseDocument):
 
 	def save_version(self):
 		"""Save version info"""
-		if not self._doc_before_save and frappe.flags.in_patch: return
+
+		# don't track version under following conditions
+		if (not getattr(self.meta, 'track_changes', False)
+			or self.doctype == 'Version'
+			or self.flags.ignore_version
+			or frappe.flags.in_install
+			or (not self._doc_before_save and frappe.flags.in_patch)):
+			return
 
 		version = frappe.new_doc('Version')
 		if not self._doc_before_save:
@@ -1087,6 +1091,7 @@ class Document(BaseDocument):
 		elif version.set_diff(self._doc_before_save, self):
 			version.insert(ignore_permissions=True)
 			if not frappe.flags.in_migrate:
+				# follow since you made a change?
 				follow_document(self.doctype, self.name, frappe.session.user)
 
 	@staticmethod

From a9a09794655e447ff2fe51c31223d10b370a95dd Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 11:36:29 +0530
Subject: [PATCH 290/361] fix(minor): cache_key in cache_manager

---
 frappe/cache_manager.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py
index a3ac2b2841..3586ff153f 100644
--- a/frappe/cache_manager.py
+++ b/frappe/cache_manager.py
@@ -122,8 +122,7 @@ def get_doctype_map(doctype, name, filters, order_by=None):
 	return items
 
 def clear_doctype_map(doctype, name):
-	frappe.scrub(doctype) + '_map'
-	frappe.cache().hdel(cache_key, name)
+	frappe.cache().hdel(frappe.scrub(doctype) + '_map', name)
 
 def build_table_count_cache():
 	if (frappe.flags.in_patch

From 3d6b3870bd4b97e902dc36e64ea4d4a40426920d Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 11:42:06 +0530
Subject: [PATCH 291/361] fix(minor): document.py:notify_consumers

---
 frappe/model/document.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 556f43fd4f..9d79de2acb 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -984,7 +984,7 @@ class Document(BaseDocument):
 		if (self.doctype, self.name) in frappe.flags.currently_saving:
 			frappe.flags.currently_saving.remove((self.doctype, self.name))
 
-		self.notify_consumers()
+		self.notify_consumers(doc_before_save)
 		self.latest = None
 
 	def clear_cache(self):
@@ -1009,7 +1009,7 @@ class Document(BaseDocument):
 			}
 			frappe.publish_realtime("list_update", data, after_commit=True)
 
-	def notify_consumers(self):
+	def notify_consumers(self, doc_before_save):
 		# make event update log for doctypes having event consumers
 		if (not frappe.flags.in_install and not frappe.flags.in_migrate
 			and check_doctype_has_consumers(self.doctype)):

From b4b620e7389c0a0b47e28f5a9679ef0211e31433 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 12:11:46 +0530
Subject: [PATCH 292/361] fix(refactor): event_update_log - move code from
 document.py and cleanup hooks

---
 .../doctype/event_producer/event_producer.py  |  5 ++
 .../event_update_log/event_update_log.py      | 51 ++++++++++++++++---
 frappe/hooks.py                               |  6 +--
 frappe/model/document.py                      | 45 ----------------
 4 files changed, 50 insertions(+), 57 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index dccacce4f3..bf47b8e29e 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -38,6 +38,11 @@ class EventProducer(Document):
 			self.db_set('incoming_change', 0)
 			self.reload()
 
+		frappe.cache_manager.clear_doctype_map('Event Producer', self.name)
+
+	def on_trash(self):
+		frappe.cache_manager.clear_doctype_map('Event Producer', self.name)
+
 	def check_url(self):
 		if not validate_url(self.producer_url):
 			frappe.throw(_('Invalid URL'))
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index 21e3b303e8..db76d21738 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -9,17 +9,38 @@ from frappe.utils.background_jobs import get_jobs
 from frappe.model import no_value_fields, table_fields
 
 class EventUpdateLog(Document):
-	pass
+	def after_insert(self):
+		"""Send update notification updates to event consumers
+		whenever update log is generated"""
+		enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers'
+		jobs = get_jobs()
+		if not jobs or enqueued_method not in jobs[frappe.local.site]:
+			frappe.enqueue(enqueued_method, doctype=doc.ref_doctype, queue='long',
+				enqueue_after_commit=True)
 
+def notify_consumers(doc, event):
+	'''called via triggers'''
+	# make event update log for doctypes having event consumers
+	if frappe.flags.in_install or frappe.flags.in_migrate:
+		return
 
-def notify_consumers(doc, _method=None):
-	"""Send update notification updates to event consumers
-	whenever update log is generated"""
-	enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers'
-	jobs = get_jobs()
-	if not jobs or enqueued_method not in jobs[frappe.local.site]:
-		frappe.enqueue(enqueued_method, doctype=doc.ref_doctype, queue='long', enqueue_after_commit=True)
+	consumers = check_doctype_has_consumers(doc.doctype)
 
+	if consumers:
+		doc_before_save = doc.get_doc_before_save()
+		if doc.flags.update_log_for_doc_creation:
+			make_event_update_log(doc, update_type='Create')
+			doc.flags.update_log_for_doc_creation = False
+		else:
+			diff = get_update(doc_before_save, doc)
+			if diff:
+				doc.diff = diff
+				make_event_update_log(doc, update_type='Update')
+
+def check_doctype_has_consumers(doctype):
+	"""Check if doctype has event consumers for event streaming"""
+	return frappe.cache_manager.get_doctype_map('Event Producer', doctype,
+		dict(ref_doctype = doctype, status='Approved'))
 
 def get_update(old, new, for_child=False):
 	"""
@@ -60,6 +81,20 @@ def get_update(old, new, for_child=False):
 		return out
 	return None
 
+def make_event_update_log(doc, update_type):
+	"""Save update info for doctypes that have event consumers"""
+	if update_type != 'Delete':
+		# diff for update type, doc for create type
+		data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff)
+	else:
+		data = None
+	log_doc = frappe.get_doc({
+		'doctype': 'Event Update Log',
+		'update_type': update_type,
+		'ref_doctype': doc.doctype,
+		'docname': doc.name,
+		'data': data
+	}).insert(ignore_permissions=True)
 
 def make_maps(old_value, new_value):
 	"""make maps"""
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 894e72a121..f50ad0b374 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -132,7 +132,8 @@ doc_events = {
 			"frappe.core.doctype.activity_log.feed.update_feed",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.apply",
-			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone"
+			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone",
+			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
 		],
 		"after_rename": "frappe.desk.notifications.clear_doctype_notifications",
 		"on_cancel": [
@@ -163,9 +164,6 @@ doc_events = {
 	"Page": {
 		"after_insert": "frappe.cache_manager.build_domain_restriced_page_cache",
 		"after_save": "frappe.cache_manager.build_domain_restriced_page_cache",
-	},
-	"Event Update Log": {
-		"after_insert": "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
 	}
 }
 
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 9d79de2acb..2b171547d1 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -984,7 +984,6 @@ class Document(BaseDocument):
 		if (self.doctype, self.name) in frappe.flags.currently_saving:
 			frappe.flags.currently_saving.remove((self.doctype, self.name))
 
-		self.notify_consumers(doc_before_save)
 		self.latest = None
 
 	def clear_cache(self):
@@ -1009,20 +1008,6 @@ class Document(BaseDocument):
 			}
 			frappe.publish_realtime("list_update", data, after_commit=True)
 
-	def notify_consumers(self, doc_before_save):
-		# make event update log for doctypes having event consumers
-		if (not frappe.flags.in_install and not frappe.flags.in_migrate
-			and check_doctype_has_consumers(self.doctype)):
-			if self.flags.update_log_for_doc_creation:
-				make_event_update_log(self, update_type='Create')
-				self.flags.update_log_for_doc_creation = False
-			else:
-				from frappe.event_streaming.doctype.event_update_log.event_update_log import get_update
-				diff = get_update(doc_before_save, self)
-				if diff:
-					self.diff = diff
-					make_event_update_log(self, update_type='Update')
-
 	def db_set(self, fieldname, value=None, update_modified=True, notify=False, commit=False):
 		"""Set a value in the document object, update the timestamp and update the database.
 
@@ -1348,34 +1333,4 @@ def execute_action(doctype, name, action, **kwargs):
 		doc.notify_update()
 
 
-def make_event_update_log(doc, update_type):
-	"""Save update info for doctypes that have event consumers"""
-	if update_type != 'Delete':
-		# diff for update type, doc for create type
-		data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff)
-	else:
-		data = None
-	log_doc = frappe.get_doc({
-		'doctype': 'Event Update Log',
-		'update_type': update_type,
-		'ref_doctype': doc.doctype,
-		'docname': doc.name,
-		'data': data
-	})
-	log_doc.insert(ignore_permissions=True)
-	frappe.db.commit()
 
-
-def check_doctype_has_consumers(doctype):
-	"""Check if doctype has event consumers for event streaming"""
-	if not frappe.db.exists('DocType', 'Event Consumer'):
-		return False
-
-	event_consumers = frappe.get_all('Event Consumer Document Type', {
-		'ref_doctype': doctype,
-		'status': 'Approved'
-	}, limit=1)
-
-	if len(event_consumers) and event_consumers[0]:
-		return True
-	return False

From b09beb914777972ccd04eb2bfbbd6b52250475ab Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 13:02:30 +0530
Subject: [PATCH 293/361] fix(tests): cleanup

---
 .../milestone_tracker/milestone_tracker.py    | 10 +++++---
 .../test_milestone_tracker.py                 |  9 +++++++-
 frappe/database/database.py                   |  2 +-
 .../event_update_log/event_update_log.py      | 23 +++++++++++--------
 frappe/hooks.py                               |  6 ++++-
 frappe/model/delete_doc.py                    |  5 ----
 .../energy_point_rule/energy_point_rule.py    |  4 +++-
 .../doctype/workflow/test_workflow.py         | 12 +++++-----
 8 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
index 11109fd248..8d1eebfef8 100644
--- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
@@ -38,6 +38,10 @@ def evaluate_milestone(doc, event):
 		return
 
 	# track milestones related to this doctype
-	for d in frappe.cache_manager.get_doctype_map('Milestone Tracker', doc.doctype,
-		dict(document_type = doc.doctype, disabled=0)):
-		frappe.get_doc('Milestone Tracker', d.name).apply(doc)
+	for d in get_milestone_trackers(doc.doctype):
+		frappe.get_doc('Milestone Tracker', d.get('name')).apply(doc)
+
+def get_milestone_trackers(doctype):
+	return frappe.cache_manager.get_doctype_map('Milestone Tracker', doctype,
+		dict(document_type = doctype, disabled=0))
+
diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
index c9bb6b7d5f..9476c566d6 100644
--- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
@@ -4,11 +4,17 @@
 from __future__ import unicode_literals
 
 import frappe
+import frappe.cache_manager
 import unittest
 
+from .milestone_tracker import get_milestone_trackers
+
 class TestMilestoneTracker(unittest.TestCase):
 	def test_milestone(self):
 		frappe.db.sql('delete from `tabMilestone Tracker`')
+
+		frappe.cache().delete_key('milestone_tracker_map')
+
 		frappe.get_doc(dict(
 			doctype = 'Milestone Tracker',
 			document_type = 'ToDo',
@@ -17,7 +23,8 @@ class TestMilestoneTracker(unittest.TestCase):
 
 		todo = frappe.get_doc(dict(
 			doctype = 'ToDo',
-			description = 'test milestone'
+			description = 'test milestone',
+			status = 'Open'
 		)).insert()
 
 		milestones = frappe.get_all('Milestone',
diff --git a/frappe/database/database.py b/frappe/database/database.py
index 8068ad877e..8332a71168 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -194,7 +194,7 @@ class Database(object):
 
 	def log_query(self, query, values, debug, explain):
 		# for debugging in tests
-		if frappe.cache().get_value('flag_print_sql'):
+		if frappe.conf.get('allow_tests') and frappe.cache().get_value('flag_print_sql'):
 			print(self.mogrify(query, values))
 
 		# debug
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index db76d21738..4fd64394cb 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -19,7 +19,7 @@ class EventUpdateLog(Document):
 				enqueue_after_commit=True)
 
 def notify_consumers(doc, event):
-	'''called via triggers'''
+	'''called via hooks'''
 	# make event update log for doctypes having event consumers
 	if frappe.flags.in_install or frappe.flags.in_migrate:
 		return
@@ -28,14 +28,19 @@ def notify_consumers(doc, event):
 
 	if consumers:
 		doc_before_save = doc.get_doc_before_save()
-		if doc.flags.update_log_for_doc_creation:
-			make_event_update_log(doc, update_type='Create')
-			doc.flags.update_log_for_doc_creation = False
+
+		if event=='after_insert':
+			doc.flags.event_update_log = make_event_update_log(doc, update_type='Create')
+		elif event=='on_trash':
+			make_event_update_log(doc, update_type='Delete')
 		else:
-			diff = get_update(doc_before_save, doc)
-			if diff:
-				doc.diff = diff
-				make_event_update_log(doc, update_type='Update')
+			# on_update
+			# called after saving
+			if not doc.flags.event_update_log: # if not already inserted
+				diff = get_update(doc_before_save, doc)
+				if diff:
+					doc.diff = diff
+					make_event_update_log(doc, update_type='Update')
 
 def check_doctype_has_consumers(doctype):
 	"""Check if doctype has event consumers for event streaming"""
@@ -88,7 +93,7 @@ def make_event_update_log(doc, update_type):
 		data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff)
 	else:
 		data = None
-	log_doc = frappe.get_doc({
+	return frappe.get_doc({
 		'doctype': 'Event Update Log',
 		'update_type': update_type,
 		'ref_doctype': doc.doctype,
diff --git a/frappe/hooks.py b/frappe/hooks.py
index f50ad0b374..6277998b04 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -127,6 +127,9 @@ standard_queries = {
 
 doc_events = {
 	"*": {
+		"after_insert": [
+			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
+		],
 		"on_update": [
 			"frappe.desk.notifications.clear_doctype_notifications",
 			"frappe.core.doctype.activity_log.feed.update_feed",
@@ -142,7 +145,8 @@ doc_events = {
 		],
 		"on_trash": [
 			"frappe.desk.notifications.clear_doctype_notifications",
-			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions"
+			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
+			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
 		],
 		"on_change": [
 			"frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points"
diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py
index 98dbce1d8f..a38470e3f5 100644
--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@@ -18,7 +18,6 @@ from frappe.model.naming import revert_series_if_last
 from frappe.utils.global_search import delete_for_document
 from frappe.desk.doctype.tag.tag import delete_tags_for_document
 from frappe.exceptions import FileNotFoundError
-from frappe.model.document import make_event_update_log, check_doctype_has_consumers
 
 doctypes_to_skip = ("Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", "File",
 	"Version", "Document Follow", "Comment" , "View Log", "Tag Link", "Notification Log", "Email Queue")
@@ -121,10 +120,6 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
 		# delete tag link entry
 		delete_tags_for_document(doc)
 
-		# update log if doctype has event consumers
-		if not frappe.flags.in_install and not frappe.flags.in_migrate and check_doctype_has_consumers(doc.doctype):
-			make_event_update_log(doc, update_type='Delete')
-
 		if doc and not for_reload:
 			add_to_deleted_document(doc)
 			if not frappe.flags.in_patch:
diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py
index d04448dc0f..c3ebdffde0 100644
--- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py
+++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 import frappe.cache_manager
+from frappe.model import log_types
 from frappe.model.document import Document
 from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled
 from frappe.social.doctype.energy_point_log.energy_point_log import \
@@ -85,7 +86,8 @@ def process_energy_points(doc, state):
 		or frappe.flags.in_install
 		or frappe.flags.in_migrate
 		or frappe.flags.in_import
-		or frappe.flags.in_setup_wizard):
+		or frappe.flags.in_setup_wizard
+		or doc.doctype in log_types):
 		return
 
 	if not is_energy_point_enabled():
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index f23bdc5652..22a9c97c1f 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -13,17 +13,15 @@ class TestWorkflow(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
 		make_test_records("User")
-		frappe.print_sql(True)
-
-	@classmethod
-	def tearDownClass(cls):
-		frappe.print_sql(False)
 
 	def setUp(self):
 		if not getattr(self, 'workflow', None):
 			self.workflow = create_todo_workflow()
 		frappe.set_user('Administrator')
 
+	def tearDown(self):
+		self.workflow.db_set('is_active', 0)
+
 	def test_default_condition(self):
 		'''test default condition is set'''
 		todo = create_new_todo()
@@ -128,7 +126,9 @@ class TestWorkflow(unittest.TestCase):
 
 def create_todo_workflow():
 	if frappe.db.exists('Workflow', 'Test ToDo'):
-		return frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True)
+		workflow = frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True)
+		workflow.db_set('is_active', 1)
+		return workflow
 	else:
 		frappe.get_doc(dict(doctype='Role',
 			role_name='Test Approver')).insert(ignore_if_duplicate=True)

From 5ead49c21297e0e305a575b0e880f202c758031a Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 14:15:54 +0530
Subject: [PATCH 294/361] fix(tests): fix caching for event_consumer.py

---
 .../doctype/event_consumer/event_consumer.py         |  5 +++++
 .../doctype/event_producer/event_producer.py         |  2 --
 .../doctype/event_producer/test_event_producer.py    | 12 ++++++------
 .../doctype/event_update_log/event_update_log.py     |  9 +++------
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
index bd285e5401..a3c5f1fd05 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
@@ -25,6 +25,11 @@ class EventConsumer(Document):
 		else:
 			frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
 
+		frappe.cache_manager.clear_doctype_map('Event Consumer', self.name)
+
+	def on_trash(self):
+		frappe.cache_manager.clear_doctype_map('Event Consumer', self.name)
+
 	def update_consumer_status(self):
 		consumer_site = get_consumer_site(self.callback_url)
 		event_producer = consumer_site.get_doc('Event Producer', get_url())
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index bf47b8e29e..80b14b856d 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -38,8 +38,6 @@ class EventProducer(Document):
 			self.db_set('incoming_change', 0)
 			self.reload()
 
-		frappe.cache_manager.clear_doctype_map('Event Producer', self.name)
-
 	def on_trash(self):
 		frappe.cache_manager.clear_doctype_map('Event Producer', self.name)
 
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index f10dc906ba..e5115ece4a 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -12,13 +12,13 @@ from frappe.event_streaming.doctype.event_producer.event_producer import pull_fr
 producer_url = 'http://test_site_producer:8000'
 
 class TestEventProducer(unittest.TestCase):
-	@classmethod
-	def setUpClass(cls):
-		frappe.print_sql(True)
+	# @classmethod
+	# def setUpClass(cls):
+	# 	frappe.print_sql(True)
 
-	@classmethod
-	def tearDownClass(cls):
-		frappe.print_sql(False)
+	# @classmethod
+	# def tearDownClass(cls):
+	# 	frappe.print_sql(False)
 
 	def setUp(self):
 		create_event_producer(producer_url)
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index 4fd64394cb..4b655172f1 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -15,7 +15,7 @@ class EventUpdateLog(Document):
 		enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers'
 		jobs = get_jobs()
 		if not jobs or enqueued_method not in jobs[frappe.local.site]:
-			frappe.enqueue(enqueued_method, doctype=doc.ref_doctype, queue='long',
+			frappe.enqueue(enqueued_method, doctype=self.ref_doctype, queue='long',
 				enqueue_after_commit=True)
 
 def notify_consumers(doc, event):
@@ -25,10 +25,7 @@ def notify_consumers(doc, event):
 		return
 
 	consumers = check_doctype_has_consumers(doc.doctype)
-
 	if consumers:
-		doc_before_save = doc.get_doc_before_save()
-
 		if event=='after_insert':
 			doc.flags.event_update_log = make_event_update_log(doc, update_type='Create')
 		elif event=='on_trash':
@@ -37,14 +34,14 @@ def notify_consumers(doc, event):
 			# on_update
 			# called after saving
 			if not doc.flags.event_update_log: # if not already inserted
-				diff = get_update(doc_before_save, doc)
+				diff = get_update(doc.get_doc_before_save(), doc)
 				if diff:
 					doc.diff = diff
 					make_event_update_log(doc, update_type='Update')
 
 def check_doctype_has_consumers(doctype):
 	"""Check if doctype has event consumers for event streaming"""
-	return frappe.cache_manager.get_doctype_map('Event Producer', doctype,
+	return frappe.cache_manager.get_doctype_map('Event Consumer', doctype,
 		dict(ref_doctype = doctype, status='Approved'))
 
 def get_update(old, new, for_child=False):

From b92b97fd9ae4abec47c5543e6a282df5b7b82956 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 14:59:42 +0530
Subject: [PATCH 295/361] fix(minor): clear cache for maps

---
 .../doctype/assignment_rule/assignment_rule.py        |  7 +++++--
 .../doctype/milestone_tracker/milestone_tracker.py    |  4 ++--
 .../milestone_tracker/test_milestone_tracker.py       |  5 ++++-
 frappe/cache_manager.py                               | 11 +++++++----
 .../doctype/event_consumer/event_consumer.py          |  4 ++--
 .../doctype/event_producer/event_producer.py          |  3 ---
 .../doctype/energy_point_log/test_energy_point_log.py |  4 ++++
 .../doctype/energy_point_rule/energy_point_rule.py    |  5 +++--
 8 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py
index c64b9af221..0120687461 100644
--- a/frappe/automation/doctype/assignment_rule/assignment_rule.py
+++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py
@@ -20,10 +20,13 @@ class AssignmentRule(Document):
 			frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days)))
 
 	def on_update(self): # pylint: disable=no-self-use
-		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.name)
+		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
 
 	def after_rename(self, old, new, merge): # pylint: disable=no-self-use
-		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.name)
+		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
+
+	def on_trash(self, old, new, merge): # pylint: disable=no-self-use
+		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
 
 	def apply_unassign(self, doc, assignments):
 		if (self.unassign_condition and
diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
index 8d1eebfef8..388620bfb4 100644
--- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py
@@ -11,10 +11,10 @@ from frappe.model import log_types
 
 class MilestoneTracker(Document):
 	def on_update(self):
-		frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.name)
+		frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type)
 
 	def on_trash(self):
-		frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.name)
+		frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type)
 
 	def apply(self, doc):
 		before_save = doc.get_doc_before_save()
diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
index 9476c566d6..34b6a608a9 100644
--- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
@@ -15,7 +15,7 @@ class TestMilestoneTracker(unittest.TestCase):
 
 		frappe.cache().delete_key('milestone_tracker_map')
 
-		frappe.get_doc(dict(
+		milestone_tracker = frappe.get_doc(dict(
 			doctype = 'Milestone Tracker',
 			document_type = 'ToDo',
 			track_field = 'status'
@@ -47,3 +47,6 @@ class TestMilestoneTracker(unittest.TestCase):
 		self.assertEqual(milestones[0].track_field, 'status')
 		self.assertEqual(milestones[0].value, 'Closed')
 
+		# cleanup
+		frappe.db.sql('delete from tabMilestone')
+		milestone_tracker.delete()
\ No newline at end of file
diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py
index 3586ff153f..a58ee90632 100644
--- a/frappe/cache_manager.py
+++ b/frappe/cache_manager.py
@@ -11,12 +11,15 @@ from frappe.desk.notifications import (delete_notification_count_for,
 
 common_default_keys = ["__default", "__global"]
 
+doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map',
+	'milestone_tracker_map', 'event_consumer_map')
+
 global_cache_keys = ("app_hooks", "installed_apps",
 		"app_modules", "module_app", "system_settings",
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
 		'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version',
 		'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts',
-		'sitemap_routes', 'db_tables')
+		'sitemap_routes', 'db_tables') + doctype_map_keys
 
 user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
 		"defaults", "user_permissions", "home_page", "linked_with",
@@ -24,8 +27,8 @@ user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
 		"has_role:Page", "has_role:Report", "desk_sidebar_items")
 
 doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified",
-		"linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map', 'data_import_column_header_map')
-
+		"linked_doctypes", 'notifications', 'workflow' ,
+		'data_import_column_header_map') + doctype_map_keys
 
 def clear_user_cache(user=None):
 	cache = frappe.cache()
@@ -102,7 +105,7 @@ def clear_doctype_cache(doctype=None):
 	# Clear all document's cache. To clear documents of a specific DocType document_cache should be restructured
 	clear_document_cache()
 
-def get_doctype_map(doctype, name, filters, order_by=None):
+def get_doctype_map(doctype, name, filters=None, order_by=None):
 	cache = frappe.cache()
 	cache_key = frappe.scrub(doctype) + '_map'
 	doctype_map = cache.hget(cache_key, name)
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
index a3c5f1fd05..aa7b49034e 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
@@ -25,10 +25,10 @@ class EventConsumer(Document):
 		else:
 			frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
 
-		frappe.cache_manager.clear_doctype_map('Event Consumer', self.name)
+		frappe.cache_manager.clear_doctype_map('Event Consumer', self.ref_doctype)
 
 	def on_trash(self):
-		frappe.cache_manager.clear_doctype_map('Event Consumer', self.name)
+		frappe.cache_manager.clear_doctype_map('Event Consumer', self.ref_doctype)
 
 	def update_consumer_status(self):
 		consumer_site = get_consumer_site(self.callback_url)
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index 80b14b856d..dccacce4f3 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -38,9 +38,6 @@ class EventProducer(Document):
 			self.db_set('incoming_change', 0)
 			self.reload()
 
-	def on_trash(self):
-		frappe.cache_manager.clear_doctype_map('Event Producer', self.name)
-
 	def check_url(self):
 		if not validate_url(self.producer_url):
 			frappe.throw(_('Invalid URL'))
diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py
index 1ecaba9cd5..9ccef02eb3 100644
--- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py
+++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py
@@ -10,10 +10,14 @@ from frappe.utils.testutils import add_custom_field, clear_custom_fields
 from frappe.desk.form.assign_to import add as assign_to
 
 class TestEnergyPointLog(unittest.TestCase):
+	def setUp(self):
+		frappe.cache().delete_value('energy_point_rule_map')
+
 	def tearDown(self):
 		frappe.set_user('Administrator')
 		frappe.db.sql('DELETE FROM `tabEnergy Point Log`')
 		frappe.db.sql('DELETE FROM `tabEnergy Point Rule`')
+		frappe.cache().delete_value('energy_point_rule_map')
 
 	def test_user_energy_point(self):
 		frappe.set_user('test@example.com')
diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py
index c3ebdffde0..f21a12bdda 100644
--- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py
+++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py
@@ -14,10 +14,11 @@ from frappe.social.doctype.energy_point_log.energy_point_log import \
 
 class EnergyPointRule(Document):
 	def on_update(self):
-		frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.name)
+		print('clearing...')
+		frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.reference_doctype)
 
 	def on_trash(self):
-		frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.name)
+		frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.reference_doctype)
 
 	def apply(self, doc):
 		if self.rule_condition_satisfied(doc):

From 4585917dc96d7ee59a42b02951f004287c815bc1 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 15:38:08 +0530
Subject: [PATCH 296/361] fix(minor): extra params on on_trash

---
 frappe/automation/doctype/assignment_rule/assignment_rule.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py
index 0120687461..c4bd49b870 100644
--- a/frappe/automation/doctype/assignment_rule/assignment_rule.py
+++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py
@@ -25,7 +25,7 @@ class AssignmentRule(Document):
 	def after_rename(self, old, new, merge): # pylint: disable=no-self-use
 		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
 
-	def on_trash(self, old, new, merge): # pylint: disable=no-self-use
+	def on_trash(self): # pylint: disable=no-self-use
 		frappe.cache_manager.clear_doctype_map('Assignment Rule', self.document_type)
 
 	def apply_unassign(self, doc, assignments):

From 81a02794b8fc065c34ccda29ab2fb57d6ddb9f43 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 18:00:13 +0530
Subject: [PATCH 297/361] fix(event_consumer): choose the right doctype

---
 frappe/cache_manager.py                                       | 2 +-
 .../event_streaming/doctype/event_consumer/event_consumer.py  | 4 ++--
 .../doctype/event_update_log/event_update_log.py              | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py
index a58ee90632..dabc78a9f6 100644
--- a/frappe/cache_manager.py
+++ b/frappe/cache_manager.py
@@ -12,7 +12,7 @@ from frappe.desk.notifications import (delete_notification_count_for,
 common_default_keys = ["__default", "__global"]
 
 doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map',
-	'milestone_tracker_map', 'event_consumer_map')
+	'milestone_tracker_map', 'event_consumer_document_type_map')
 
 global_cache_keys = ("app_hooks", "installed_apps",
 		"app_modules", "module_app", "system_settings",
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
index aa7b49034e..a53d046be5 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
@@ -25,10 +25,10 @@ class EventConsumer(Document):
 		else:
 			frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
 
-		frappe.cache_manager.clear_doctype_map('Event Consumer', self.ref_doctype)
+		frappe.cache().delete_value('event_consumer_document_type_map')
 
 	def on_trash(self):
-		frappe.cache_manager.clear_doctype_map('Event Consumer', self.ref_doctype)
+		frappe.cache().delete_value('event_consumer_document_type_map')
 
 	def update_consumer_status(self):
 		consumer_site = get_consumer_site(self.callback_url)
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index 4b655172f1..cb2c3b2abf 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -41,7 +41,7 @@ def notify_consumers(doc, event):
 
 def check_doctype_has_consumers(doctype):
 	"""Check if doctype has event consumers for event streaming"""
-	return frappe.cache_manager.get_doctype_map('Event Consumer', doctype,
+	return frappe.cache_manager.get_doctype_map('Event Consumer Document Type', doctype,
 		dict(ref_doctype = doctype, status='Approved'))
 
 def get_update(old, new, for_child=False):

From 049f04a4520bf890a7250c4af87325a88b0ccd07 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 13 Aug 2020 22:48:58 +0530
Subject: [PATCH 298/361] fix(minor): remove unused import

---
 .../doctype/milestone_tracker/test_milestone_tracker.py         | 2 --
 1 file changed, 2 deletions(-)

diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
index 34b6a608a9..05db3b025e 100644
--- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
+++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
@@ -7,8 +7,6 @@ import frappe
 import frappe.cache_manager
 import unittest
 
-from .milestone_tracker import get_milestone_trackers
-
 class TestMilestoneTracker(unittest.TestCase):
 	def test_milestone(self):
 		frappe.db.sql('delete from `tabMilestone Tracker`')

From df8075bd1debd564d76c082905ece5133722dbbb Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 08:45:01 +0530
Subject: [PATCH 299/361] debug(minor): print trace for failed events in tests

---
 frappe/event_streaming/doctype/event_producer/event_producer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index dccacce4f3..fb48021f36 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -213,6 +213,8 @@ def sync(update, producer_site, event_producer, in_retry=False):
 
 	except Exception:
 		if in_retry:
+			if frappe.flags.in_test:
+				print(frappe.get_traceback())
 			return 'Failed'
 		log_event_sync(update, event_producer.name, 'Failed', frappe.get_traceback())
 

From dad59ce8cf263ca55b796389be698bcec35fec1a Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 09:02:33 +0530
Subject: [PATCH 300/361] fix(tests): select for update in reset_event_producer

---
 .../doctype/event_producer/test_event_producer.py               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index e5115ece4a..46052f0d93 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -306,7 +306,7 @@ def create_event_producer(producer_url):
 	event_producer.save()
 
 def reset_configuration(producer_url):
-	event_producer = frappe.get_doc('Event Producer', producer_url)
+	event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 	event_producer.producer_doctypes = []
 	event_producer.producer_url = producer_url
 	event_producer.append('producer_doctypes', {

From 6336756c8a1ff054fdc20795e28c203635e6849e Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 09:21:51 +0530
Subject: [PATCH 301/361] fix(tests): test_event_producer

---
 .../doctype/event_producer/test_event_producer.py           | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 46052f0d93..f9cc769592 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -107,7 +107,7 @@ class TestEventProducer(unittest.TestCase):
 	def test_dynamic_link_dependencies_synced(self):
 		producer = get_remote_site()
 		#unsubscribe for Note to check whether dependency is fulfilled
-		event_producer = frappe.get_doc('Event Producer', producer_url)
+		event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 		event_producer.producer_doctypes = []
 		event_producer.append('producer_doctypes', {
 			'ref_doctype': 'ToDo',
@@ -135,7 +135,7 @@ class TestEventProducer(unittest.TestCase):
 	def test_naming_configuration(self):
 		#test with use_same_name = 0
 		producer = get_remote_site()
-		event_producer = frappe.get_doc('Event Producer', producer_url)
+		event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 		event_producer.producer_doctypes = []
 		event_producer.append('producer_doctypes', {
 			'ref_doctype': 'ToDo',
@@ -176,7 +176,7 @@ class TestEventProducer(unittest.TestCase):
 
 	def test_mapping(self):
 		producer = get_remote_site()
-		event_producer = frappe.get_doc('Event Producer', producer_url)
+		event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 		event_producer.producer_doctypes = []
 		mapping = [{
 			'local_fieldname': 'description',

From 8d09b373087201861ff283f982e7cf00db171f2c Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Fri, 14 Aug 2020 12:39:52 +0530
Subject: [PATCH 302/361] feat(Event Streaming): allow consumers to unsubscribe

---
 .../event_consumer_document_type.json          | 18 ++++++++++++++++--
 .../doctype/event_producer/event_producer.py   |  3 ++-
 .../event_producer_document_type.json          | 14 +++++++++++++-
 .../event_update_log/event_update_log.py       |  2 +-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
index 7463a518ba..71dcc63127 100644
--- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
+++ b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
@@ -1,14 +1,17 @@
 {
+ "actions": [],
  "creation": "2019-10-03 21:10:54.754651",
  "doctype": "DocType",
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
   "ref_doctype",
-  "status"
+  "status",
+  "unsubscribed"
  ],
  "fields": [
   {
+   "columns": 4,
    "fieldname": "ref_doctype",
    "fieldtype": "Link",
    "in_list_view": 1,
@@ -18,16 +21,27 @@
    "reqd": 1
   },
   {
+   "columns": 4,
    "default": "Pending",
    "fieldname": "status",
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Approval Status",
    "options": "Pending\nApproved\nRejected"
+  },
+  {
+   "columns": 2,
+   "default": "0",
+   "fieldname": "unsubscribed",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Unsubscribed",
+   "read_only": 1
   }
  ],
  "istable": 1,
- "modified": "2019-10-29 15:26:32.436528",
+ "links": [],
+ "modified": "2020-08-14 12:38:40.918620",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Consumer Document Type",
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index fb48021f36..73aea114ab 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -107,7 +107,8 @@ class EventProducer(Document):
 
 					event_consumer.consumer_doctypes.append({
 						'ref_doctype': ref_doctype,
-						'status': get_approval_status(config, ref_doctype)
+						'status': get_approval_status(config, ref_doctype),
+						'unsubscribed': entry.unsubscribe
 					})
 				if frappe.flags.in_test:
 					event_consumer.in_test = True
diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
index f473879600..e5fe9497f8 100644
--- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
+++ b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
@@ -8,11 +8,13 @@
   "ref_doctype",
   "status",
   "use_same_name",
+  "unsubscribe",
   "has_mapping",
   "mapping"
  ],
  "fields": [
   {
+   "columns": 3,
    "fieldname": "ref_doctype",
    "fieldtype": "Link",
    "in_list_view": 1,
@@ -36,6 +38,7 @@
    "options": "Document Type Mapping"
   },
   {
+   "columns": 2,
    "default": "0",
    "description": "If this is checked the documents will have the same name as they have on the Event Producer's site",
    "fieldname": "use_same_name",
@@ -44,6 +47,7 @@
    "label": "Use Same Name"
   },
   {
+   "columns": 3,
    "default": "Pending",
    "fieldname": "status",
    "fieldtype": "Select",
@@ -51,11 +55,19 @@
    "label": "Approval Status",
    "options": "Pending\nApproved\nRejected",
    "read_only": 1
+  },
+  {
+   "columns": 2,
+   "default": "0",
+   "fieldname": "unsubscribe",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Unsubscribe"
   }
  ],
  "istable": 1,
  "links": [],
- "modified": "2019-12-30 11:42:36.032351",
+ "modified": "2020-08-14 11:38:01.278996",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Producer Document Type",
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index cb2c3b2abf..646331a02c 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -42,7 +42,7 @@ def notify_consumers(doc, event):
 def check_doctype_has_consumers(doctype):
 	"""Check if doctype has event consumers for event streaming"""
 	return frappe.cache_manager.get_doctype_map('Event Consumer Document Type', doctype,
-		dict(ref_doctype = doctype, status='Approved'))
+		dict(ref_doctype=doctype, status='Approved', unsubscribed=0))
 
 def get_update(old, new, for_child=False):
 	"""

From bb0e8af0fb5d6b8a5044338e69a5bc0202420f64 Mon Sep 17 00:00:00 2001
From: Rucha Mahabal <ruchamahabal2@gmail.com>
Date: Fri, 14 Aug 2020 13:48:18 +0530
Subject: [PATCH 303/361] fix(test): unsubscribe doctypes in tearDown

---
 .../doctype/event_producer/test_event_producer.py   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index f9cc769592..4fea55eb39 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -23,6 +23,8 @@ class TestEventProducer(unittest.TestCase):
 	def setUp(self):
 		create_event_producer(producer_url)
 
+	def tearDown(self):
+		unsubscribe_doctypes(producer_url)
 
 	def test_insert(self):
 		producer = get_remote_site()
@@ -290,7 +292,12 @@ def get_mapping(mapping_name, local, remote, field_map):
 
 def create_event_producer(producer_url):
 	if frappe.db.exists('Event Producer', producer_url):
+		event_producer = frappe.get_doc('Event Producer', producer_url)
+		for entry in event_producer.producer_doctypes:
+			entry.unsubscribe = 0
+		event_producer.save()
 		return
+
 	event_producer = frappe.new_doc('Event Producer')
 	event_producer.producer_doctypes = []
 	event_producer.producer_url = producer_url
@@ -329,3 +336,9 @@ def get_remote_site():
 		frappe_authorization_source='Event Consumer'
 	)
 	return producer_site
+
+def unsubscribe_doctypes(producer_url):
+	event_producer = frappe.get_doc('Event Producer', producer_url)
+	for entry in event_producer.producer_doctypes:
+		entry.unsubscribe = 1
+	event_producer.save()
\ No newline at end of file

From f6bc6786ac6a152c822d9682dc03a06b5a8197bf Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 14:48:04 +0530
Subject: [PATCH 304/361] debug: test_workflow

---
 frappe/commands/site.py                           | 2 +-
 frappe/workflow/doctype/workflow/test_workflow.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index b72d98c433..d343d10126 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -296,7 +296,7 @@ def migrate(context, rebuild_website=False, skip_failing=False, skip_search_inde
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-	print("Compiling Python Files...")
+	print("Compiling Python files...")
 	compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*'))
 
 @click.command('migrate-to')
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 22a9c97c1f..ca3fd495a5 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -35,7 +35,9 @@ class TestWorkflow(unittest.TestCase):
 		'''test simple workflow'''
 		todo = doc or self.test_default_condition()
 
+		print_sql(True)
 		apply_workflow(todo, 'Approve')
+		print_sql(False)
 		# default condition is set
 		self.assertEqual(todo.workflow_state, 'Approved')
 		self.assertEqual(todo.status, 'Closed')

From 2af7c7dc337170183412f7a7a3732e0e3d0de8af Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 15:06:21 +0530
Subject: [PATCH 305/361] debug: test_workflow

---
 frappe/workflow/doctype/workflow/test_workflow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index ca3fd495a5..506ffb834d 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -35,9 +35,9 @@ class TestWorkflow(unittest.TestCase):
 		'''test simple workflow'''
 		todo = doc or self.test_default_condition()
 
-		print_sql(True)
+		frpape.print_sql(True)
 		apply_workflow(todo, 'Approve')
-		print_sql(False)
+		frappe.print_sql(False)
 		# default condition is set
 		self.assertEqual(todo.workflow_state, 'Approved')
 		self.assertEqual(todo.status, 'Closed')

From 7ee1fe3caf3320251265b31e478c69ec126e094e Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 14 Aug 2020 15:23:05 +0530
Subject: [PATCH 306/361] debug: test_workflow

---
 frappe/workflow/doctype/workflow/test_workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 506ffb834d..120afe36be 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -35,7 +35,7 @@ class TestWorkflow(unittest.TestCase):
 		'''test simple workflow'''
 		todo = doc or self.test_default_condition()
 
-		frpape.print_sql(True)
+		frappe.print_sql(True)
 		apply_workflow(todo, 'Approve')
 		frappe.print_sql(False)
 		# default condition is set

From 6535751e1d934300799aeebfc8743ee6024b2c70 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 09:56:45 +0530
Subject: [PATCH 307/361] debug(tests): test_workflow

---
 .travis/consumer_db/mariadb.json                  | 1 +
 .travis/consumer_db/postgres.json                 | 1 +
 .travis/producer_db/mariadb.json                  | 1 +
 .travis/producer_db/postgres.json                 | 1 +
 frappe/workflow/doctype/workflow/test_workflow.py | 4 ++--
 5 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.travis/consumer_db/mariadb.json b/.travis/consumer_db/mariadb.json
index e08ad70174..fb5b3bc976 100644
--- a/.travis/consumer_db/mariadb.json
+++ b/.travis/consumer_db/mariadb.json
@@ -2,6 +2,7 @@
     "db_host": "localhost",
     "db_name": "test_frappe_consumer",
     "db_password": "test_frappe",
+    "allow_tests": true,
     "db_type": "mariadb",
     "auto_email_id": "test@example.com",
     "mail_server": "smtp.example.com",
diff --git a/.travis/consumer_db/postgres.json b/.travis/consumer_db/postgres.json
index 5923f602ba..fed9fdfde2 100644
--- a/.travis/consumer_db/postgres.json
+++ b/.travis/consumer_db/postgres.json
@@ -3,6 +3,7 @@
     "db_name": "test_frappe_consumer",
     "db_password": "test_frappe",
     "db_type": "postgres",
+    "allow_tests": true,
     "auto_email_id": "test@example.com",
     "mail_server": "smtp.example.com",
     "mail_login": "test@example.com",
diff --git a/.travis/producer_db/mariadb.json b/.travis/producer_db/mariadb.json
index 8860241d80..988282a554 100644
--- a/.travis/producer_db/mariadb.json
+++ b/.travis/producer_db/mariadb.json
@@ -2,6 +2,7 @@
     "db_host": "localhost",
     "db_name": "test_frappe_producer",
     "db_password": "test_frappe",
+    "allow_tests": true,
     "db_type": "mariadb",
     "auto_email_id": "test@example.com",
     "mail_server": "smtp.example.com",
diff --git a/.travis/producer_db/postgres.json b/.travis/producer_db/postgres.json
index 4d846eee98..6426e99058 100644
--- a/.travis/producer_db/postgres.json
+++ b/.travis/producer_db/postgres.json
@@ -3,6 +3,7 @@
     "db_name": "test_frappe_producer",
     "db_password": "test_frappe",
     "db_type": "postgres",
+    "allow_tests": true,
     "auto_email_id": "test@example.com",
     "mail_server": "smtp.example.com",
     "mail_login": "test@example.com",
diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py
index 120afe36be..adcd98a85d 100644
--- a/frappe/workflow/doctype/workflow/test_workflow.py
+++ b/frappe/workflow/doctype/workflow/test_workflow.py
@@ -20,6 +20,7 @@ class TestWorkflow(unittest.TestCase):
 		frappe.set_user('Administrator')
 
 	def tearDown(self):
+		frappe.print_sql(False)
 		self.workflow.db_set('is_active', 0)
 
 	def test_default_condition(self):
@@ -33,11 +34,10 @@ class TestWorkflow(unittest.TestCase):
 
 	def test_approve(self, doc=None):
 		'''test simple workflow'''
+		frappe.print_sql(True)
 		todo = doc or self.test_default_condition()
 
-		frappe.print_sql(True)
 		apply_workflow(todo, 'Approve')
-		frappe.print_sql(False)
 		# default condition is set
 		self.assertEqual(todo.workflow_state, 'Approved')
 		self.assertEqual(todo.status, 'Closed')

From b71acfd5fa4a6909e8a8b334f51d5c0818879993 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 10:45:58 +0530
Subject: [PATCH 308/361] fix(minor): print query and values directly if
 mogrify fails

---
 frappe/database/database.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 8332a71168..90ac3e083e 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -214,7 +214,10 @@ class Database(object):
 		if not values:
 			return query
 		else:
-			return self._cursor.mogrify(query, values)
+			try:
+				return self._cursor.mogrify(query, values)
+			except:
+				return (query, values)
 
 	def explain_query(self, query, values=None):
 		"""Print `EXPLAIN` in error log."""

From 56853a0533aa09e589d644f977bd4a76082c8e39 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 11:10:48 +0530
Subject: [PATCH 309/361] fix(linting): sider

---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 90ac3e083e..2b8564481a 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -216,7 +216,7 @@ class Database(object):
 		else:
 			try:
 				return self._cursor.mogrify(query, values)
-			except:
+			except: # noqa: E722
 				return (query, values)
 
 	def explain_query(self, query, values=None):

From 53e1701b42a9310b6d27479c251691d610c78ae0 Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Mon, 17 Aug 2020 17:59:39 +0530
Subject: [PATCH 310/361] fix: Remove scroll to top in form on save

---
 frappe/public/js/frappe/form/form.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js
index b4bc758399..ff48ad2f60 100644
--- a/frappe/public/js/frappe/form/form.js
+++ b/frappe/public/js/frappe/form/form.js
@@ -568,9 +568,6 @@ frappe.ui.form.Form = class FrappeForm {
 		if(!save_action) save_action = "Save";
 		this.validate_form_action(save_action, resolve);
 
-		if((!this.meta.in_dialog || this.in_form) && !this.meta.istable) {
-			frappe.utils.scroll_to(0);
-		}
 		var after_save = function(r) {
 			if(!r.exc) {
 				if (["Save", "Update", "Amend"].indexOf(save_action)!==-1) {

From 0192bcf45fe5c204416c5058233cd23b9e759bbc Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 22:19:44 +0530
Subject: [PATCH 311/361] fix(minor): frappe.db.set_value to allow multiple
 values

---
 frappe/database/database.py | 15 ++++++---------
 frappe/tests/test_db.py     | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 2b8564481a..d9755abd33 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -660,20 +660,17 @@ class Database(object):
 
 		if dn and dt!=dn:
 			# with table
-			values = dict(
-				name=self.get_value(dt, dn, 'name', for_update=for_update)
-			)
-
-			values.update(to_update)
-
 			set_values = []
 			for key in to_update:
 				set_values.append('`{0}`=%({0})s'.format(key))
 
-			self.sql("""update `tab{0}`
-				set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
-				values, debug=debug)
+			for name in self.get_values(dt, dn, 'name', for_update=for_update):
+				values = dict(name=name[0])
+				values.update(to_update)
 
+				self.sql("""update `tab{0}`
+					set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
+					values, debug=debug)
 		else:
 			# for singles
 			keys = list(to_update)
diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py
index bb479d67c7..d1f608f48d 100644
--- a/frappe/tests/test_db.py
+++ b/frappe/tests/test_db.py
@@ -22,6 +22,21 @@ class TestDB(unittest.TestCase):
 		self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0],
 			frappe.db.get_value("User", {"name": [">=", "t"]}))
 
+	def test_set_value(self):
+		todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert()
+		todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert()
+
+		frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1')
+		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1')
+
+		# multiple set-value
+		frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')),
+			'description', 'change 2')
+
+		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2')
+		self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2')
+
+
 	def test_escape(self):
 		frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8"))
 

From b6cd4330f21b21ada5362b1759e0debb8d97ed11 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 17:09:04 +0530
Subject: [PATCH 312/361] feat(reports): add columns and filters via
 configuration

---
 frappe/core/doctype/doctype/doctype.json      |   2 +-
 frappe/core/doctype/report/report.js          |  20 --
 frappe/core/doctype/report/report.json        |  52 +++-
 frappe/core/doctype/report/report.py          | 269 ++++++++++--------
 frappe/core/doctype/report/test_report.py     |  38 +++
 frappe/core/doctype/report_column/__init__.py |   0
 .../doctype/report_column/report_column.json  |  61 ++++
 .../doctype/report_column/report_column.py    |  10 +
 frappe/core/doctype/report_filter/__init__.py |   0
 .../doctype/report_filter/report_filter.json  |  71 +++++
 .../doctype/report_filter/report_filter.py    |  10 +
 .../js/frappe/views/reports/query_report.js   |  13 +-
 12 files changed, 395 insertions(+), 151 deletions(-)
 create mode 100644 frappe/core/doctype/report_column/__init__.py
 create mode 100644 frappe/core/doctype/report_column/report_column.json
 create mode 100644 frappe/core/doctype/report_column/report_column.py
 create mode 100644 frappe/core/doctype/report_filter/__init__.py
 create mode 100644 frappe/core/doctype/report_filter/report_filter.json
 create mode 100644 frappe/core/doctype/report_filter/report_filter.py

diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json
index b890f743cc..6cdf2e40ac 100644
--- a/frappe/core/doctype/doctype/doctype.json
+++ b/frappe/core/doctype/doctype/doctype.json
@@ -609,7 +609,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2020-08-06 12:59:32.369093",
+ "modified": "2020-08-06 12:59:32.369095",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",
diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js
index c410e9aa1a..14e9b3a901 100644
--- a/frappe/core/doctype/report/report.js
+++ b/frappe/core/doctype/report/report.js
@@ -42,26 +42,6 @@ frappe.ui.form.on('Report', {
 		}
 	},
 
-	report_type: function(frm) {
-		frm.set_intro("");
-		switch(frm.doc.report_type) {
-			case "Report Builder":
-				frm.set_intro(__("Report Builder reports are managed directly by the report builder. Nothing to do."));
-				break;
-			case "Query Report":
-				frm.set_intro(__("Write a SELECT query. Note result is not paged (all data is sent in one go).")
-					+ __("To format columns, give column labels in the query.") + "<br>"
-					+ __("[Label]:[Field Type]/[Options]:[Width]") + "<br><br>"
-					+ __("Example:") + "<br>"
-					+ "Employee:Link/Employee:200" + "<br>"
-					+ "Rate:Currency:120" + "<br>")
-				break;
-			case "Script Report":
-				frm.set_intro(__("Write a Python file in the same folder where this is saved and return column and result."));
-				break;
-		}
-	},
-
 	set_doctype_roles: function(frm) {
 		return frm.call('set_doctype_roles').then(() => {
 			frm.refresh_field('roles');
diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json
index 40d2417a56..5b3593e658 100644
--- a/frappe/core/doctype/report/report.json
+++ b/frappe/core/doctype/report/report.json
@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "autoname": "field:report_name",
  "creation": "2013-03-09 15:45:57",
  "doctype": "DocType",
@@ -17,10 +18,15 @@
   "disabled",
   "disable_prepared_report",
   "prepared_report",
+  "filters_section",
+  "filters",
+  "columns_section",
+  "columns",
   "section_break_6",
   "query",
-  "javascript",
   "report_script",
+  "client_code_section",
+  "javascript",
   "json",
   "permission_rules",
   "roles"
@@ -94,7 +100,8 @@
   },
   {
    "fieldname": "section_break_6",
-   "fieldtype": "Section Break"
+   "fieldtype": "Section Break",
+   "label": "Query / Script"
   },
   {
    "depends_on": "eval:doc.report_type==\"Query Report\"",
@@ -142,15 +149,50 @@
    "read_only": 1
   },
   {
-   "depends_on": "eval:doc.report_type==\"Script Report\" && doc.is_standard===\"No\"",
-   "description": "output in the form of `data = [columns, result]`",
+   "depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"",
+   "description": "Filters will be accessible via <code>filters</code>. <br><br>Send output as <code>result = [result]</code>, or for old style <code>data = [columns], [result]</code>",
    "fieldname": "report_script",
    "fieldtype": "Code",
    "label": "Script"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "filters",
+   "fieldname": "filters_section",
+   "fieldtype": "Section Break",
+   "label": "Filters"
+  },
+  {
+   "fieldname": "filters",
+   "fieldtype": "Table",
+   "label": "Filters",
+   "options": "Report Filter"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "columns",
+   "fieldname": "columns_section",
+   "fieldtype": "Section Break",
+   "label": "Columns"
+  },
+  {
+   "fieldname": "columns",
+   "fieldtype": "Table",
+   "label": "Columns",
+   "options": "Report Column"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "javascript",
+   "fieldname": "client_code_section",
+   "fieldtype": "Section Break",
+   "label": "Client Code"
   }
  ],
  "idx": 1,
- "modified": "2019-10-09 15:43:08.577610",
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2020-08-17 16:49:28.474274",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Report",
diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index dd695d484e..bbdab54686 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -51,6 +51,9 @@ class Report(Document):
 	def on_trash(self):
 		delete_custom_role('report', self.name)
 
+	def get_columns(self):
+		return [d.as_dict(no_default_fields = True) for d in self.columns]
+
 	def set_doctype_roles(self):
 		if not self.get('roles') and self.is_standard == 'No':
 			meta = frappe.get_meta(self.ref_doctype)
@@ -99,8 +102,8 @@ class Report(Document):
 		if not self.query.lower().startswith("select"):
 			frappe.throw(_("Query must be a SELECT"), title=_('Report Document Error'))
 
-		result = [list(t) for t in frappe.db.sql(self.query, filters)]
-		columns = [cstr(c[0]) for c in frappe.db.get_description()]
+		result = [list(t) for t in frappe.db.sql(self.query, filters, debug=True)]
+		columns = self.get_columns() or [cstr(c[0]) for c in frappe.db.get_description()]
 
 		return [columns, result]
 
@@ -134,135 +137,155 @@ class Report(Document):
 
 	def execute_script(self, filters):
 		# server script
-		loc = {"filters": frappe._dict(filters), 'data':[]}
+		loc = {"filters": frappe._dict(filters), 'data':None, 'result':None}
 		safe_exec(self.report_script, None, loc)
-		return loc['data']
+		if loc['data']:
+			return loc['data']
+		else:
+			return self.get_columns(), loc['result']
 
 	def get_data(self, filters=None, limit=None, user=None, as_dict=False, ignore_prepared_report=False):
-		columns = []
-		out = []
-
 		if self.report_type in ('Query Report', 'Script Report', 'Custom Report'):
-			# query and script reports
-			data = frappe.desk.query_report.run(self.name,
-				filters=filters, user=user, ignore_prepared_report=ignore_prepared_report)
-
-			for d in data.get('columns'):
-				if isinstance(d, dict):
-					col = frappe._dict(d)
-					if not col.fieldname:
-						col.fieldname = col.label
-					columns.append(col)
-				else:
-					fieldtype, options = "Data", None
-					parts = d.split(':')
-					if len(parts) > 1:
-						if parts[1]:
-							fieldtype, options = parts[1], None
-							if fieldtype and '/' in fieldtype:
-								fieldtype, options = fieldtype.split('/')
-
-					columns.append(frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0], options=options))
-
-			out += data.get('result')
+			columns, result = self.run_query_report(filters, user, ignore_prepared_report)
 		else:
-			# standard report
-			params = json.loads(self.json)
-
-			if params.get('fields'):
-				columns = params.get('fields')
-			elif params.get('columns'):
-				columns = params.get('columns')
-			else:
-				columns = [['name', self.ref_doctype]]
-				for df in frappe.get_meta(self.ref_doctype).fields:
-					if df.in_list_view:
-						columns.append([df.fieldname, self.ref_doctype])
-
-			_filters = params.get('filters') or []
-
-			if filters:
-				for key, value in iteritems(filters):
-					condition, _value = '=', value
-					if isinstance(value, (list, tuple)):
-						condition, _value = value
-					_filters.append([key, condition, _value])
-
-			def _format(parts):
-				# sort by is saved as DocType.fieldname, covert it to sql
-				return '`tab{0}`.`{1}`'.format(*parts)
-
-			if params.get('sort_by'):
-				order_by = _format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order')
-			elif params.get('order_by'):
-				order_by = params.get('order_by')
-			else:
-				order_by = _format([self.ref_doctype, 'modified']) + ' desc'
-
-			if params.get('sort_by_next'):
-				order_by += ', ' + _format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')
-
-			group_by = None
-			if params.get('group_by'):
-				group_by_args = frappe._dict(params['group_by'])
-				group_by = group_by_args['group_by']
-				order_by = '_aggregate_column desc'
-
-			result = frappe.get_list(self.ref_doctype,
-				fields = [
-					get_group_by_field(group_by_args, c[1]) if c[0] == '_aggregate_column' and group_by_args
-					else _format([c[1], c[0]])
-					for c in columns
-				],
-				filters=_filters,
-				order_by = order_by,
-				as_list=True,
-				limit=limit,
-				group_by=group_by,
-				user=user)
-
-			_columns = []
-
-			for (fieldname, doctype) in columns:
-				meta = frappe.get_meta(doctype)
-
-				if meta.get_field(fieldname):
-					field = meta.get_field(fieldname)
-				else:
-					if fieldname == '_aggregate_column':
-						label = get_group_by_column_label(group_by_args, meta)
-					else:
-						label = meta.get_label(fieldname)
-
-					field = frappe._dict(fieldname=fieldname, label=label)
-					# since name is the primary key for a document, it will always be a Link datatype
-					if fieldname == "name":
-						field.fieldtype = "Link"
-						field.options = doctype
-
-				_columns.append(field)
-			columns = _columns
-
-			out = out + [list(d) for d in result]
-
-			if params.get('add_totals_row'):
-				out = append_totals_row(out)
+			columns, result = self.run_standard_report(filters, limit, user)
 
 		if as_dict:
-			data = []
-			for row in out:
-				if isinstance(row, (list, tuple)):
-					_row = frappe._dict()
-					for i, val in enumerate(row):
-						_row[columns[i].get('fieldname')] = val
-				elif isinstance(row, dict):
-					# no need to convert from dict to dict
-					_row = frappe._dict(row)
-				data.append(_row)
-		else:
-			data = out
-		return columns, data
+			result = self.build_data_dict(result)
 
+		return columns, result
+
+	def run_query_report(self, filters, user, ignore_prepared_report=False):
+		columns, result = [], []
+		data = frappe.desk.query_report.run(self.name,
+			filters=filters, user=user, ignore_prepared_report=ignore_prepared_report)
+
+		for d in data.get('columns'):
+			if isinstance(d, dict):
+				col = frappe._dict(d)
+				if not col.fieldname:
+					col.fieldname = col.label
+				columns.append(col)
+			else:
+				fieldtype, options = "Data", None
+				parts = d.split(':')
+				if len(parts) > 1:
+					if parts[1]:
+						fieldtype, options = parts[1], None
+						if fieldtype and '/' in fieldtype:
+							fieldtype, options = fieldtype.split('/')
+
+				columns.append(frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0], options=options))
+
+		result += data.get('result')
+
+		return columns, result
+
+	def run_standard_report(self, filters, limit, user):
+		params = json.loads(self.json)
+		columns = self.get_standard_report_columns(params)
+		result = []
+
+		_result = frappe.get_list(self.ref_doctype,
+			fields = [Report._format([c[1], c[0]]) for c in columns],
+			filters = self.get_standard_report_filters(params, filters),
+			order_by = self.get_standard_report_order_by(params),
+			as_list = True,
+			limit = limit,
+			user = user)
+
+		columns = self.build_standard_report_columns(columns)
+
+		result = result + [list(d) for d in _result]
+
+		if params.get('add_totals_row'):
+			result = append_totals_row(out)
+
+		return columns, result
+
+	@staticmethod
+	def _format(parts):
+		# sort by is saved as DocType.fieldname, covert it to sql
+		return '`tab{0}`.`{1}`'.format(*parts)
+
+	def get_standard_report_columns(self, params):
+		if params.get('fields'):
+			columns = params.get('fields')
+		elif params.get('columns'):
+			columns = params.get('columns')
+		elif params.get('fields'):
+			columns = params.get('fields')
+		else:
+			columns = [['name', self.ref_doctype]]
+			for df in frappe.get_meta(self.ref_doctype).fields:
+				if df.in_list_view:
+					columns.append([df.fieldname, self.ref_doctype])
+
+		return columns
+
+	def get_standard_report_filters(self, params, filters):
+		_filters = params.get('filters') or []
+
+		if filters:
+			for key, value in iteritems(filters):
+				condition, _value = '=', value
+				if isinstance(value, (list, tuple)):
+					condition, _value = value
+				_filters.append([key, condition, _value])
+
+		return _filters
+
+	def get_standard_report_order_by(self, params):
+		if params.get('sort_by'):
+			order_by = Report._format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order')
+
+		elif params.get('order_by'):
+			order_by = params.get('order_by')
+		else:
+			order_by = Report._format([self.ref_doctype, 'modified']) + ' desc'
+
+		if params.get('sort_by_next'):
+			order_by += ', ' + Report._format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')
+
+		return order_by
+
+	def build_standard_report_columns(self, columns):
+		_columns = []
+
+		for (fieldname, doctype) in columns:
+			meta = frappe.get_meta(doctype)
+
+			if meta.get_field(fieldname):
+				field = meta.get_field(fieldname)
+			else:
+				if fieldname == '_aggregate_column':
+					label = get_group_by_column_label(group_by_args, meta)
+				else:
+					label = meta.get_label(fieldname)
+
+				field = frappe._dict(fieldname=fieldname, label=label)
+
+				# since name is the primary key for a document, it will always be a Link datatype
+				if fieldname == "name":
+					field.fieldtype = "Link"
+					field.options = doctype
+
+			_columns.append(field)
+		return _columns
+
+	def build_data_dict(self, result):
+		data = []
+		for row in result:
+			if isinstance(row, (list, tuple)):
+				_row = frappe._dict()
+				for i, val in enumerate(row):
+					_row[columns[i].get('fieldname')] = val
+			elif isinstance(row, dict):
+				# no need to convert from dict to dict
+				_row = frappe._dict(row)
+			data.append(_row)
+
+		return data
 
 	@Document.whitelist
 	def toggle_disable(self, disable):
diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py
index b8b18205b4..805b903300 100644
--- a/frappe/core/doctype/report/test_report.py
+++ b/frappe/core/doctype/report/test_report.py
@@ -111,3 +111,41 @@ data = [
 		# check values
 		self.assertTrue('System User' in [d.get('type') for d in data[1]])
 
+	def test_script_report_with_columns(self):
+		report_name = 'Test Script Report With Columns'
+
+		if frappe.db.exists("Report", report_name):
+			frappe.delete_doc('Report', report_name)
+
+		report = frappe.get_doc({
+			'doctype': 'Report',
+			'ref_doctype': 'User',
+			'report_name': report_name,
+			'report_type': 'Script Report',
+			'is_standard': 'No',
+			'columns': [
+				dict(fieldname='type', label='Type', fieldtype='Data'),
+				dict(fieldname='value', label='Value', fieldtype='Int'),
+			]
+		}).insert(ignore_permissions=True)
+
+		report.report_script = '''
+totals = {}
+for user in frappe.get_all('User', fields = ['name', 'user_type', 'creation']):
+	if not user.user_type in totals:
+		totals[user.user_type] = 0
+	totals[user.user_type] = totals[user.user_type] + 1
+
+result = [
+		{"type":key, "value": value} for key, value in totals.items()
+	]
+'''
+
+		report.save()
+		data = report.get_data()
+
+		# check columns
+		self.assertEqual(data[0][0]['label'], 'Type')
+
+		# check values
+		self.assertTrue('System User' in [d.get('type') for d in data[1]])
diff --git a/frappe/core/doctype/report_column/__init__.py b/frappe/core/doctype/report_column/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/core/doctype/report_column/report_column.json b/frappe/core/doctype/report_column/report_column.json
new file mode 100644
index 0000000000..53b5dff9b6
--- /dev/null
+++ b/frappe/core/doctype/report_column/report_column.json
@@ -0,0 +1,61 @@
+{
+ "actions": [],
+ "creation": "2020-01-14 11:28:37.583656",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "fieldname",
+  "label",
+  "fieldtype",
+  "options",
+  "width"
+ ],
+ "fields": [
+  {
+   "fieldname": "fieldname",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Fieldname",
+   "reqd": 1
+  },
+  {
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label",
+   "reqd": 1
+  },
+  {
+   "fieldname": "fieldtype",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Fieldtype",
+   "options": "Check\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nInt\nLink\nSelect\nTime",
+   "reqd": 1
+  },
+  {
+   "fieldname": "options",
+   "fieldtype": "Data",
+   "label": "Options"
+  },
+  {
+   "fieldname": "width",
+   "fieldtype": "Int",
+   "label": "Width"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2020-08-17 14:32:17.174796",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Report Column",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/core/doctype/report_column/report_column.py b/frappe/core/doctype/report_column/report_column.py
new file mode 100644
index 0000000000..69c88b7bda
--- /dev/null
+++ b/frappe/core/doctype/report_column/report_column.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class ReportColumn(Document):
+	pass
diff --git a/frappe/core/doctype/report_filter/__init__.py b/frappe/core/doctype/report_filter/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/core/doctype/report_filter/report_filter.json b/frappe/core/doctype/report_filter/report_filter.json
new file mode 100644
index 0000000000..9d277db11d
--- /dev/null
+++ b/frappe/core/doctype/report_filter/report_filter.json
@@ -0,0 +1,71 @@
+{
+ "actions": [],
+ "creation": "2020-01-14 11:38:58.016498",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "fieldname",
+  "label",
+  "fieldtype",
+  "mandatory",
+  "options",
+  "wildcard_filter"
+ ],
+ "fields": [
+  {
+   "fieldname": "fieldname",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Fieldname",
+   "reqd": 1
+  },
+  {
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label",
+   "reqd": 1
+  },
+  {
+   "fieldname": "fieldtype",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Fieldtype",
+   "options": "Check\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nInt\nLink\nSelect\nTime",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "mandatory",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Mandatory"
+  },
+  {
+   "fieldname": "options",
+   "fieldtype": "Data",
+   "label": "Options"
+  },
+  {
+   "default": "0",
+   "description": "Will add \"%\" before and after the query",
+   "fieldname": "wildcard_filter",
+   "fieldtype": "Check",
+   "label": "Wildcard Filter"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2020-08-17 16:15:46.937267",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Report Filter",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/core/doctype/report_filter/report_filter.py b/frappe/core/doctype/report_filter/report_filter.py
new file mode 100644
index 0000000000..d85a1a5a65
--- /dev/null
+++ b/frappe/core/doctype/report_filter/report_filter.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class ReportFilter(Document):
+	pass
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index e0b9019ca3..706cde13b7 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -366,6 +366,12 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 						this.report_settings.html_format = settings.html_format;
 						this.report_settings.execution_time = settings.execution_time || 0;
 						frappe.query_reports[this.report_name] = this.report_settings;
+
+						if (this.report_doc.filters && !this.report_settings.filters) {
+							// add configured filters
+							this.report_settings.filters = this.report_doc.filters;
+						}
+
 						resolve();
 					});
 				}).catch(reject);
@@ -1109,8 +1115,11 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			.map(f => {
 				var v = f.get_value();
 				// hidden fields dont have $input
-				if(f.df.hidden) v = f.value;
-				if(v === '%') v = null;
+				if (f.df.hidden) v = f.value;
+				if (v === '%') v = null;
+				if (f.df.wildcard_filter) {
+					v = `%${v}%`;
+				}
 				return {
 					[f.df.fieldname]: v
 				};

From e3f686276e87d82fed74b428dc26db9850fea325 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 17:26:27 +0530
Subject: [PATCH 313/361] fix(minor): pass columns to build_data_dict in
 report.py

---
 frappe/core/doctype/report/report.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index bbdab54686..790df3da3a 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -151,7 +151,7 @@ class Report(Document):
 			columns, result = self.run_standard_report(filters, limit, user)
 
 		if as_dict:
-			result = self.build_data_dict(result)
+			result = self.build_data_dict(result, columns)
 
 		return columns, result
 
@@ -273,7 +273,7 @@ class Report(Document):
 			_columns.append(field)
 		return _columns
 
-	def build_data_dict(self, result):
+	def build_data_dict(self, result, columns):
 		data = []
 		for row in result:
 			if isinstance(row, (list, tuple)):

From 6bd38fd00ad733b80e8558152585d73e390b05c7 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Mon, 17 Aug 2020 22:27:07 +0530
Subject: [PATCH 314/361] fix(minor): report.py

---
 frappe/core/doctype/report/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 790df3da3a..b9584ba122 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -199,7 +199,7 @@ class Report(Document):
 		result = result + [list(d) for d in _result]
 
 		if params.get('add_totals_row'):
-			result = append_totals_row(out)
+			result = append_totals_row(result)
 
 		return columns, result
 

From a15e54aec4b095ff42b16551cc58515c674d6e16 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 18 Aug 2020 15:23:21 +0530
Subject: [PATCH 315/361] fix: set is_query_report for custom reports

---
 frappe/desk/desktop.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py
index ae9d070976..148ae87249 100644
--- a/frappe/desk/desktop.py
+++ b/frappe/desk/desktop.py
@@ -47,11 +47,11 @@ class Workspace:
 
 		self.allowed_pages = get_allowed_pages(cache=True)
 		self.allowed_reports = get_allowed_reports(cache=True)
-		
+
 		if not minimal:
 			self.onboarding_doc = self.get_onboarding_doc()
 			self.onboarding = None
-		
+
 			self.table_counts = get_table_with_counts()
 		self.restricted_doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache()
 		self.restricted_pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache()
@@ -59,7 +59,7 @@ class Workspace:
 	def is_page_allowed(self):
 		cards = self.doc.cards + get_custom_reports_and_doctypes(self.doc.module) + self.extended_cards
 		shortcuts = self.doc.shortcuts + self.extended_shortcuts
-		
+
 		for section in cards:
 			links = loads(section.links) if isinstance(section.links, string_types) else section.links
 			for item in links:
@@ -195,7 +195,7 @@ class Workspace:
 				'docs_url': self.onboarding_doc.documentation_url,
 				'items': self.get_onboarding_steps()
 			}
-	
+
 	@handle_not_exist
 	def get_cards(self):
 		cards = self.doc.cards
@@ -303,7 +303,7 @@ class Workspace:
 			if self.is_item_allowed(item.link_to, item.type) and _in_active_domains(item):
 				if item.type == "Report":
 					report = self.allowed_reports.get(item.link_to, {})
-					if report.get("report_type") in ["Query Report", "Script Report"]:
+					if report.get("report_type") in ["Query Report", "Script Report", "Custom Report"]:
 						new_item['is_query_report'] = 1
 					else:
 						new_item['ref_doctype'] = report.get('ref_doctype')
@@ -358,7 +358,7 @@ def get_desk_sidebar_items(flatten=False, cache=True):
 	_cache = frappe.cache()
 	if cache:
 		pages = _cache.get_value("desk_sidebar_items", user=frappe.session.user)
-	
+
 	if not pages or not cache:
 		# don't get domain restricted pages
 		blocked_modules = frappe.get_doc('User', frappe.session.user).get_blocked_modules()
@@ -377,7 +377,7 @@ def get_desk_sidebar_items(flatten=False, cache=True):
 		order_by = "pin_to_top desc, pin_to_bottom asc, name asc"
 		all_pages = frappe.get_all("Desk Page", fields=["name", "category"], filters=filters, order_by=order_by, ignore_permissions=True)
 		pages = []
-		
+
 		# Filter Page based on Permission
 		for page in all_pages:
 			try:

From a9bbdc01826f9f4a17c240a807e7735ec593efdf Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 18 Aug 2020 13:42:18 +0530
Subject: [PATCH 316/361] fix: handle both list and dict results

---
 frappe/desk/query_report.py | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index d0a32ef076..656b49046a 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -216,20 +216,32 @@ def add_data_to_custom_columns(columns, result):
 	return data
 
 def reorder_data_for_custom_columns(custom_columns, columns, result, report_type):
-	custom_column_labels = [col["label"] for col in custom_columns]
+	if not result:
+		return []
 
 	if report_type == 'Query Report':
+		# Assume list result for query reports
+		# Query report columns exclisvuely use Label
+		custom_column_labels = [col["label"] for col in custom_columns]
 		original_column_labels = [col.split(":")[0] for col in columns]
-	else:
-		original_column_labels = [col["label"] for col in columns]
+		return get_columns_from_list(custom_column_labels, original_column_labels, result)
 
+	if type(result[0]) == "list":
+		# If the result is a list of lists
+		original_column_names = [col["fieldname"] for col in columns]
+		return get_columns_from_list(custom_column_names, original_column_names, result)
+	else:
+		# If the result is a list of dicts
+		return get_columns_from_dict(custom_column_names, result)
+
+def get_columns_from_list(columns, target_columns, result):
 	reordered_result = []
 
 	for res in result:
 		r = []
-		for col_name in custom_column_labels:
+		for col_name in columns:
 			try:
-				idx = original_column_labels.index(col_name)
+				idx = target_columns.index(col_name)
 				r.append(res[idx])
 			except ValueError:
 				pass
@@ -238,6 +250,21 @@ def reorder_data_for_custom_columns(custom_columns, columns, result, report_type
 
 	return reordered_result
 
+def get_columns_from_dict(columns, result):
+	reordered_result = []
+
+	for res in result:
+		r = {}
+		for col_name in columns:
+			try:
+				r[col_name] = res[col_name]
+			except KeyError:
+				pass
+
+		reordered_result.append(r)
+
+	return reordered_result
+
 def get_prepared_report_result(report, filters, dn="", user=None):
 	latest_report_data = {}
 	doc = None

From 5aea22eb337821b9b62df1849c6cf23b676b9b4e Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 18 Aug 2020 17:05:19 +0530
Subject: [PATCH 317/361] fix(minor): report.py: adapt recent group_by changes

---
 frappe/core/doctype/report/report.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index b9584ba122..04bf535e79 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -185,16 +185,20 @@ class Report(Document):
 		params = json.loads(self.json)
 		columns = self.get_standard_report_columns(params)
 		result = []
+		order_by, group_by_args = self.get_standard_report_order_by(params)
 
 		_result = frappe.get_list(self.ref_doctype,
-			fields = [Report._format([c[1], c[0]]) for c in columns],
+			fields = [
+				get_group_by_field(group_by_args, c[1]) if c[0] == '_aggregate_column' and group_by_args
+				else _format([c[1], c[0]])
+			],
 			filters = self.get_standard_report_filters(params, filters),
-			order_by = self.get_standard_report_order_by(params),
+			order_by = order_by,
 			as_list = True,
 			limit = limit,
 			user = user)
 
-		columns = self.build_standard_report_columns(columns)
+		columns = self.build_standard_report_columns(columns, group_by_args)
 
 		result = result + [list(d) for d in _result]
 
@@ -236,6 +240,7 @@ class Report(Document):
 		return _filters
 
 	def get_standard_report_order_by(self, params):
+		group_by_args = None
 		if params.get('sort_by'):
 			order_by = Report._format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order')
 
@@ -247,9 +252,15 @@ class Report(Document):
 		if params.get('sort_by_next'):
 			order_by += ', ' + Report._format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')
 
-		return order_by
+		group_by = None
+		if params.get('group_by'):
+			group_by_args = frappe._dict(params['group_by'])
+			group_by = group_by_args['group_by']
+			order_by = '_aggregate_column desc'
 
-	def build_standard_report_columns(self, columns):
+		return order_by, group_by_args
+
+	def build_standard_report_columns(self, columns, group_by_args):
 		_columns = []
 
 		for (fieldname, doctype) in columns:

From 6100169233d510920f8d230e3ba0d7f92ac06bbd Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 18 Aug 2020 17:16:31 +0530
Subject: [PATCH 318/361] fix(minor): report.py: adapt recent group_by changes

---
 frappe/core/doctype/report/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index 04bf535e79..d65d0db1ce 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -190,7 +190,7 @@ class Report(Document):
 		_result = frappe.get_list(self.ref_doctype,
 			fields = [
 				get_group_by_field(group_by_args, c[1]) if c[0] == '_aggregate_column' and group_by_args
-				else _format([c[1], c[0]])
+				else Report._format([c[1], c[0]]) for c in columns
 			],
 			filters = self.get_standard_report_filters(params, filters),
 			order_by = order_by,

From 9786485f7036d7a632f6340aa7328559db4c0826 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 18 Aug 2020 17:35:00 +0530
Subject: [PATCH 319/361] fix(minor): report.py: adapt recent group_by changes

---
 frappe/core/doctype/report/report.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py
index d65d0db1ce..8b7a03aa28 100644
--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@@ -185,7 +185,7 @@ class Report(Document):
 		params = json.loads(self.json)
 		columns = self.get_standard_report_columns(params)
 		result = []
-		order_by, group_by_args = self.get_standard_report_order_by(params)
+		order_by, group_by, group_by_args = self.get_standard_report_order_by(params)
 
 		_result = frappe.get_list(self.ref_doctype,
 			fields = [
@@ -194,6 +194,7 @@ class Report(Document):
 			],
 			filters = self.get_standard_report_filters(params, filters),
 			order_by = order_by,
+			group_by = group_by,
 			as_list = True,
 			limit = limit,
 			user = user)
@@ -258,7 +259,7 @@ class Report(Document):
 			group_by = group_by_args['group_by']
 			order_by = '_aggregate_column desc'
 
-		return order_by, group_by_args
+		return order_by, group_by, group_by_args
 
 	def build_standard_report_columns(self, columns, group_by_args):
 		_columns = []

From b83e0b9abd63014cd9c3b331518c79885381547d Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Tue, 18 Aug 2020 17:54:27 +0530
Subject: [PATCH 320/361] fix: add an explicit check for schedule sending
 (#11291)

---
 frappe/email/doctype/newsletter/newsletter.json   | 15 +++++++++------
 frappe/email/doctype/newsletter/newsletter.py     |  3 ++-
 .../email/doctype/newsletter/test_newsletter.py   |  1 +
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json
index 1ec64826da..4804b3d6fa 100644
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@@ -8,7 +8,7 @@
  "engine": "InnoDB",
  "field_order": [
   "send_from",
-  "column_break_2",
+  "schedule_sending",
   "schedule_send",
   "recipients",
   "email_group",
@@ -114,19 +114,22 @@
    "label": "Recipients"
   },
   {
+   "depends_on": "eval: doc.schedule_sending",
    "fieldname": "schedule_send",
    "fieldtype": "Datetime",
    "label": "Schedule Send"
   },
-  {
-   "fieldname": "column_break_2",
-   "fieldtype": "Column Break"
-  },
   {
    "default": "0",
    "fieldname": "send_attachments",
    "fieldtype": "Check",
    "label": "Send Attachments"
+  },
+  {
+   "default": "0",
+   "fieldname": "schedule_sending",
+   "fieldtype": "Check",
+   "label": "Schedule Sending"
   }
  ],
  "has_web_view": 1,
@@ -136,7 +139,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2020-07-21 16:25:17.687476",
+ "modified": "2020-08-17 18:11:59.541686",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",
diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py
index 48688afdb6..849c21f768 100755
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@@ -271,7 +271,8 @@ def send_scheduled_email():
 	"""Send scheduled newsletter to the recipients."""
 	scheduled_newsletter = frappe.get_all('Newsletter', filters = {
 		'schedule_send': ('<=', now_datetime()),
-		'email_sent': 0
+		'email_sent': 0,
+		'schedule_sending': 1
 	}, fields = ['name'], ignore_ifnull=True)
 	for newsletter in scheduled_newsletter:
 		send_newsletter(newsletter.name)
diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py
index 5a13f99e56..bb339165d3 100644
--- a/frappe/email/doctype/newsletter/test_newsletter.py
+++ b/frappe/email/doctype/newsletter/test_newsletter.py
@@ -69,6 +69,7 @@ class TestNewsletter(unittest.TestCase):
 			"send_from": "Test Sender <test_sender@example.com>",
 			"message": "Testing my news.",
 			"published": published,
+			"schedule_sending": bool(schedule_send),
 			"schedule_send": schedule_send
 		}).insert(ignore_permissions=True)
 

From 90da644834ce2e25512f18dc77449ff01a6cc4c1 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 19 Aug 2020 12:48:38 +0530
Subject: [PATCH 321/361] fix: add parrenttype in query

---
 frappe/core/doctype/doctype/doctype.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py
index 7f84555b79..5c558f3bd6 100644
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@@ -989,7 +989,8 @@ def clear_permissions_cache(doctype):
 			`tabHas Role`,
 			`tabDocPerm`
 		WHERE `tabDocPerm`.`parent` = %s
-		AND `tabDocPerm`.`role` = `tabHas Role`.`role`
+			AND `tabDocPerm`.`role` = `tabHas Role`.`role`
+			AND `tabHas Role`.`parenttype` = 'User'
 		""", doctype):
 		frappe.clear_cache(user=user)
 

From 2655a3f8dc5846a83de9316c8a4deb9f47f0f217 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Wed, 19 Aug 2020 12:49:40 +0530
Subject: [PATCH 322/361] feat: freeze dom on updating permissions

---
 frappe/core/page/permission_manager/permission_manager.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js
index 3f5d7c0a7b..7608b25766 100644
--- a/frappe/core/page/permission_manager/permission_manager.js
+++ b/frappe/core/page/permission_manager/permission_manager.js
@@ -334,6 +334,7 @@ frappe.PermissionEngine = Class.extend({
 		});
 
 		this.body.on("click", "input[type='checkbox']", function() {
+			frappe.dom.freeze();
 			var chk = $(this);
 			var args = {
 				role: chk.attr("data-role"),
@@ -348,6 +349,7 @@ frappe.PermissionEngine = Class.extend({
 				method: "update",
 				args: args,
 				callback: function(r) {
+					frappe.dom.unfreeze();
 					if(r.exc) {
 						// exception: reverse
 						chk.prop("checked", !chk.prop("checked"));

From 56521d51b5200089028c79f2cfed37369a0856ac Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Wed, 19 Aug 2020 14:36:39 +0530
Subject: [PATCH 323/361] fix: return early in parse_doc if invalid values

---
 frappe/core/doctype/data_import/importer.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 161ab5d9a0..2c10c6b0a5 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -465,7 +465,8 @@ class ImportFile:
 
 				if doctype != self.doctype and table_df:
 					child_doc = row.parse_doc(doctype, parent_doc, table_df)
-					if child_doc is None: continue
+					if child_doc is None:
+						continue
 					parent_doc[table_df.fieldname] = parent_doc.get(table_df.fieldname, [])
 					parent_doc[table_df.fieldname].append(child_doc)
 
@@ -571,6 +572,11 @@ class Row:
 	def parse_doc(self, doctype, parent_doc=None, table_df=None):
 		col_indexes = self.header.get_column_indexes(doctype, table_df)
 		values = self.get_values(col_indexes)
+
+		if all(v in INVALID_VALUES for v in values):
+			# if all values are invalid, no need to parse it
+			return None
+
 		columns = self.header.get_columns(col_indexes)
 		doc = self._parse_doc(doctype, columns, values, parent_doc, table_df)
 		return doc
@@ -590,22 +596,17 @@ class Row:
 		for key in frappe.model.default_fields + ("__islocal",):
 			doc.pop(key, None)
 
-		record_is_empty = True
 		for col, value in zip(columns, values):
 			df = col.df
 			if value in INVALID_VALUES:
 				value = None
 
 			if value is not None:
-				record_is_empty = False
 				value = self.validate_value(value, col)
 
 			if value is not None:
 				doc[df.fieldname] = self.parse_value(value, col)
 
-		if record_is_empty:
-			return None
-
 		is_table = frappe.get_meta(doctype).istable
 		is_update = self.import_type == UPDATE
 		if is_table and is_update:

From a4dfac507b130b986460ff9848610fbe6052e429 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 19 Aug 2020 14:57:59 +0530
Subject: [PATCH 324/361] fix(minor): use get_list in client.get_value for
 tighter checks

---
 frappe/client.py         | 12 +++++++++---
 frappe/tests/test_api.py | 32 ++++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/frappe/client.py b/frappe/client.py
index 045e28dd8f..a813e3ec55 100644
--- a/frappe/client.py
+++ b/frappe/client.py
@@ -75,17 +75,23 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 	filters = get_safe_filters(filters)
 
 	try:
-		fieldname = json.loads(fieldname)
+		fields = json.loads(fieldname)
 	except (TypeError, ValueError):
 		# name passed, not json
-		pass
+		fields = [fieldname]
 
 	# check whether the used filters were really parseable and usable
 	# and did not just result in an empty string or dict
 	if not filters:
 		filters = None
 
-	return frappe.db.get_value(doctype, filters, fieldname, as_dict=as_dict, debug=debug)
+	value = frappe.get_list(doctype, filters=filters, fields=fields, debug=debug, limit=1)
+	if as_dict:
+		value = value[0] if value else {}
+	else:
+		value = value[0].fieldname
+
+	return value
 
 @frappe.whitelist()
 def get_single_value(doctype, field):
diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 2472f3191d..0edc1d4759 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -4,18 +4,14 @@ from __future__ import unicode_literals
 
 import unittest, frappe, os
 from frappe.core.doctype.user.user import generate_keys
-from frappe.frappeclient import FrappeClient
+from frappe.frappeclient import FrappeClient, FrappeException
 from frappe.utils.data import get_url
 
 import requests
 import base64
 
 class TestAPI(unittest.TestCase):
-	def test_insert_many(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
-		frappe.db.commit()
-
+	def insert_docs(self):
 		server.insert_many([
 			{"doctype": "Note", "public": True, "title": "Sing"},
 			{"doctype": "Note", "public": True, "title": "a"},
@@ -24,6 +20,13 @@ class TestAPI(unittest.TestCase):
 			{"doctype": "Note", "public": True, "title": "sixpence"},
 		])
 
+	def test_insert_many(self, server):
+		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
+		frappe.db.commit()
+
+		self.insert_docs(server)
+
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'Sing'}))
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'a'}))
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'song'}))
@@ -41,6 +44,8 @@ class TestAPI(unittest.TestCase):
 
 	def test_list_docs(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		self.insert_docs(server)
+
 		doc_list = server.get_list("Note")
 
 		self.assertTrue(len(doc_list))
@@ -56,6 +61,21 @@ class TestAPI(unittest.TestCase):
 		doc = server.get_doc("Note", "get_this")
 		self.assertTrue(doc)
 
+	def test_get_value(self):
+		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		frappe.db.sql("delete from `tabNote` where title = 'get_value'")
+		frappe.db.commit()
+
+		test_content = "test get value"
+
+		server.insert_many([
+			{"doctype": "Note", "public": True, "title": "get_value", "content": test_content},
+		])
+		self.assertEqual(server.get_value("Note", "content", {"title": "get_value"}).get('content'), test_content)
+
+		self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"})
+
+
 	def test_update_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
 		frappe.db.sql("delete from `tabNote` where title in ('Sing','sing')")

From 89717b00606d8efbccef6b7ad0cbd540f827e7b2 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 19 Aug 2020 15:03:17 +0530
Subject: [PATCH 325/361] fix(tests): test_insert_many

---
 frappe/tests/test_api.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 0edc1d4759..3e559ac0f3 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -11,7 +11,11 @@ import requests
 import base64
 
 class TestAPI(unittest.TestCase):
-	def insert_docs(self):
+	def test_insert_many(self):
+		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
+		frappe.db.commit()
+
 		server.insert_many([
 			{"doctype": "Note", "public": True, "title": "Sing"},
 			{"doctype": "Note", "public": True, "title": "a"},
@@ -20,13 +24,6 @@ class TestAPI(unittest.TestCase):
 			{"doctype": "Note", "public": True, "title": "sixpence"},
 		])
 
-	def test_insert_many(self, server):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')")
-		frappe.db.commit()
-
-		self.insert_docs(server)
-
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'Sing'}))
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'a'}))
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'song'}))
@@ -44,8 +41,6 @@ class TestAPI(unittest.TestCase):
 
 	def test_list_docs(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
-		self.insert_docs(server)
-
 		doc_list = server.get_list("Note")
 
 		self.assertTrue(len(doc_list))

From c72b68d0fb41938bdad915becf3374ab8ce92478 Mon Sep 17 00:00:00 2001
From: barredterra <14891507+barredterra@users.noreply.github.com>
Date: Wed, 19 Aug 2020 11:53:13 +0200
Subject: [PATCH 326/361] fix: remove obsolete argument

---
 frappe/website/website_generator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py
index c406679e54..fc08abeed9 100644
--- a/frappe/website/website_generator.py
+++ b/frappe/website/website_generator.py
@@ -154,7 +154,7 @@ class WebsiteGenerator(Document):
 		# Check if the route is changed
 		if old_doc and old_doc.route != self.route:
 			# Remove the route from index if the route has changed
-			remove_document_from_index("web_routes", old_doc.route)
+			remove_document_from_index(old_doc.route)
 
 	def update_website_search_index(self):
 		"""
@@ -169,4 +169,4 @@ class WebsiteGenerator(Document):
 			frappe.enqueue(update_index_for_path, path=self.route)
 		elif self.route:
 			# If the website is not published
-			remove_document_from_index("web_routes", self.route)
+			remove_document_from_index(self.route)

From 1545d5fc9fa1d12f1d13de29d8767baa75c8ac95 Mon Sep 17 00:00:00 2001
From: Revant Nandgaonkar <revant.one@gmail.com>
Date: Wed, 19 Aug 2020 20:13:35 +0530
Subject: [PATCH 327/361] fix: browserslist caniuse-lite is outdated

---
 yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index e8f0f40d09..96edef8766 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -921,9 +921,9 @@ caniuse-api@^3.0.0:
     lodash.uniq "^4.5.0"
 
 caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939:
-  version "1.0.30000940"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000940.tgz#19f2b1497fbfa5b96b615963097c3757f27989ce"
-  integrity sha512-rp/086IBUfCsNgBpko6DGQv674jRjeXPesDatDB2kxrkmDfD+S5Gesw+uT8YjpRWvLKLMRBy72SLRZ8I0EgQFw==
+  version "1.0.30001116"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001116.tgz"
+  integrity sha512-f2lcYnmAI5Mst9+g0nkMIznFGsArRmZ0qU+dnq8l91hymdc2J3SFbiPhOJEeDqC1vtE8nc1qNQyklzB8veJefQ==
 
 capture-stack-trace@^1.0.0:
   version "1.0.1"

From 06310553690e85ba3e06e423f1b960f7fffe5e0f Mon Sep 17 00:00:00 2001
From: Mangesh-Khairnar <mkhairnar10@gmail.com>
Date: Wed, 19 Aug 2020 22:32:45 +0530
Subject: [PATCH 328/361] feat: add validation for twilio credentials

---
 .../desk_page/integrations/integrations.json     | 16 ++++++++--------
 .../twilio_number_group/twilio_number_group.json |  6 +++++-
 .../doctype/twilio_settings/twilio_settings.py   | 11 ++++++++++-
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/frappe/integrations/desk_page/integrations/integrations.json b/frappe/integrations/desk_page/integrations/integrations.json
index 9201e223f8..cbf7c9c085 100644
--- a/frappe/integrations/desk_page/integrations/integrations.json
+++ b/frappe/integrations/desk_page/integrations/integrations.json
@@ -10,11 +10,6 @@
    "label": "Google Services",
    "links": "[\n    {\n        \"description\": \"Google API Settings.\",\n        \"label\": \"Google Settings\",\n        \"name\": \"Google Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Google Contacts Integration.\",\n        \"label\": \"Google Contacts\",\n        \"name\": \"Google Contacts\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Google Calendar Integration.\",\n        \"label\": \"Google Calendar\",\n        \"name\": \"Google Calendar\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Google Drive Integration.\",\n        \"label\": \"Google Drive\",\n        \"name\": \"Google Drive\",\n        \"type\": \"doctype\"\n    }\n]"
   },
-  {
-   "hidden": 0,
-   "label": "Webhook",
-   "links": "[\n    {\n        \"description\": \"Webhooks calling API requests into web apps\",\n        \"label\": \"Webhook\",\n        \"name\": \"Webhook\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Slack Webhooks for internal integration\",\n        \"label\": \"Slack Webhook URL\",\n        \"name\": \"Slack Webhook URL\",\n        \"type\": \"doctype\"\n    }\n]"
-  },
   {
    "hidden": 0,
    "label": "Authentication",
@@ -23,7 +18,12 @@
   {
    "hidden": 0,
    "label": "Payments",
-   "links": "[\n    {\n        \"description\": \"Braintree payment gateway settings\",\n        \"label\": \"Braintree Settings\",\n        \"name\": \"Braintree Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"PayPal payment gateway settings\",\n        \"label\": \"PayPal Settings\",\n        \"name\": \"PayPal Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Razorpay Payment gateway settings\",\n        \"label\": \"Razorpay Settings\",\n        \"name\": \"Razorpay Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Stripe payment gateway settings\",\n        \"label\": \"Stripe Settings\",\n        \"name\": \"Stripe Settings\",\n        \"type\": \"doctype\"\n    }\n]"
+   "links": "[\n    {\n        \"description\": \"Braintree payment gateway settings\",\n        \"label\": \"Braintree Settings\",\n        \"name\": \"Braintree Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"PayPal payment gateway settings\",\n        \"label\": \"PayPal Settings\",\n        \"name\": \"PayPal Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Razorpay Payment gateway settings\",\n        \"label\": \"Razorpay Settings\",\n        \"name\": \"Razorpay Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Stripe payment gateway settings\",\n        \"label\": \"Stripe Settings\",\n        \"name\": \"Stripe Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Paytm payment gateway settings\",\n        \"label\": \"Paytm Settings\",\n        \"name\": \"Paytm Settings\",\n        \"type\": \"doctype\"\n    }\n]"
+  },
+  {
+   "hidden": 0,
+   "label": "Settings",
+   "links": "[\n    {\n        \"description\": \"Webhooks calling API requests into web apps\",\n        \"label\": \"Webhook\",\n        \"name\": \"Webhook\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Slack Webhooks for internal integration\",\n        \"label\": \"Slack Webhook URL\",\n        \"name\": \"Slack Webhook URL\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"Twilio Settings for WhatsApp integration\",\n        \"label\": \"Twilio Settings\",\n        \"name\": \"Twilio Settings\",\n        \"type\": \"doctype\"\n    },\n    {\n        \"description\": \"SMS Settings for sending sms\",\n        \"label\": \"SMS Settings\",\n        \"name\": \"SMS Settings\",\n        \"type\": \"doctype\"\n    }\n]"
   }
  ],
  "category": "Administration",
@@ -34,11 +34,11 @@
  "docstatus": 0,
  "doctype": "Desk Page",
  "extends_another_page": 0,
- "icon": "frapicon-dashboard",
+ "hide_custom": 0,
  "idx": 0,
  "is_standard": 1,
  "label": "Integrations",
- "modified": "2020-04-01 11:24:40.751651",
+ "modified": "2020-08-20 23:04:04.528572",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Integrations",
diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
index 1790581ca7..9d51e4b452 100644
--- a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
+++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json
@@ -14,12 +14,16 @@
    "fieldtype": "Data",
    "in_list_view": 1,
    "label": "Phone Number",
+   "options": "Phone",
+   "show_days": 1,
+   "show_seconds": 1,
    "unique": 1
   }
  ],
+ "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-03-02 14:54:34.396254",
+ "modified": "2020-08-20 22:48:57.166791",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Twilio Number Group",
diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.py b/frappe/integrations/doctype/twilio_settings/twilio_settings.py
index ba0565b3af..6c698d719a 100644
--- a/frappe/integrations/doctype/twilio_settings/twilio_settings.py
+++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.py
@@ -11,7 +11,16 @@ from frappe.utils.password import get_decrypted_password
 from six import string_types
 
 class TwilioSettings(Document):
-	pass
+	def validate(self):
+		self.validate_twilio_credentials()
+
+	def validate_twilio_credentials(self):
+		try:
+			auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
+			client = Client(self.account_sid, auth_token)
+			client.api.accounts(self.account_sid).fetch()
+		except Exception:
+			frappe.throw(_("Invalid Account SID or Auth Token."))
 
 def send_whatsapp_message(sender, receiver_list, message):
 	import json

From f4bdf61a67c75b65cbfa08a28e982058bff9ee10 Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 20 Aug 2020 11:42:12 +0530
Subject: [PATCH 329/361] fix: Duplicate Toggle width button in Navbar Settings

---
 frappe/patches.txt                                 |  1 +
 .../v13_0/add_toggle_width_in_navbar_settings.py   |  3 +++
 .../patches/v13_0/remove_duplicate_navbar_items.py | 14 ++++++++++++++
 3 files changed, 18 insertions(+)
 create mode 100644 frappe/patches/v13_0/remove_duplicate_navbar_items.py

diff --git a/frappe/patches.txt b/frappe/patches.txt
index 8657be1fc5..06a6dc6b19 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -301,3 +301,4 @@ frappe.patches.v13_0.generate_theme_files_in_public_folder
 frappe.patches.v13_0.increase_password_length
 frappe.patches.v13_0.add_toggle_width_in_navbar_settings
 frappe.patches.v13_0.rename_notification_fields
+frappe.patches.v13_0.remove_duplicate_navbar_items
diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
index 4d241cd8c7..59acb77480 100644
--- a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
+++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py
@@ -4,6 +4,9 @@ import frappe
 def execute():
 	navbar_settings = frappe.get_single("Navbar Settings")
 
+	if frappe.db.exists('Navbar Item', {'item_label': 'Toggle Full Width'}):
+		return
+
 	for navbar_item in navbar_settings.settings_dropdown[5:]:
 		navbar_item.idx = navbar_item.idx + 1
 
diff --git a/frappe/patches/v13_0/remove_duplicate_navbar_items.py b/frappe/patches/v13_0/remove_duplicate_navbar_items.py
new file mode 100644
index 0000000000..7076f53e7c
--- /dev/null
+++ b/frappe/patches/v13_0/remove_duplicate_navbar_items.py
@@ -0,0 +1,14 @@
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+	navbar_settings = frappe.get_single("Navbar Settings")
+	duplicate_items = []
+
+	for navbar_item in navbar_settings.settings_dropdown:
+		if navbar_item.item_label == 'Toggle Full Width':
+			duplicate_items.append(navbar_item)
+
+	if len(duplicate_items) > 1:
+		navbar_settings.remove(duplicate_item[0])
+		navbar_settings.save()

From c2f3b1184845727e27274f7a2065df832c8bb4ca Mon Sep 17 00:00:00 2001
From: Deepesh Garg <deepeshgarg6@gmail.com>
Date: Thu, 20 Aug 2020 11:48:45 +0530
Subject: [PATCH 330/361] fix: Typo

---
 frappe/patches/v13_0/remove_duplicate_navbar_items.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches/v13_0/remove_duplicate_navbar_items.py b/frappe/patches/v13_0/remove_duplicate_navbar_items.py
index 7076f53e7c..cb4de4ca07 100644
--- a/frappe/patches/v13_0/remove_duplicate_navbar_items.py
+++ b/frappe/patches/v13_0/remove_duplicate_navbar_items.py
@@ -10,5 +10,5 @@ def execute():
 			duplicate_items.append(navbar_item)
 
 	if len(duplicate_items) > 1:
-		navbar_settings.remove(duplicate_item[0])
+		navbar_settings.remove(duplicate_items[0])
 		navbar_settings.save()

From 275a70e9d0cb4591bd3622fbd54b4e1e6dda1271 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 20 Aug 2020 18:33:29 +0530
Subject: [PATCH 331/361] feat(minor): allow frappe.db.sql for read in server
 script

---
 frappe/tests/test_safe_exec.py | 11 +++++++++--
 frappe/utils/safe_exec.py      |  8 ++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/frappe/tests/test_safe_exec.py b/frappe/tests/test_safe_exec.py
index 65657b4739..95bbae6746 100644
--- a/frappe/tests/test_safe_exec.py
+++ b/frappe/tests/test_safe_exec.py
@@ -1,5 +1,5 @@
 from __future__ import unicode_literals
-import unittest
+import unittest, frappe
 from frappe.utils.safe_exec import safe_exec
 
 class TestSafeExec(unittest.TestCase):
@@ -7,4 +7,11 @@ class TestSafeExec(unittest.TestCase):
 		self.assertRaises(ImportError, safe_exec, 'import os')
 
 	def test_internal_attributes(self):
-		self.assertRaises(SyntaxError, safe_exec, '().__class__.__call__')
\ No newline at end of file
+		self.assertRaises(SyntaxError, safe_exec, '().__class__.__call__')
+
+	def test_sql(self):
+		_locals = dict(out=None)
+		safe_exec('''out = frappe.db.sql("select name from tabDocType where name='DocType'")''', None, _locals)
+		self.assertEqual(_locals['out'][0][0], 'DocType')
+
+		self.assertRaises(frappe.PermissionError, safe_exec, 'frappe.db.sql("update tabToDo set description=NULL")')
\ No newline at end of file
diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index d97e770d88..bae1dbd746 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -114,6 +114,7 @@ def get_safe_globals():
 			get_single_value = frappe.db.get_single_value,
 			get_default = frappe.db.get_default,
 			escape = frappe.db.escape,
+			sql = read_sql
 		)
 
 	if frappe.response:
@@ -132,6 +133,13 @@ def get_safe_globals():
 
 	return out
 
+def read_sql(query, *args, **kwargs):
+	'''a wrapper for frappe.db.sql to allow reads'''
+	if query.strip().split(None, 1)[0].lower() == 'select':
+		return frappe.db.sql(query, *args, **kwargs)
+	else:
+		raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
+
 def _getitem(obj, key):
 	# guard function for RestrictedPython
 	# allow any key to be accessed as long as it does not start with underscore

From 1a50b594ddaccd4fe059b56708b91acb71595cf6 Mon Sep 17 00:00:00 2001
From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Date: Thu, 20 Aug 2020 18:45:18 +0530
Subject: [PATCH 332/361] fix: Re-run reload_countries_and_currencies patch
 (#11327)

- to create newly introduced currency on existing sites
---
 frappe/patches.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/patches.txt b/frappe/patches.txt
index 8657be1fc5..54d7512f1b 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -205,7 +205,7 @@ frappe.patches.v9_1.resave_domain_settings
 frappe.patches.v9_1.revert_domain_settings
 frappe.patches.v9_1.move_feed_to_activity_log
 execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True)
-frappe.patches.v10_0.reload_countries_and_currencies
+frappe.patches.v10_0.reload_countries_and_currencies # 20-10-2020
 frappe.patches.v10_0.refactor_social_login_keys
 frappe.patches.v10_0.enable_chat_by_default_within_system_settings
 frappe.patches.v10_0.remove_custom_field_for_disabled_domain

From eb87e6b77358c8bddd860cea90592d08891871bb Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Fri, 21 Aug 2020 13:30:20 +0530
Subject: [PATCH 333/361] fix: Set fieldname when attaching to field

---
 frappe/public/js/frappe/file_uploader/FileUploader.vue | 7 +++++++
 frappe/public/js/frappe/file_uploader/index.js         | 2 ++
 frappe/public/js/frappe/form/controls/attach.js        | 1 +
 3 files changed, 10 insertions(+)

diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue
index 845fbf92b4..f8d9253931 100644
--- a/frappe/public/js/frappe/file_uploader/FileUploader.vue
+++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue
@@ -144,6 +144,9 @@ export default {
 		docname: {
 			default: null
 		},
+		fieldname: {
+			default: null
+		},
 		folder: {
 			default: 'Home'
 		},
@@ -406,6 +409,10 @@ export default {
 					form_data.append('docname', this.docname);
 				}
 
+				if (this.fieldname) {
+					form_data.append('fieldname', this.fieldname);
+				}
+
 				if (this.method) {
 					form_data.append('method', this.method);
 				}
diff --git a/frappe/public/js/frappe/file_uploader/index.js b/frappe/public/js/frappe/file_uploader/index.js
index f81a1f1776..62a7bff822 100644
--- a/frappe/public/js/frappe/file_uploader/index.js
+++ b/frappe/public/js/frappe/file_uploader/index.js
@@ -7,6 +7,7 @@ export default class FileUploader {
 		on_success,
 		doctype,
 		docname,
+		fieldname,
 		files,
 		folder,
 		restrictions,
@@ -28,6 +29,7 @@ export default class FileUploader {
 					show_upload_button: !Boolean(this.dialog),
 					doctype,
 					docname,
+					fieldname,
 					method,
 					folder,
 					on_success,
diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js
index 15cbd3b043..fe662c1ada 100644
--- a/frappe/public/js/frappe/form/controls/attach.js
+++ b/frappe/public/js/frappe/form/controls/attach.js
@@ -66,6 +66,7 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({
 		if (this.frm) {
 			options.doctype = this.frm.doctype;
 			options.docname = this.frm.docname;
+			options.fieldname = this.df.fieldname;
 		}
 
 		if (this.df.options) {

From 85cac9122e9e2d9a478d455e3049ef47f38d9a9e Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchamp.faris@gmail.com>
Date: Fri, 21 Aug 2020 13:31:07 +0530
Subject: [PATCH 334/361] test: For file attachment on update

---
 frappe/core/doctype/file/test_file.py | 46 +++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py
index ec4f97bf67..85397ea1ee 100644
--- a/frappe/core/doctype/file/test_file.py
+++ b/frappe/core/doctype/file/test_file.py
@@ -328,3 +328,49 @@ class TestFile(unittest.TestCase):
 		self.assertTrue(os.path.exists(file2.get_full_path()))
 
 
+class TestAttachment(unittest.TestCase):
+	test_doctype = 'Test For Attachment'
+
+	def setUp(self):
+		if frappe.db.exists('DocType', self.test_doctype):
+			return
+
+		frappe.get_doc(
+			doctype='DocType',
+			name=self.test_doctype,
+			module='Custom',
+			custom=1,
+			fields=[
+				{'label': 'Title', 'fieldname': 'title', 'fieldtype': 'Data'},
+				{'label': 'Attachment', 'fieldname': 'attachment', 'fieldtype': 'Attach'},
+			]
+		).insert()
+
+	def tearDown(self):
+		frappe.delete_doc('DocType', self.test_doctype)
+
+	def test_file_attachment_on_update(self):
+		doc = frappe.get_doc(
+			doctype=self.test_doctype,
+			title='test for attachment on update'
+		).insert()
+
+		file = frappe.get_doc({
+			'doctype': 'File',
+			'file_name': 'test_attach.txt',
+			'content': 'Test Content'
+		})
+		file.save()
+
+		doc.attachment = file.file_url
+		doc.save()
+
+		exists = frappe.db.exists('File', {
+			'file_name': 'test_attach.txt',
+			'file_url': file.file_url,
+			'attached_to_doctype': self.test_doctype,
+			'attached_to_name': doc.name,
+			'attached_to_field': 'attachment'
+		})
+
+		self.assertTrue(exists)

From ffa785bb479b1f84b8b4d5d4a43c18cbaabdf301 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Fri, 21 Aug 2020 15:14:59 +0530
Subject: [PATCH 335/361] fix(client.py): fix get_value for name

---
 frappe/client.py         | 2 ++
 frappe/tests/test_api.py | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/frappe/client.py b/frappe/client.py
index a813e3ec55..6f19e78183 100644
--- a/frappe/client.py
+++ b/frappe/client.py
@@ -73,6 +73,8 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 		frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError)
 
 	filters = get_safe_filters(filters)
+	if isinstance(filters, string_types):
+		filters = {"name": filters}
 
 	try:
 		fields = json.loads(fieldname)
diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 3e559ac0f3..3139237cc9 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -67,6 +67,10 @@ class TestAPI(unittest.TestCase):
 			{"doctype": "Note", "public": True, "title": "get_value", "content": test_content},
 		])
 		self.assertEqual(server.get_value("Note", "content", {"title": "get_value"}).get('content'), test_content)
+		name = server.get_value("Note", "name", {"title": "get_value"}).get('name')
+
+		# test by name
+		self.assertEqual(server.get_value("Note", "content", name).get('content'), test_content)
 
 		self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"})
 

From fa1e8aef0271f11390f63e6a01152a38151182a0 Mon Sep 17 00:00:00 2001
From: Saurabh <saurabh@erpnext.com>
Date: Fri, 21 Aug 2020 15:58:32 +0530
Subject: [PATCH 336/361] fix: upload file backup on S3 (#11326)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1. if file backup is scheduled, and system not have file backup tar t…hen prepare files bakcup before uploading it to s3

2. fix regex for public files
---
 .../doctype/s3_backup_settings/s3_backup_settings.py   |  7 ++++++-
 frappe/integrations/offsite_backup_utils.py            | 10 ++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
index c8b007ba7b..470b301204 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
@@ -8,7 +8,7 @@ import os.path
 import frappe
 import boto3
 from frappe import _
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
+from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, generate_files_backup
 from frappe.model.document import Document
 from frappe.utils import cint
 from frappe.utils.background_jobs import enqueue
@@ -125,6 +125,11 @@ def backup_to_s3():
 	else:
 		if backup_files:
 			db_filename, site_config, files_filename, private_files = get_latest_backup_file(with_files=backup_files)
+
+			if not files_filename or not private_files:
+				generate_files_backup()
+				db_filename, site_config, files_filename, private_files = get_latest_backup_file(with_files=backup_files)
+
 		else:
 			db_filename, site_config = get_latest_backup_file()
 
diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index a551d8edf1..7d33dd14bc 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -90,3 +90,13 @@ def validate_file_size():
 
 	if file_size > 1:
 		frappe.flags.create_new_backup = False
+
+def generate_files_backup():
+	from frappe.utils.backups import BackupGenerator
+
+	backup = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name,
+		frappe.conf.db_password, db_host = frappe.db.host,
+		db_type=frappe.conf.db_type, db_port=frappe.conf.db_port)
+
+	backup.set_backup_file_name()
+	backup.zip_files()
\ No newline at end of file

From 2b80e710bc7f982b6a0d30cd6d296bba089766ac Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Sat, 22 Aug 2020 21:15:45 +0530
Subject: [PATCH 337/361] fix(minor): client.get_value fix for single

---
 frappe/client.py         | 7 ++++++-
 frappe/tests/test_api.py | 5 +++++
 requirements.txt         | 2 +-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/frappe/client.py b/frappe/client.py
index 6f19e78183..158a143ad0 100644
--- a/frappe/client.py
+++ b/frappe/client.py
@@ -87,7 +87,12 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 	if not filters:
 		filters = None
 
-	value = frappe.get_list(doctype, filters=filters, fields=fields, debug=debug, limit=1)
+
+	if frappe.get_meta(doctype).issingle:
+		value = frappe.db.get_values_from_single(fields, None, doctype, as_dict=as_dict, debug=debug)
+	else:
+		value = frappe.get_list(doctype, filters=filters, fields=fields, debug=debug, limit=1)
+
 	if as_dict:
 		value = value[0] if value else {}
 	else:
diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 3139237cc9..45107612cc 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -74,6 +74,11 @@ class TestAPI(unittest.TestCase):
 
 		self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"})
 
+	def test_get_single(self):
+		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix')
+		self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix')
+		frappe.db.set_value('Website Settings', None, 'title_prefix', '')
 
 	def test_update_doc(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
diff --git a/requirements.txt b/requirements.txt
index a88d0e3c1d..af3104cce7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -34,7 +34,7 @@ oauthlib==3.1.0
 openpyxl==2.6.4
 passlib==1.7.2
 pdfkit==0.6.1
-Pillow==6.2.2
+Pillow==7.1.0
 premailer==3.6.1
 psycopg2-binary==2.8.4
 pyasn1==0.4.8

From 7895ddf8654094dc9f8ea157beb651696c116d1e Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Sat, 22 Aug 2020 21:18:38 +0530
Subject: [PATCH 338/361] fix(minor): client.get_value fix for single

---
 frappe/client.py         | 2 +-
 frappe/tests/test_api.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/client.py b/frappe/client.py
index 158a143ad0..0db18421ef 100644
--- a/frappe/client.py
+++ b/frappe/client.py
@@ -89,7 +89,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 
 
 	if frappe.get_meta(doctype).issingle:
-		value = frappe.db.get_values_from_single(fields, None, doctype, as_dict=as_dict, debug=debug)
+		value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug)
 	else:
 		value = frappe.get_list(doctype, filters=filters, fields=fields, debug=debug, limit=1)
 
diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 45107612cc..01ad11dde3 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -77,6 +77,7 @@ class TestAPI(unittest.TestCase):
 	def test_get_single(self):
 		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
 		server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix')
+		self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix')
 		self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix')
 		frappe.db.set_value('Website Settings', None, 'title_prefix', '')
 

From 07f5484ef29bca0e72987316edd94ef4de41b3e4 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Sun, 23 Aug 2020 13:19:38 +0530
Subject: [PATCH 339/361] fix(test): attempt to fix recorder test

---
 cypress/integration/recorder.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js
index 878d8af848..ed2a9c86ba 100644
--- a/cypress/integration/recorder.js
+++ b/cypress/integration/recorder.js
@@ -61,7 +61,7 @@ context('Recorder', () => {
 
 		cy.visit('/desk#recorder');
 
-		cy.contains('.list-row-container span', 'frappe.desk.reportview.get').click();
+		cy.get('.list-row-container span').contains('frappe.desk.reportview.get').click();
 
 		cy.location('hash').should('contain', '#recorder/request/');
 		cy.get('form').should('contain', 'frappe.desk.reportview.get');

From df7a57529541bf02d12c93fa90c7c9f1d1862a45 Mon Sep 17 00:00:00 2001
From: Saurabh <saurabh@erpnext.com>
Date: Mon, 24 Aug 2020 11:55:28 +0530
Subject: [PATCH 340/361] fix: Sanitize 2FA response (#11263)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
---
 frappe/twofactor.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/frappe/twofactor.py b/frappe/twofactor.py
index 253636764f..dfa3f394e5 100644
--- a/frappe/twofactor.py
+++ b/frappe/twofactor.py
@@ -187,9 +187,7 @@ def process_2fa_for_otp_app(user, otp_secret, otp_issuer):
 		otp_setup_completed = False
 
 	verification_obj = {
-		'totp_uri': totp_uri,
 		'method': 'OTP App',
-		'qrcode': get_qr_svg_code(totp_uri),
 		'setup': otp_setup_completed
 	}
 	return verification_obj

From 65165b6be636b3360d361f926e864a84c2fd907f Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 24 Aug 2020 13:09:20 +0530
Subject: [PATCH 341/361] fix: translation syntax

---
 frappe/core/page/permission_manager/permission_manager.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js
index 7608b25766..0d3267c7d5 100644
--- a/frappe/core/page/permission_manager/permission_manager.js
+++ b/frappe/core/page/permission_manager/permission_manager.js
@@ -376,8 +376,7 @@ frappe.PermissionEngine = Class.extend({
 							options:me.options.roles, reqd:1,fieldname:"role"},
 						{fieldtype:"Select", label:__("Permission Level"),
 							options:[0,1,2,3,4,5,6,7,8,9], reqd:1, fieldname: "permlevel",
-							description: __("Level 0 is for document level permissions, \
-								higher levels for field level permissions.")}
+							description: __("Level 0 is for document level permissions, higher levels for field level permissions.")}
 					]
 				});
 				if(me.get_doctype()) {

From 246938f779b7fd255e782906ef04023a92e51d38 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scmmishra@users.noreply.github.com>
Date: Mon, 24 Aug 2020 07:47:32 +0000
Subject: [PATCH 342/361] fix: use list

Co-authored-by: Rohan <Alchez@users.noreply.github.com>
---
 frappe/desk/query_report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 656b49046a..db945915e9 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -226,7 +226,7 @@ def reorder_data_for_custom_columns(custom_columns, columns, result, report_type
 		original_column_labels = [col.split(":")[0] for col in columns]
 		return get_columns_from_list(custom_column_labels, original_column_labels, result)
 
-	if type(result[0]) == "list":
+	if isinstance(result[0], list):
 		# If the result is a list of lists
 		original_column_names = [col["fieldname"] for col in columns]
 		return get_columns_from_list(custom_column_names, original_column_names, result)

From 96a3d35d6c1bdf20e3a274b54ac3e2acef5e7617 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 24 Aug 2020 13:22:11 +0530
Subject: [PATCH 343/361] fix: add custom_column_names

---
 frappe/desk/query_report.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index db945915e9..7a06859d20 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -221,11 +221,12 @@ def reorder_data_for_custom_columns(custom_columns, columns, result, report_type
 
 	if report_type == 'Query Report':
 		# Assume list result for query reports
-		# Query report columns exclisvuely use Label
+		# Query report columns exclusively use Label
 		custom_column_labels = [col["label"] for col in custom_columns]
 		original_column_labels = [col.split(":")[0] for col in columns]
 		return get_columns_from_list(custom_column_labels, original_column_labels, result)
 
+	custom_column_names = [col["fieldname"] for col in custom_columns]
 	if isinstance(result[0], list):
 		# If the result is a list of lists
 		original_column_names = [col["fieldname"] for col in columns]

From 9ba45f03607a4480fd06327c049a7195eca85061 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Mon, 24 Aug 2020 13:59:28 +0530
Subject: [PATCH 344/361] fix: handle tuple and list

---
 frappe/desk/query_report.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 7a06859d20..8952d6290a 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -227,11 +227,12 @@ def reorder_data_for_custom_columns(custom_columns, columns, result, report_type
 		return get_columns_from_list(custom_column_labels, original_column_labels, result)
 
 	custom_column_names = [col["fieldname"] for col in custom_columns]
-	if isinstance(result[0], list):
+	if isinstance(result[0], list) or isinstance(result[0], tuple):
 		# If the result is a list of lists
 		original_column_names = [col["fieldname"] for col in columns]
 		return get_columns_from_list(custom_column_names, original_column_names, result)
 	else:
+		print(result[0])
 		# If the result is a list of dicts
 		return get_columns_from_dict(custom_column_names, result)
 

From 534cf5c512dd9ac124ad193ef47f92ce50dcceaf Mon Sep 17 00:00:00 2001
From: Saurabh <saurabh@erpnext.com>
Date: Mon, 24 Aug 2020 16:38:51 +0530
Subject: [PATCH 345/361] fix: Handle empty conditions in s3 backup (#11258)

---
 .../s3_backup_settings/s3_backup_settings.py       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
index 470b301204..c59b0ddd5b 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
@@ -138,9 +138,14 @@ def backup_to_s3():
 
 	upload_file_to_s3(db_filename, folder, conn, bucket)
 	upload_file_to_s3(site_config, folder, conn, bucket)
+
 	if backup_files:
-		upload_file_to_s3(private_files, folder, conn, bucket)
-		upload_file_to_s3(files_filename, folder, conn, bucket)
+		if private_files:
+			upload_file_to_s3(private_files, folder, conn, bucket)
+
+		if files_filename:
+			upload_file_to_s3(files_filename, folder, conn, bucket)
+
 	delete_old_backups(doc.backup_limit, bucket)
 
 
@@ -165,14 +170,15 @@ def delete_old_backups(limit, bucket):
 			aws_access_key_id=doc.access_key_id,
 			aws_secret_access_key=doc.get_password('secret_access_key'),
 			endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com'
-			)
+		)
+
 	bucket = s3.Bucket(bucket)
 	objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/')
 	if objects:
 		for obj in objects.get('CommonPrefixes'):
 			all_backups.append(obj.get('Prefix'))
 
-	oldest_backup = sorted(all_backups)[0]
+	oldest_backup = sorted(all_backups)[0] if all_backups else ''
 
 	if len(all_backups) > backup_limit:
 		print("Deleting Backup: {0}".format(oldest_backup))

From d4ae8830076c79d4db849da331854127afbf2fe8 Mon Sep 17 00:00:00 2001
From: Prssanna Desai <prssud@gmail.com>
Date: Mon, 24 Aug 2020 19:40:57 +0530
Subject: [PATCH 346/361] fix: remove print statement

---
 frappe/desk/query_report.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 8952d6290a..a1cfd02132 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -232,7 +232,6 @@ def reorder_data_for_custom_columns(custom_columns, columns, result, report_type
 		original_column_names = [col["fieldname"] for col in columns]
 		return get_columns_from_list(custom_column_names, original_column_names, result)
 	else:
-		print(result[0])
 		# If the result is a list of dicts
 		return get_columns_from_dict(custom_column_names, result)
 

From 1c5cff7746bdf607d556a5c042d637bf3cec1967 Mon Sep 17 00:00:00 2001
From: Rohan <rohan@bloomstack.com>
Date: Mon, 24 Aug 2020 23:39:47 +0530
Subject: [PATCH 347/361] fix: preload URLs breaking for external requests

---
 frappe/website/render.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/website/render.py b/frappe/website/render.py
index 73fb3fbd38..a5842f0694 100644
--- a/frappe/website/render.py
+++ b/frappe/website/render.py
@@ -149,8 +149,8 @@ def add_preload_headers(response):
 			preload.append(("style", elem.get("href")))
 
 		links = []
-		for type, link in preload:
-			links.append("</{}>; rel=preload; as={}".format(link.lstrip("/"), type))
+		for _type, link in preload:
+			links.append("<{}>; rel=preload; as={}".format(link, _type))
 
 		if links:
 			response.headers["Link"] = ",".join(links)

From c12668ecbf3f6fd6862e4202f71ebfcf08807b52 Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Mon, 24 Aug 2020 23:07:36 +0000
Subject: [PATCH 348/361] fix: package.json, yarn.lock & .snyk to reduce
 vulnerabilities

The following vulnerabilities are fixed with a Snyk patch:
- https://snyk.io/vuln/SNYK-JS-LODASH-567746
---
 .snyk        |    4 +-
 package.json |    2 +-
 yarn.lock    | 2080 ++++++++++++++++++++++++++++++++++++++------------
 3 files changed, 1594 insertions(+), 492 deletions(-)

diff --git a/.snyk b/.snyk
index 0dfecc6136..6e7bb44986 100644
--- a/.snyk
+++ b/.snyk
@@ -1,5 +1,5 @@
 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.14.1
+version: v1.19.0
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
   SNYK-JS-AWESOMPLETE-174474:
@@ -63,3 +63,5 @@ patch:
         patched: '2020-04-30T23:02:32.330Z'
     - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash:
         patched: '2020-04-30T23:02:32.330Z'
+    - quill-image-resize > lodash:
+        patched: '2020-08-24T23:06:37.710Z'
diff --git a/package.json b/package.json
index d48d2a3f65..fe2ef2a694 100644
--- a/package.json
+++ b/package.json
@@ -43,7 +43,7 @@
     "qz-tray": "^2.0.8",
     "redis": "^2.8.0",
     "showdown": "^1.9.1",
-    "snyk": "^1.316.1",
+    "snyk": "^1.382.0",
     "socket.io": "^2.3.0",
     "superagent": "^3.8.2",
     "touch": "^3.1.0",
diff --git a/yarn.lock b/yarn.lock
index 93ce879ebb..9e85a639aa 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,13 @@
 # yarn lockfile v1
 
 
+"@arcanis/slice-ansi@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@arcanis/slice-ansi/-/slice-ansi-1.0.2.tgz#35331e41a1062e3c53c01ad2ec1555c5c1959d8f"
+  integrity sha512-lDL63z0W/L/WTgqrwVOuNyMAsTv+pvjybd21z9SWdStmQoXT59E/iVWwat3gYjcdTNBf6oHAMoyFm8dtjpXEYw==
+  dependencies:
+    grapheme-splitter "^1.0.4"
+
 "@babel/code-frame@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
@@ -43,12 +50,41 @@
     debug "^3.1.0"
     lodash.once "^4.1.1"
 
-"@snyk/cli-interface@1.5.0":
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-1.5.0.tgz#b9dbe6ebfb86e67ffabf29d4e0d28a52670ac456"
-  integrity sha512-+Qo+IO3YOXWgazlo+CKxOuWFLQQdaNCJ9cSfhFQd687/FuesaIxWdInaAdfpsLScq0c6M1ieZslXgiZELSzxbg==
+"@nodelib/fs.scandir@2.1.3":
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz#3a582bdb53804c6ba6d146579c46e52130cf4a3b"
+  integrity sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==
   dependencies:
-    tslib "^1.9.3"
+    "@nodelib/fs.stat" "2.0.3"
+    run-parallel "^1.1.9"
+
+"@nodelib/fs.stat@2.0.3", "@nodelib/fs.stat@^2.0.2":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz#34dc5f4cabbc720f4e60f75a747e7ecd6c175bd3"
+  integrity sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==
+
+"@nodelib/fs.walk@^1.2.3":
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz#011b9202a70a6366e436ca5c065844528ab04976"
+  integrity sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==
+  dependencies:
+    "@nodelib/fs.scandir" "2.1.3"
+    fastq "^1.6.0"
+
+"@sindresorhus/is@^0.14.0":
+  version "0.14.0"
+  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea"
+  integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==
+
+"@sindresorhus/is@^2.1.1":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-2.1.1.tgz#ceff6a28a5b4867c2dd4a1ba513de278ccbe8bb1"
+  integrity sha512-/aPsuoj/1Dw/kzhkgz+ES6TxG0zfTMGLwuK2ZG00k/iJzYHTLCE8mVU8EPqEOp/lmxPoq1C1C9RYToRKb2KEfg==
+
+"@sindresorhus/is@^3.0.0":
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-3.1.2.tgz#548650de521b344e3781fbdb0ece4aa6f729afb8"
+  integrity sha512-JiX9vxoKMmu8Y3Zr2RVathBL1Cdu4Nt4MuNWemt1Nc06A0RAin9c5FArkhGsyMBWfCu4zj+9b+GxtjAnE4qqLQ==
 
 "@snyk/cli-interface@2.3.2", "@snyk/cli-interface@^2.0.3":
   version "2.3.2"
@@ -57,30 +93,40 @@
   dependencies:
     tslib "^1.9.3"
 
-"@snyk/cli-interface@2.5.0":
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-2.5.0.tgz#ac4176063dec629a42b13e349a200c2a6ceb028f"
-  integrity sha512-XMc2SCFH4RBSncZgoPb+BBlNq0NYpEpCzptKi69qyMpBy0VsRqIQqddedaazMCU1xEpXTytq6KMYpzUafZzp5Q==
-  dependencies:
-    tslib "^1.9.3"
-
-"@snyk/cli-interface@2.6.0":
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-2.6.0.tgz#e64a00af0304b2055fe786f17325346cacdd4912"
-  integrity sha512-jtk0gf80v4mFyDqaQNokD8GOPMTXpIUL35ewg6jtmuZw41xt56WF9kqCjiiViSRRRYA0RK+RuiVfmJA5pxvMUQ==
+"@snyk/cli-interface@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-2.6.1.tgz#1e4e7acf5f9aca1267ddf3c1bb98a469011c907a"
+  integrity sha512-3X+OwwwT9j0r2ObqxYIiAgdaHsTW71b92PN3wawGAxl4YgPRrRVw8Fouhe41I4WJsn7OlKUNedylZguvpYg9qw==
   dependencies:
     "@snyk/graphlib" "2.1.9-patch"
     tslib "^1.9.3"
 
-"@snyk/cocoapods-lockfile-parser@3.2.0":
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/@snyk/cocoapods-lockfile-parser/-/cocoapods-lockfile-parser-3.2.0.tgz#d59d82a89fb8ea4c8ab7f8068f94b8b14d0a9b39"
-  integrity sha512-DyFqZudOlGXHBOVneLnQnyQ97xZLq+PTF9PhWOmrEzH/tKcLyXhdW/WmDPVNJVyNvogyRZ4cXIj487xy/EeZEw==
+"@snyk/cli-interface@2.8.0":
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-2.8.0.tgz#ee5b45b7c75942163875b29e712c44f9d7f36bb3"
+  integrity sha512-St/G39iJG1zQK15L24kcVYM2gmFc/ylBCcBqU2DMZKJKwOPccKLUO6s+dWIUXMccQ+DFS6TuHPvuAKQNi9C4Yg==
   dependencies:
-    "@snyk/dep-graph" "1.18.2"
-    "@snyk/ruby-semver" "^2.0.4"
+    "@snyk/dep-graph" "1.19.0"
+    "@snyk/graphlib" "2.1.9-patch"
+    tslib "^1.9.3"
+
+"@snyk/cli-interface@2.8.1":
+  version "2.8.1"
+  resolved "https://registry.yarnpkg.com/@snyk/cli-interface/-/cli-interface-2.8.1.tgz#67a0c34e8473dd433d6c206c41fa8b0e12426621"
+  integrity sha512-pALcfgoY0hAavy/pBlDIqEu+FFC5m+D4bMnCwlQ26mObL/zzxp2+Ohx+HykCIom62u2J94SzAtRLFdm/2TgoOw==
+  dependencies:
+    "@snyk/dep-graph" "1.19.0"
+    "@snyk/graphlib" "2.1.9-patch"
+    tslib "^1.9.3"
+
+"@snyk/cocoapods-lockfile-parser@3.5.1":
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/@snyk/cocoapods-lockfile-parser/-/cocoapods-lockfile-parser-3.5.1.tgz#572907ae12abf77c1ea3dd497f7d1ced31104aa7"
+  integrity sha512-0bzajH/HdP3k5cOZKUmT/xqmHZFuWN124c/lrqh+U6Q1Z9Bt7TLOB2ifLKL+1I4rq+IgOesGWJYG1KhxBy3RLw==
+  dependencies:
+    "@snyk/dep-graph" "1.19.3"
+    "@snyk/ruby-semver" "^3.0.0"
     "@types/js-yaml" "^3.12.1"
-    core-js "^3.2.0"
     js-yaml "^3.13.1"
     source-map-support "^0.5.7"
     tslib "^1.10.0"
@@ -92,55 +138,39 @@
   dependencies:
     "@snyk/lodash" "^4.17.15-patch"
 
-"@snyk/configstore@3.2.0-rc1", "@snyk/configstore@^3.2.0-rc1":
-  version "3.2.0-rc1"
-  resolved "https://registry.yarnpkg.com/@snyk/configstore/-/configstore-3.2.0-rc1.tgz#385c050d11926a26d0335a4b3be9e55f90f6e0ac"
-  integrity sha512-CV3QggFY8BY3u8PdSSlUGLibqbqCG1zJRmGM2DhnhcxQDRRPTGTP//l7vJphOVsUP1Oe23+UQsj7KRWpRUZiqg==
-  dependencies:
-    dot-prop "^5.2.0"
-    graceful-fs "^4.1.2"
-    make-dir "^1.0.0"
-    unique-string "^1.0.0"
-    write-file-atomic "^2.0.0"
-    xdg-basedir "^3.0.0"
-
-"@snyk/dep-graph@1.18.2":
-  version "1.18.2"
-  resolved "https://registry.yarnpkg.com/@snyk/dep-graph/-/dep-graph-1.18.2.tgz#59bc60850b2e4624a2d4dfb6f72a416c82586c1e"
-  integrity sha512-v7tIiCH4LmYOSc0xGHKSxSZ2PEDv8zDlYU7ZKSH+1Hk8Qvj3YYEFvtV1iFBHUEQFUen4kQA6lWxlwF8chsNw+w==
+"@snyk/dep-graph@1.19.0":
+  version "1.19.0"
+  resolved "https://registry.yarnpkg.com/@snyk/dep-graph/-/dep-graph-1.19.0.tgz#dfb2699520225e715083f6dd590bb91b55e99ba1"
+  integrity sha512-/0phOICMk4hkX2KtZgi+4KNd5G9oYDIlxQDQk+ui2xl4gonPvK6Q5MFzHP7Xet1YY/XoU33ox41i+IO48qZ+zQ==
   dependencies:
     "@snyk/graphlib" "2.1.9-patch"
-    "@snyk/lodash" "4.17.15-patch"
-    object-hash "^1.3.1"
-    prettier "^1.19.1"
+    lodash.isequal "^4.5.0"
+    object-hash "^2.0.3"
     semver "^6.0.0"
-    source-map-support "^0.5.11"
+    source-map-support "^0.5.19"
+    tslib "^2.0.0"
+
+"@snyk/dep-graph@1.19.3", "@snyk/dep-graph@^1.17.0", "@snyk/dep-graph@^1.19.0":
+  version "1.19.3"
+  resolved "https://registry.yarnpkg.com/@snyk/dep-graph/-/dep-graph-1.19.3.tgz#f20521baf060f83f052fd6b55fad8b377833418d"
+  integrity sha512-WJLUFKBokoFK5imi0t8Dkyj+uqtS/5Ziuf4oE/OOFX30UqP1ffMDkv9/3sqBJQVQ9FjdgsX3Cm8JZMtMlYRc6w==
+  dependencies:
+    "@snyk/graphlib" "2.1.9-patch.2"
+    lodash.isequal "^4.5.0"
+    object-hash "^2.0.3"
+    semver "^6.0.0"
+    source-map-support "^0.5.19"
+    tslib "^1.13.0"
+
+"@snyk/docker-registry-v2-client@^1.13.5":
+  version "1.13.5"
+  resolved "https://registry.yarnpkg.com/@snyk/docker-registry-v2-client/-/docker-registry-v2-client-1.13.5.tgz#8d862f0c53d4a9a25db09cd48b4cd44aa8e385c9"
+  integrity sha512-lgJiC071abCpFVLp47OnykU8MMrhdQe386Wt6QaDmjI0s2DQn/S58NfdLrPU7s6l4zoGT7UwRW9+7paozRgFTA==
+  dependencies:
+    needle "^2.5.0"
+    parse-link-header "^1.0.1"
     tslib "^1.10.0"
 
-"@snyk/dep-graph@1.18.3":
-  version "1.18.3"
-  resolved "https://registry.yarnpkg.com/@snyk/dep-graph/-/dep-graph-1.18.3.tgz#7bf675f694ecbc329f67cca6456ae67f3f45f516"
-  integrity sha512-7qWRTIJdZuc5VzDjdV2+03AHElyAZmhq7eV9BRu+jqrYjo9ohWBGEZgYslrTdvfqfJ8rkdrG3j0/0Aa25IxJcg==
-  dependencies:
-    "@snyk/graphlib" "2.1.9-patch"
-    "@snyk/lodash" "4.17.15-patch"
-    object-hash "^2.0.3"
-    semver "^7.3.2"
-    source-map-support "^0.5.19"
-    tslib "^1.11.1"
-
-"@snyk/dep-graph@^1.18.2":
-  version "1.18.4"
-  resolved "https://registry.yarnpkg.com/@snyk/dep-graph/-/dep-graph-1.18.4.tgz#4dfb1681d56090ca1f04a3bcd6593cbb8bf1a576"
-  integrity sha512-SePWsDyD7qrLxFifIieEl4GqyOODfOnP0hmUweTG5YcMroAV5nARGAUcjxREGzbXMcUpPfZhAaqFjYgzUDH8dQ==
-  dependencies:
-    "@snyk/graphlib" "2.1.9-patch"
-    "@snyk/lodash" "4.17.15-patch"
-    object-hash "^2.0.3"
-    semver "^7.3.2"
-    source-map-support "^0.5.19"
-    tslib "^1.11.1"
-
 "@snyk/gemfile@1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@snyk/gemfile/-/gemfile-1.2.0.tgz#919857944973cce74c650e5428aaf11bcd5c0457"
@@ -153,6 +183,28 @@
   dependencies:
     "@snyk/lodash" "4.17.15-patch"
 
+"@snyk/graphlib@2.1.9-patch.2":
+  version "2.1.9-patch.2"
+  resolved "https://registry.yarnpkg.com/@snyk/graphlib/-/graphlib-2.1.9-patch.2.tgz#571255808f5bf291f42d51f1e1f0adc8f9145be9"
+  integrity sha512-BjJzOXDNzoEMBOjSks7vadu5f0c39SeorJMi9vUvvWM5dcE22CZqcN9VMRW5DYTifUJiCWszkm5TOyfYfB0bfg==
+  dependencies:
+    lodash.clone "^4.5.0"
+    lodash.constant "^3.0.0"
+    lodash.filter "^4.6.0"
+    lodash.foreach "^4.5.0"
+    lodash.has "^4.5.2"
+    lodash.isarray "^4.0.0"
+    lodash.isempty "^4.4.0"
+    lodash.isfunction "^3.0.9"
+    lodash.isundefined "^3.0.1"
+    lodash.keys "^4.2.0"
+    lodash.map "^4.6.0"
+    lodash.reduce "^4.6.0"
+    lodash.size "^4.2.0"
+    lodash.transform "^4.6.0"
+    lodash.union "^4.6.0"
+    lodash.values "^4.3.0"
+
 "@snyk/inquirer@6.2.2-patch":
   version "6.2.2-patch"
   resolved "https://registry.yarnpkg.com/@snyk/inquirer/-/inquirer-6.2.2-patch.tgz#14bfd111493eebdb8858f7ac4a98f8d274d10a91"
@@ -172,13 +224,12 @@
     strip-ansi "^5.0.0"
     through "^2.3.6"
 
-"@snyk/java-call-graph-builder@1.8.1":
-  version "1.8.1"
-  resolved "https://registry.yarnpkg.com/@snyk/java-call-graph-builder/-/java-call-graph-builder-1.8.1.tgz#89505a495de73dad22e69bf3d96572e6bd6b925f"
-  integrity sha512-2G96dChYYXV73G8y9U0fi45dH6ybOjUSRBTJrMnmNkHJoOp1bzz8L4p5rkRypHQqr4SBS1EdCQeRw1eWRLm+Lg==
+"@snyk/java-call-graph-builder@1.13.1":
+  version "1.13.1"
+  resolved "https://registry.yarnpkg.com/@snyk/java-call-graph-builder/-/java-call-graph-builder-1.13.1.tgz#641d0f929baa95cd30f8e16f938b4d21499a1e9b"
+  integrity sha512-oOCSIyOMplV73a1agcXKXlFYQftK5esUUaFRTf90GOxQwKy8R9tZtKdP+CdutlgvjRP286DQ+7GlvKYsGGZbWg==
   dependencies:
     "@snyk/graphlib" "2.1.9-patch"
-    "@snyk/lodash" "4.17.15-patch"
     ci-info "^2.0.0"
     debug "^4.1.1"
     glob "^7.1.6"
@@ -195,13 +246,12 @@
   resolved "https://registry.yarnpkg.com/@snyk/lodash/-/lodash-4.17.15-patch.tgz#fb61af14b75d10a20015b40af5d0423944af89dc"
   integrity sha512-e4+t34bGyjjRnwXwI14hqye9J/nRbG9iwaqTgXWHskm5qC+iK0UrjgYdWXiHJCf3Plbpr+1rpW+4LPzZnCGMhQ==
 
-"@snyk/rpm-parser@^1.1.0":
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/@snyk/rpm-parser/-/rpm-parser-1.1.0.tgz#cf9aaee87bc85133d2d791c293e5e590d1206346"
-  integrity sha512-+DyCagvnpyBjwYTxaPMQGLW4rkpKAw1Jrh8YbZCg7Ix172InBxdve/0zud18Lu2H6xWtDDdMvRDdfl82wlTBvA==
+"@snyk/rpm-parser@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@snyk/rpm-parser/-/rpm-parser-2.0.0.tgz#4ded7fa4b0a8efca7699359e4ca7a79bfbe38bc1"
+  integrity sha512-bWjQY5Xk3TcfVpeo8M5BhhSUEdPr2P19AWW13CHPu6sFZkckLWEcjQycnBsVD6RBmxGXecJ1YNui8dq6soHoYQ==
   dependencies:
-    event-loop-spinner "1.1.0"
-    typescript "3.8.3"
+    event-loop-spinner "^2.0.0"
 
 "@snyk/ruby-semver@2.2.0":
   version "2.2.0"
@@ -210,73 +260,116 @@
   dependencies:
     "@snyk/lodash" "4.17.15-patch"
 
-"@snyk/ruby-semver@^2.0.4":
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/@snyk/ruby-semver/-/ruby-semver-2.0.4.tgz#457686ea7a4d60b10efddde99587efb3a53ba884"
-  integrity sha512-ceMD4CBS3qtAg+O0BUvkKdsheUNCqi+/+Rju243Ul8PsUgZnXmGiqfk/2z7DCprRQnxUTra4+IyeDQT7wAheCQ==
+"@snyk/ruby-semver@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@snyk/ruby-semver/-/ruby-semver-3.0.0.tgz#592e648343a88eccb7d787a97d3ad6c9df1dd61f"
+  integrity sha512-GoSRcwNuJ/mK3Q+tqelRJlylPh8K3RZRWh3ZpkOKm1gQPdG+z0wt+LipSIHxGR8yBDl5bQjwTrPLkL49/N1V6Q==
   dependencies:
-    lodash "^4.17.14"
+    lodash.escaperegexp "^4.1.0"
+    lodash.flatten "^4.4.0"
+    lodash.uniq "^4.5.0"
+    tslib "^1.13.0"
 
-"@snyk/snyk-cocoapods-plugin@2.2.0":
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/@snyk/snyk-cocoapods-plugin/-/snyk-cocoapods-plugin-2.2.0.tgz#abf01c15ace3a0d2db298c08317340fb9092c78e"
-  integrity sha512-Ux7hXKawbk30niGBToGkKqHyKzhT3E7sCl0FNkPkHaaGZwPwhFCDyNFxBd4uGgWiQ+kT+RjtH5ahna+bSP69Yg==
+"@snyk/snyk-cocoapods-plugin@2.5.0":
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/@snyk/snyk-cocoapods-plugin/-/snyk-cocoapods-plugin-2.5.0.tgz#8dedb9d3165e5c8cd72ab13fdb49397083a15866"
+  integrity sha512-arK4VHzNh/D9vCFQFeAiSP+rMRXwLbzaRoIKucodf8Q/3KftIo/byeDmoc2Cc7awR1HPo5E391bwBNH5ra8UqA==
   dependencies:
-    "@snyk/cli-interface" "1.5.0"
-    "@snyk/cocoapods-lockfile-parser" "3.2.0"
-    "@snyk/dep-graph" "^1.18.2"
+    "@snyk/cli-interface" "2.6.1"
+    "@snyk/cocoapods-lockfile-parser" "3.5.1"
+    "@snyk/dep-graph" "^1.19.0"
     source-map-support "^0.5.7"
-    tslib "^1.10.0"
+    tslib "^2.0.0"
 
-"@snyk/update-notifier@^2.5.1-rc2":
-  version "2.5.1-rc2"
-  resolved "https://registry.yarnpkg.com/@snyk/update-notifier/-/update-notifier-2.5.1-rc2.tgz#14bf816114b5698a255289d7170157f254202fad"
-  integrity sha512-dlled3mfpnAt3cQb5hxkFiqfPCj4Yk0xV8Yl5P8PeVv1pUmO7vI4Ka4Mjs4r6CYM5f9kZhviFPQQcWOIDlMRcw==
+"@snyk/snyk-docker-pull@^3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@snyk/snyk-docker-pull/-/snyk-docker-pull-3.2.0.tgz#07c47b8be2d899d51d720099a73a0d89effe5d99"
+  integrity sha512-uWKtjh29I/d0mfmfBN7w6RwwNBQxQVKrauF5ND/gqb0PVsKV22GIpkI+viWjI7KNKso6/B0tMmsv7TX2tsNcLQ==
   dependencies:
-    "@snyk/configstore" "3.2.0-rc1"
-    boxen "^1.3.0"
-    chalk "^2.3.2"
-    import-lazy "^2.1.0"
-    is-ci "^1.0.10"
-    is-installed-globally "^0.1.0"
-    is-npm "^1.0.0"
-    latest-version "^3.1.0"
-    semver-diff "^2.0.0"
-    xdg-basedir "^3.0.0"
+    "@snyk/docker-registry-v2-client" "^1.13.5"
+    child-process "^1.0.2"
+    tar-stream "^2.1.2"
+    tmp "^0.1.0"
 
-"@types/agent-base@^4.2.0":
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/@types/agent-base/-/agent-base-4.2.0.tgz#00644e8b395b40e1bf50aaf1d22cabc1200d5051"
-  integrity sha512-8mrhPstU+ZX0Ugya8tl5DsDZ1I5ZwQzbL/8PA0z8Gj0k9nql7nkaMzmPVLj+l/nixWaliXi+EBiLA8bptw3z7Q==
+"@szmarczak/http-timer@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421"
+  integrity sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==
   dependencies:
-    "@types/events" "*"
+    defer-to-connect "^1.0.1"
+
+"@szmarczak/http-timer@^4.0.5":
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.5.tgz#bfbd50211e9dfa51ba07da58a14cdfd333205152"
+  integrity sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==
+  dependencies:
+    defer-to-connect "^2.0.0"
+
+"@types/cacheable-request@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.1.tgz#5d22f3dded1fd3a84c0bbeb5039a7419c2c91976"
+  integrity sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==
+  dependencies:
+    "@types/http-cache-semantics" "*"
+    "@types/keyv" "*"
     "@types/node" "*"
+    "@types/responselike" "*"
+
+"@types/color-name@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
+  integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==
 
 "@types/debug@^4.1.4":
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd"
   integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==
 
+"@types/emscripten@^1.38.0":
+  version "1.39.4"
+  resolved "https://registry.yarnpkg.com/@types/emscripten/-/emscripten-1.39.4.tgz#d61990c0cee72c4e475de737a140b51fe925a2c8"
+  integrity sha512-k3LLVMFrdNA9UCvMDPWMbFrGPNb+GcPyw29ktJTo1RCN7RmxFG5XzPZcPKRlnLuLT/FRm8wp4ohvDwNY7GlROQ==
+
 "@types/estree@0.0.39":
   version "0.0.39"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
   integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==
 
-"@types/events@*":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
-  integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
+"@types/glob@^7.1.1":
+  version "7.1.3"
+  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.3.tgz#e6ba80f36b7daad2c685acd9266382e68985c183"
+  integrity sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==
+  dependencies:
+    "@types/minimatch" "*"
+    "@types/node" "*"
 
 "@types/hosted-git-info@^2.7.0":
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/@types/hosted-git-info/-/hosted-git-info-2.7.0.tgz#5199b4df62d53f51d60520d61513eaa4b2dd3d3d"
   integrity sha512-OW/D8GqCyQtH8F7xDdDxzPJTBgknZeZhlCakUcBCya2rYPRN53F+0YJVwSPyiyAhrknnjkl3P9qVk0oBI4S1qw==
 
+"@types/http-cache-semantics@*":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz#9140779736aa2655635ee756e2467d787cfe8a2a"
+  integrity sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A==
+
 "@types/js-yaml@^3.12.1":
   version "3.12.2"
   resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.12.2.tgz#a35a1809c33a68200fb6403d1ad708363c56470a"
   integrity sha512-0CFu/g4mDSNkodVwWijdlr8jH7RoplRWNgovjFLEZeT+QEbbZXjBmCe3HwaWheAlCbHwomTwzZoSedeOycABug==
 
+"@types/keyv@*":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.1.tgz#e45a45324fca9dab716ab1230ee249c9fb52cfa7"
+  integrity sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==
+  dependencies:
+    "@types/node" "*"
+
+"@types/minimatch@*":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
+  integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
+
 "@types/node@*":
   version "13.7.4"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-13.7.4.tgz#76c3cb3a12909510f52e5dc04a6298cdf9504ffd"
@@ -287,6 +380,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-11.10.4.tgz#3f5fc4f0f322805f009e00ab35a2ff3d6b778e42"
   integrity sha512-wa09itaLE8L705aXd8F80jnFpxz3Y1/KRHfKsYL2bPc0XF+wEWu8sR9n5bmeu8Ba1N9z2GRNzm/YdHcghLkLKg==
 
+"@types/node@^13.7.0":
+  version "13.13.15"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.15.tgz#fe1cc3aa465a3ea6858b793fd380b66c39919766"
+  integrity sha512-kwbcs0jySLxzLsa2nWUAGOd/s21WU1jebrEdtzhsj1D4Yps1EOuyI1Qcu+FD56dL7NRNIJtDDjcqIG22NwkgLw==
+
 "@types/node@^6.14.4":
   version "6.14.9"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-6.14.9.tgz#733583e21ef0eab85a9737dfafbaa66345a92ef0"
@@ -302,6 +400,13 @@
   resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.1.tgz#48fd98c1561fe718b61733daed46ff115b496e18"
   integrity sha512-eqz8c/0kwNi/OEHQfvIuJVLTst3in0e7uTKeuY+WL/zfKn0xVujOTp42bS/vUUokhK5P2BppLd9JXMOMHcgbjA==
 
+"@types/responselike@*", "@types/responselike@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29"
+  integrity sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/semver@^5.5.0":
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-5.5.0.tgz#146c2a29ee7d3bae4bf2fcb274636e264c813c45"
@@ -360,11 +465,98 @@
     postcss-modules-sync "^1.0.0"
     source-map "0.6.*"
 
-"@yarnpkg/lockfile@^1.0.2":
+"@yarnpkg/core@^2.0.0-rc.29":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/core/-/core-2.1.1.tgz#637786eb2acd09b48db7af59f2287934a0a5afd7"
+  integrity sha512-qeBxz8nHjKAbGTP2ZcXBnXGfM7+cN0A73mIai/24uru1ayvCIgfjWL1uIj/MM+m+K5lJX0Dcn94ZBHWits9JWQ==
+  dependencies:
+    "@arcanis/slice-ansi" "^1.0.2"
+    "@yarnpkg/fslib" "^2.1.0"
+    "@yarnpkg/json-proxy" "^2.1.0"
+    "@yarnpkg/libzip" "^2.1.0"
+    "@yarnpkg/parsers" "^2.1.0"
+    "@yarnpkg/pnp" "^2.1.0"
+    "@yarnpkg/shell" "^2.1.0"
+    camelcase "^5.3.1"
+    chalk "^3.0.0"
+    ci-info "^2.0.0"
+    clipanion "^2.4.2"
+    cross-spawn "7.0.3"
+    diff "^4.0.1"
+    globby "^10.0.1"
+    got "^11.1.3"
+    json-file-plus "^3.3.1"
+    logic-solver "^2.0.1"
+    micromatch "^4.0.2"
+    mkdirp "^0.5.1"
+    p-limit "^2.2.0"
+    pluralize "^7.0.0"
+    pretty-bytes "^5.1.0"
+    semver "^7.1.2"
+    stream-to-promise "^2.2.0"
+    tar "^4.4.6"
+    tslib "^1.13.0"
+    tunnel "^0.0.6"
+
+"@yarnpkg/fslib@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/fslib/-/fslib-2.1.0.tgz#c7039ec369a8fde06932cbddb3d3e473d789c458"
+  integrity sha512-E+f8w5yQZnTf1soyTWy7qdf+GmHsY+A0yEN4Di44/Txk6XRIMruyc1ShDi93mOI6ilnXxD87rNms18zJ8WnspA==
+  dependencies:
+    "@yarnpkg/libzip" "^2.1.0"
+    tslib "^1.13.0"
+
+"@yarnpkg/json-proxy@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/json-proxy/-/json-proxy-2.1.0.tgz#362a161678cd7dda74b47b4fc848a2f1730d16cd"
+  integrity sha512-rOgCg2DkyviLgr80mUMTt9vzdf5RGOujQB26yPiXjlz4WNePLBshKlTNG9rKSoKQSOYEQcw6cUmosfOKDatrCw==
+  dependencies:
+    "@yarnpkg/fslib" "^2.1.0"
+    tslib "^1.13.0"
+
+"@yarnpkg/libzip@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/libzip/-/libzip-2.1.0.tgz#f45d861a853555a063f90a85142199e7cf181b71"
+  integrity sha512-39c7KuSWcYUqVxlBLZwfqdD/D6lS+jplNVWd6uAnk8EpnacaYGJRegvkqWyfw5c8KHukNMeEGF5JHrXPZYBM0w==
+  dependencies:
+    "@types/emscripten" "^1.38.0"
+    tslib "^1.13.0"
+
+"@yarnpkg/lockfile@^1.0.2", "@yarnpkg/lockfile@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31"
   integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==
 
+"@yarnpkg/parsers@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/parsers/-/parsers-2.1.0.tgz#52615acdb54803d6404d6fb8f15fa124936bff1f"
+  integrity sha512-75OYQ6PMs1C3zm+W+T1xhLyVDX78zXQGEVHpWd4o/QwpAbhneB3/5FXVGRzI3gjPPWWSb/pKOPB1S6p0xmQD2Q==
+  dependencies:
+    js-yaml "^3.10.0"
+    tslib "^1.13.0"
+
+"@yarnpkg/pnp@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/pnp/-/pnp-2.1.0.tgz#0eba881ccd4675594198e7f1e327fe11a845291b"
+  integrity sha512-b8NlB71EFifv1jDX47nFaRXrykROxHcS7YuGb2dQ+Gp9gqJ0thIaZ3yB9+qWF8acdWtNcMpjCug4xkfAAR5Odw==
+  dependencies:
+    "@types/node" "^13.7.0"
+    "@yarnpkg/fslib" "^2.1.0"
+    tslib "^1.13.0"
+
+"@yarnpkg/shell@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/shell/-/shell-2.1.0.tgz#e0855e59d4df663d8fe9044950a7fd390110cd4b"
+  integrity sha512-9i9ZWqeKHGV0DOfdxTVq5zl73Li8Fg947v57uLBEaytNF+HywkDfouNkg/6HfgBrpI0WH8OJ9Pz/uDaE5cpctw==
+  dependencies:
+    "@yarnpkg/fslib" "^2.1.0"
+    "@yarnpkg/parsers" "^2.1.0"
+    clipanion "^2.4.2"
+    cross-spawn "7.0.3"
+    fast-glob "^3.2.2"
+    stream-buffers "^3.0.2"
+    tslib "^1.13.0"
+
 abbrev@1, abbrev@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@@ -438,12 +630,12 @@ amdefine@>=0.0.4:
   resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
   integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=
 
-ansi-align@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-2.0.0.tgz#c36aeccba563b89ceb556f3690f0b1d9e3547f7f"
-  integrity sha1-w2rsy6VjuJzrVW82kPCx2eNUf38=
+ansi-align@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.0.tgz#b536b371cf687caaef236c18d3e21fe3797467cb"
+  integrity sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==
   dependencies:
-    string-width "^2.0.0"
+    string-width "^3.0.0"
 
 ansi-escapes@3.2.0, ansi-escapes@^3.2.0:
   version "3.2.0"
@@ -470,6 +662,11 @@ ansi-regex@^4.1.0:
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997"
   integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
 
+ansi-regex@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
+  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+
 ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@@ -482,11 +679,24 @@ ansi-styles@^3.2.0, ansi-styles@^3.2.1:
   dependencies:
     color-convert "^1.9.0"
 
+ansi-styles@^4.1.0:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359"
+  integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==
+  dependencies:
+    "@types/color-name" "^1.1.1"
+    color-convert "^2.0.1"
+
 ansicolors@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/ansicolors/-/ansicolors-0.3.2.tgz#665597de86a9ffe3aa9bfbe6cae5c6ea426b4979"
   integrity sha1-ZlWX3oap/+Oqm/vmyuXG6kJrSXk=
 
+any-promise@^1.1.0, any-promise@~1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
+  integrity sha1-q8av7tzqUugJzcA3au0845Y10X8=
+
 aproba@^1.0.3:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
@@ -547,6 +757,11 @@ array-flatten@1.1.1:
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
   integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=
 
+array-union@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d"
+  integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==
+
 array-unique@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
@@ -562,7 +777,7 @@ asap@~2.0.3:
   resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
   integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=
 
-asn1@~0.2.3:
+asn1@~0.2.0, asn1@~0.2.3:
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
   integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
@@ -666,6 +881,11 @@ base64-arraybuffer@0.1.5:
   resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz#73926771923b5a19747ad666aa5cd4bf9c6e9ce8"
   integrity sha1-c5JncZI7Whl0etZmqlzUv5xunOg=
 
+base64-js@^1.0.2:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1"
+  integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==
+
 base64id@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6"
@@ -684,7 +904,7 @@ base@^0.11.1:
     mixin-deep "^1.2.0"
     pascalcase "^0.1.1"
 
-bcrypt-pbkdf@^1.0.0:
+bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
   integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=
@@ -710,6 +930,15 @@ bl@^3.0.0:
   dependencies:
     readable-stream "^3.0.1"
 
+bl@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-4.0.2.tgz#52b71e9088515d0606d9dd9cc7aa48dc1f98e73a"
+  integrity sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==
+  dependencies:
+    buffer "^5.5.0"
+    inherits "^2.0.4"
+    readable-stream "^3.4.0"
+
 blob@0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/blob/-/blob-0.0.5.tgz#d680eeef25f8cd91ad533f5b01eed48e64caf683"
@@ -758,18 +987,19 @@ bootstrap@^4.4.1:
   resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-4.4.1.tgz#8582960eea0c5cd2bede84d8b0baf3789c3e8b01"
   integrity sha512-tbx5cHubwE6e2ZG7nqM3g/FZ5PQEDMWmMGNrCUBVRPHXTJaH7CBDdsLeu3eCh3B1tzAxTnAbtmrzvWEvT2NNEA==
 
-boxen@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"
-  integrity sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==
+boxen@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64"
+  integrity sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==
   dependencies:
-    ansi-align "^2.0.0"
-    camelcase "^4.0.0"
-    chalk "^2.0.1"
-    cli-boxes "^1.0.0"
-    string-width "^2.0.0"
-    term-size "^1.2.0"
-    widest-line "^2.0.0"
+    ansi-align "^3.0.0"
+    camelcase "^5.3.1"
+    chalk "^3.0.0"
+    cli-boxes "^2.2.0"
+    string-width "^4.1.0"
+    term-size "^2.1.0"
+    type-fest "^0.8.1"
+    widest-line "^3.1.0"
 
 brace-expansion@^1.1.7:
   version "1.1.11"
@@ -795,6 +1025,20 @@ braces@^2.3.1:
     split-string "^3.0.2"
     to-regex "^3.0.1"
 
+braces@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+  dependencies:
+    fill-range "^7.0.1"
+
+browserify-zlib@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/browserify-zlib/-/browserify-zlib-0.1.4.tgz#bb35f8a519f600e0fa6b8485241c979d0141fb2d"
+  integrity sha1-uzX4pRn2AOD6a4SFJByXnQFB+y0=
+  dependencies:
+    pako "~0.2.0"
+
 browserslist@^4.0.0:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.4.2.tgz#6ea8a74d6464bb0bd549105f659b41197d8f0ba2"
@@ -826,6 +1070,14 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer@^5.5.0:
+  version "5.6.0"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
+  integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+
 builtin-modules@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.0.0.tgz#1e587d44b006620d90286cc7a9238bbc6129cab1"
@@ -851,6 +1103,37 @@ cache-base@^1.0.1:
     union-value "^1.0.0"
     unset-value "^1.0.0"
 
+cacheable-lookup@^5.0.3:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3"
+  integrity sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w==
+
+cacheable-request@^6.0.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912"
+  integrity sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==
+  dependencies:
+    clone-response "^1.0.2"
+    get-stream "^5.1.0"
+    http-cache-semantics "^4.0.0"
+    keyv "^3.0.0"
+    lowercase-keys "^2.0.0"
+    normalize-url "^4.1.0"
+    responselike "^1.0.2"
+
+cacheable-request@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.1.tgz#062031c2856232782ed694a257fa35da93942a58"
+  integrity sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==
+  dependencies:
+    clone-response "^1.0.2"
+    get-stream "^5.1.0"
+    http-cache-semantics "^4.0.0"
+    keyv "^4.0.0"
+    lowercase-keys "^2.0.0"
+    normalize-url "^4.1.0"
+    responselike "^2.0.0"
+
 cachedir@1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-1.3.0.tgz#5e01928bf2d95b5edd94b0942188246740e0dbc4"
@@ -900,12 +1183,7 @@ camelcase@^3.0.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
   integrity sha1-MvxLn82vhF/N9+c7uXysImHwqwo=
 
-camelcase@^4.0.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
-  integrity sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=
-
-camelcase@^5.0.0:
+camelcase@^5.0.0, camelcase@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
@@ -925,11 +1203,6 @@ caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939:
   resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001116.tgz"
   integrity sha512-f2lcYnmAI5Mst9+g0nkMIznFGsArRmZ0qU+dnq8l91hymdc2J3SFbiPhOJEeDqC1vtE8nc1qNQyklzB8veJefQ==
 
-capture-stack-trace@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz#a6c0bbe1f38f3aa0b92238ecb6ff42c344d4135d"
-  integrity sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==
-
 caseless@~0.12.0:
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -955,6 +1228,14 @@ chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3:
     strip-ansi "^3.0.0"
     supports-color "^2.0.0"
 
+chalk@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4"
+  integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==
+  dependencies:
+    ansi-styles "^4.1.0"
+    supports-color "^7.1.0"
+
 chardet@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
@@ -965,6 +1246,16 @@ check-more-types@2.24.0:
   resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
   integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA=
 
+child-process@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/child-process/-/child-process-1.0.2.tgz#98974dc7ed1ee4c6229f8e305fa7313a6885a7f2"
+  integrity sha1-mJdNx+0e5MYin44wX6cxOmiFp/I=
+
+chownr@^1.1.1:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
+  integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
+
 ci-info@^1.5.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.6.0.tgz#2ca20dbb9ceb32d4524a683303313f0304b1e497"
@@ -992,10 +1283,10 @@ clean-css@^4.1.11:
   dependencies:
     source-map "~0.6.0"
 
-cli-boxes@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-1.0.0.tgz#4fa917c3e59c94a004cd61f8ee509da651687143"
-  integrity sha1-T6kXw+WclKAEzWH47lCdplFocUM=
+cli-boxes@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.0.tgz#538ecae8f9c6ca508e3c3c95b453fe93cb4c168d"
+  integrity sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==
 
 cli-cursor@^1.0.2:
   version "1.0.2"
@@ -1034,6 +1325,11 @@ cli-width@^2.0.0:
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639"
   integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=
 
+clipanion@^2.4.2:
+  version "2.4.4"
+  resolved "https://registry.yarnpkg.com/clipanion/-/clipanion-2.4.4.tgz#d70244c6f60feb3f4cbd509d2fcbe829fc619061"
+  integrity sha512-KjyCBz8xplftHjIK/nOqq/9b3hPlXbAAo/AxoITrO4yySpQ6a9QSJDAfOx9PfcRUHteeqbdNxZKSPfeFqQ7plg==
+
 cliui@^3.0.3, cliui@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d"
@@ -1052,6 +1348,13 @@ cliui@^5.0.0:
     strip-ansi "^5.2.0"
     wrap-ansi "^5.1.0"
 
+clone-response@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b"
+  integrity sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=
+  dependencies:
+    mimic-response "^1.0.0"
+
 clone@^2.1.1, clone@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
@@ -1091,12 +1394,19 @@ color-convert@^1.9.0, color-convert@^1.9.1:
   dependencies:
     color-name "1.1.3"
 
+color-convert@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
+  integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
+  dependencies:
+    color-name "~1.1.4"
+
 color-name@1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
   integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=
 
-color-name@^1.0.0:
+color-name@^1.0.0, color-name@~1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
@@ -1176,6 +1486,18 @@ concat-with-sourcemaps@^1.0.5:
   dependencies:
     source-map "^0.6.1"
 
+configstore@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96"
+  integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==
+  dependencies:
+    dot-prop "^5.2.0"
+    graceful-fs "^4.1.2"
+    make-dir "^3.0.0"
+    unique-string "^2.0.0"
+    write-file-atomic "^3.0.0"
+    xdg-basedir "^4.0.0"
+
 console-control-strings@^1.0.0, console-control-strings@~1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
@@ -1230,16 +1552,6 @@ core-js@^2.4.0:
   resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.11.tgz#38831469f9922bded8ee21c9dc46985e0399308c"
   integrity sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==
 
-core-js@^3.2.0:
-  version "3.6.4"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.6.4.tgz#440a83536b458114b9cb2ac1580ba377dc470647"
-  integrity sha512-4paDGScNgZP2IXXilaffL9X7968RuvwlkK3xWtZRVqgd8SYNiVKRJvkFd1aqqEuPfN7E68ZHEp9hDj6lHj4Hyw==
-
-core-js@~2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.3.0.tgz#fab83fbb0b2d8dc85fa636c4b9d34c75420c6d65"
-  integrity sha1-+rg/uwstjchfpjbEudNMdUIMbWU=
-
 core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
@@ -1266,12 +1578,14 @@ cosmiconfig@^5.0.0:
     lodash.get "^4.4.2"
     parse-json "^4.0.0"
 
-create-error-class@^3.0.0:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6"
-  integrity sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=
+cross-spawn@7.0.3:
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
+  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
   dependencies:
-    capture-stack-trace "^1.0.0"
+    path-key "^3.1.0"
+    shebang-command "^2.0.0"
+    which "^2.0.1"
 
 cross-spawn@^3.0.0:
   version "3.0.1"
@@ -1281,15 +1595,6 @@ cross-spawn@^3.0.0:
     lru-cache "^4.0.1"
     which "^1.2.9"
 
-cross-spawn@^5.0.1:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-5.1.0.tgz#e8bd0efee58fcff6f8f94510a0a554bbfa235449"
-  integrity sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=
-  dependencies:
-    lru-cache "^4.0.1"
-    shebang-command "^1.2.0"
-    which "^1.2.9"
-
 cross-spawn@^6.0.0:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -1301,10 +1606,10 @@ cross-spawn@^6.0.0:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
-crypto-random-string@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e"
-  integrity sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=
+crypto-random-string@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
+  integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
 css-color-names@0.0.4, css-color-names@^0.0.4:
   version "0.0.4"
@@ -1594,6 +1899,20 @@ decode-uri-component@^0.2.0:
   resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
   integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=
 
+decompress-response@^3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3"
+  integrity sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=
+  dependencies:
+    mimic-response "^1.0.0"
+
+decompress-response@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc"
+  integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==
+  dependencies:
+    mimic-response "^3.1.0"
+
 deep-equal@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5"
@@ -1629,6 +1948,16 @@ deep-is@~0.1.3:
   resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
   integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
 
+defer-to-connect@^1.0.1:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591"
+  integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==
+
+defer-to-connect@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.0.tgz#83d6b199db041593ac84d781b5222308ccf4c2c1"
+  integrity sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg==
+
 define-properties@^1.1.2, define-properties@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1"
@@ -1692,6 +2021,23 @@ diff@^4.0.1:
   resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
   integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
 
+dir-glob@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
+  integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==
+  dependencies:
+    path-type "^4.0.0"
+
+docker-modem@2.1.3:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-2.1.3.tgz#15432225f63db02eb5de4bb9a621b7293e5f264d"
+  integrity sha512-cwaRptBmYZwu/FyhGcqBm2MzXA77W2/E6eVkpOZVDk6PkI9Bjj84xPrXiHMA+OWjzNy+DFjgKh8Q+1hMR7/OHg==
+  dependencies:
+    debug "^4.1.1"
+    readable-stream "^3.5.0"
+    split-ca "^1.0.1"
+    ssh2 "^0.8.7"
+
 dockerfile-ast@0.0.19:
   version "0.0.19"
   resolved "https://registry.yarnpkg.com/dockerfile-ast/-/dockerfile-ast-0.0.19.tgz#b1e21138eba995d7bf5576dc30ba1130c15995c3"
@@ -1760,6 +2106,16 @@ duplexer3@^0.1.4:
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
   integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=
 
+duplexify@^3.5.0, duplexify@^3.6.0:
+  version "3.7.1"
+  resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309"
+  integrity sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==
+  dependencies:
+    end-of-stream "^1.0.0"
+    inherits "^2.0.1"
+    readable-stream "^2.0.0"
+    stream-shift "^1.0.0"
+
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -1793,6 +2149,11 @@ emoji-regex@^7.0.1:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156"
   integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==
 
+emoji-regex@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
+  integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
+
 emojis-list@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389"
@@ -1803,13 +2164,20 @@ encodeurl@~1.0.2:
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
   integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
 
-end-of-stream@^1.1.0, end-of-stream@^1.4.1:
+end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
   integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
   dependencies:
     once "^1.4.0"
 
+end-of-stream@~1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.1.0.tgz#e9353258baa9108965efc41cb0ef8ade2f3cfb07"
+  integrity sha1-6TUyWLqpEIll78QcsO+K3i88+wc=
+  dependencies:
+    once "~1.3.0"
+
 engine.io-client@~3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-3.4.0.tgz#82a642b42862a9b3f7a188f41776b2deab643700"
@@ -1934,11 +2302,6 @@ es6-promise@^4.0.3:
   resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a"
   integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==
 
-es6-promise@~3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.0.2.tgz#010d5858423a5f118979665f46486a95c6ee2bb6"
-  integrity sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=
-
 es6-promisify@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203"
@@ -1946,6 +2309,11 @@ es6-promisify@^5.0.0:
   dependencies:
     es6-promise "^4.0.3"
 
+escape-goat@^2.0.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675"
+  integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==
+
 escape-html@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
@@ -2003,13 +2371,20 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
-event-loop-spinner@1.1.0, event-loop-spinner@^1.1.0:
+event-loop-spinner@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/event-loop-spinner/-/event-loop-spinner-1.1.0.tgz#96de9c70e6e2b0b3e257b0901e25e792e3c9c8d0"
   integrity sha512-YVFs6dPpZIgH665kKckDktEVvSBccSYJmoZUfhNUdv5d3Xv+Q+SKF4Xis1jolq9aBzuW1ZZhQh/m/zU/TPdDhw==
   dependencies:
     tslib "^1.10.0"
 
+event-loop-spinner@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/event-loop-spinner/-/event-loop-spinner-2.0.0.tgz#3aa949264a7d80be45d6cb782ab5e97ffc87b9e1"
+  integrity sha512-1y4j/Mhttr8ordvHkbDsGzGrlQaSYJoXD/3YKUxiOXIk7myEn9UPfybEk/lLtrcU3D4QvCNmVUxVQaPtvAIaUw==
+  dependencies:
+    tslib "^1.10.0"
+
 eventemitter2@4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-4.1.2.tgz#0e1a8477af821a6ef3995b311bf74c23a5247f15"
@@ -2038,19 +2413,6 @@ execa@0.10.0:
     signal-exit "^3.0.0"
     strip-eof "^1.0.0"
 
-execa@^0.7.0:
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777"
-  integrity sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=
-  dependencies:
-    cross-spawn "^5.0.1"
-    get-stream "^3.0.0"
-    is-stream "^1.1.0"
-    npm-run-path "^2.0.0"
-    p-finally "^1.0.0"
-    signal-exit "^3.0.0"
-    strip-eof "^1.0.0"
-
 execa@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
@@ -2215,6 +2577,18 @@ fast-diff@1.2.0:
   resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03"
   integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==
 
+fast-glob@^3.0.3, fast-glob@^3.2.2:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3"
+  integrity sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.0"
+    merge2 "^1.3.0"
+    micromatch "^4.0.2"
+    picomatch "^2.2.1"
+
 fast-json-stable-stringify@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
@@ -2230,6 +2604,13 @@ fastparse@^1.1.1:
   resolved "https://registry.yarnpkg.com/fastparse/-/fastparse-1.1.2.tgz#91728c5a5942eced8531283c79441ee4122c35a9"
   integrity sha512-483XLLxTVIwWK3QTrMGRqUfUpoOs/0hbQrl2oz4J0pAcm3A3bu84wxTFqGqkJzewCLdME38xJLJAxBABfQT8sQ==
 
+fastq@^1.6.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.8.0.tgz#550e1f9f59bbc65fe185cb6a9b4d95357107f481"
+  integrity sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==
+  dependencies:
+    reusify "^1.0.4"
+
 fd-slicer@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65"
@@ -2274,6 +2655,13 @@ fill-range@^4.0.0:
     repeat-string "^1.6.1"
     to-regex-range "^2.1.0"
 
+fill-range@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
+  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+  dependencies:
+    to-regex-range "^5.0.1"
+
 finalhandler@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d"
@@ -2381,6 +2769,13 @@ fs-extra@5.0.0:
     jsonfile "^4.0.0"
     universalify "^0.1.0"
 
+fs-minipass@^1.2.5:
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7"
+  integrity sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==
+  dependencies:
+    minipass "^2.6.0"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -2479,13 +2874,20 @@ get-stream@^3.0.0:
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14"
   integrity sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=
 
-get-stream@^4.0.0:
+get-stream@^4.0.0, get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
   integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==
   dependencies:
     pump "^3.0.0"
 
+get-stream@^5.1.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3"
+  integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==
+  dependencies:
+    pump "^3.0.0"
+
 get-uri@^2.0.0:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-2.0.4.tgz#d4937ab819e218d4cb5ae18e4f5962bef169cc6a"
@@ -2517,21 +2919,6 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-git-up@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/git-up/-/git-up-4.0.1.tgz#cb2ef086653640e721d2042fe3104857d89007c0"
-  integrity sha512-LFTZZrBlrCrGCG07/dm1aCjjpL1z9L3+5aEeI9SBhAqSc+kiA9Or1bgZhQFNppJX6h/f5McrvJt1mQXTFm6Qrw==
-  dependencies:
-    is-ssh "^1.3.0"
-    parse-url "^5.0.0"
-
-git-url-parse@11.1.2:
-  version "11.1.2"
-  resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-11.1.2.tgz#aff1a897c36cc93699270587bea3dbcbbb95de67"
-  integrity sha512-gZeLVGY8QVKMIkckncX+iCq2/L8PlwncvDFKiWkBn9EtCfYDbliRTTp6qzyQ1VMdITUfq7293zDzfpjdiGASSQ==
-  dependencies:
-    git-up "^4.0.0"
-
 gl-mat2@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/gl-mat2/-/gl-mat2-1.0.1.tgz#142505730a5c2fe1e9f25d9ece3d0d6cc2710a30"
@@ -2547,6 +2934,13 @@ gl-vec3@^1.0.3:
   resolved "https://registry.yarnpkg.com/gl-vec3/-/gl-vec3-1.1.3.tgz#a47c62f918774a06cbed1b65bcd0288ecbb03826"
   integrity sha512-jduKUqT0SGH02l8Yl+mV1yVsDfYgQAJyXGxkJQGyxPLHRiW25DwVIRPt6uvhrEMHftJfqhqKthRcyZqNEl9Xdw==
 
+glob-parent@^5.1.0:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229"
+  integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==
+  dependencies:
+    is-glob "^4.0.1"
+
 glob@^7.0.0, glob@^7.0.3, glob@^7.1.2, glob@^7.1.3, glob@^7.1.6, glob@~7.1.1:
   version "7.1.6"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
@@ -2566,6 +2960,13 @@ global-dirs@^0.1.0:
   dependencies:
     ini "^1.3.4"
 
+global-dirs@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.0.1.tgz#acdf3bb6685bcd55cb35e8a052266569e9469201"
+  integrity sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==
+  dependencies:
+    ini "^1.3.5"
+
 global-modules@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea"
@@ -2586,6 +2987,20 @@ global-prefix@^1.0.1:
     is-windows "^1.0.1"
     which "^1.2.14"
 
+globby@^10.0.1:
+  version "10.0.2"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-10.0.2.tgz#277593e745acaa4646c3ab411289ec47a0392543"
+  integrity sha512-7dUi7RvCoT/xast/o/dLN53oqND4yk0nsHkhRgn9w65C4PofCLOoJ39iSOg+qVDdWQPIEj+eszMHQ+aLVwwQSg==
+  dependencies:
+    "@types/glob" "^7.1.1"
+    array-union "^2.1.0"
+    dir-glob "^3.0.1"
+    fast-glob "^3.0.3"
+    glob "^7.1.3"
+    ignore "^5.1.1"
+    merge2 "^1.2.3"
+    slash "^3.0.0"
+
 globule@^1.0.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/globule/-/globule-1.2.1.tgz#5dffb1b191f22d20797a9369b49eab4e9839696d"
@@ -2595,28 +3010,67 @@ globule@^1.0.0:
     lodash "~4.17.10"
     minimatch "~3.0.2"
 
-got@^6.7.1:
-  version "6.7.1"
-  resolved "https://registry.yarnpkg.com/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0"
-  integrity sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=
+got@11.4.0:
+  version "11.4.0"
+  resolved "https://registry.yarnpkg.com/got/-/got-11.4.0.tgz#1f0910310572af4efcc6890e1dacd7affb710b39"
+  integrity sha512-XysJZuZNVpaQ37Oo2LV90MIkPeYITehyy1A0QzO1JwOXm8EWuEf9eeGk2XuHePvLEGnm9AVOI37bHwD6KYyBtg==
   dependencies:
-    create-error-class "^3.0.0"
-    duplexer3 "^0.1.4"
-    get-stream "^3.0.0"
-    is-redirect "^1.0.0"
-    is-retry-allowed "^1.0.0"
-    is-stream "^1.0.0"
-    lowercase-keys "^1.0.0"
-    safe-buffer "^5.0.1"
-    timed-out "^4.0.0"
-    unzip-response "^2.0.1"
-    url-parse-lax "^1.0.0"
+    "@sindresorhus/is" "^2.1.1"
+    "@szmarczak/http-timer" "^4.0.5"
+    "@types/cacheable-request" "^6.0.1"
+    "@types/responselike" "^1.0.0"
+    cacheable-lookup "^5.0.3"
+    cacheable-request "^7.0.1"
+    decompress-response "^6.0.0"
+    http2-wrapper "^1.0.0-beta.4.5"
+    lowercase-keys "^2.0.0"
+    p-cancelable "^2.0.0"
+    responselike "^2.0.0"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6:
+got@^11.1.3:
+  version "11.5.2"
+  resolved "https://registry.yarnpkg.com/got/-/got-11.5.2.tgz#772e3f3a06d9c7589c7c94dc3c83cdb31ddbf742"
+  integrity sha512-yUhpEDLeuGiGJjRSzEq3kvt4zJtAcjKmhIiwNp/eUs75tRlXfWcHo5tcBaMQtnjHWC7nQYT5HkY/l0QOQTkVww==
+  dependencies:
+    "@sindresorhus/is" "^3.0.0"
+    "@szmarczak/http-timer" "^4.0.5"
+    "@types/cacheable-request" "^6.0.1"
+    "@types/responselike" "^1.0.0"
+    cacheable-lookup "^5.0.3"
+    cacheable-request "^7.0.1"
+    decompress-response "^6.0.0"
+    http2-wrapper "^1.0.0-beta.5.0"
+    lowercase-keys "^2.0.0"
+    p-cancelable "^2.0.0"
+    responselike "^2.0.0"
+
+got@^9.6.0:
+  version "9.6.0"
+  resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85"
+  integrity sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==
+  dependencies:
+    "@sindresorhus/is" "^0.14.0"
+    "@szmarczak/http-timer" "^1.1.2"
+    cacheable-request "^6.0.0"
+    decompress-response "^3.3.0"
+    duplexer3 "^0.1.4"
+    get-stream "^4.1.0"
+    lowercase-keys "^1.0.1"
+    mimic-response "^1.0.1"
+    p-cancelable "^1.0.0"
+    to-readable-stream "^1.0.0"
+    url-parse-lax "^3.0.0"
+
+graceful-fs@^4.1.2, graceful-fs@^4.1.6:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423"
   integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==
 
+grapheme-splitter@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e"
+  integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==
+
 graphlib@^2.1.8:
   version "2.1.8"
   resolved "https://registry.yarnpkg.com/graphlib/-/graphlib-2.1.8.tgz#5761d414737870084c92ec7b5dbcb0592c9d35da"
@@ -2624,6 +3078,18 @@ graphlib@^2.1.8:
   dependencies:
     lodash "^4.17.15"
 
+gunzip-maybe@^1.4.2:
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz#b913564ae3be0eda6f3de36464837a9cd94b98ac"
+  integrity sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==
+  dependencies:
+    browserify-zlib "^0.1.4"
+    is-deflate "^1.0.0"
+    is-gzip "^1.0.0"
+    peek-stream "^1.1.0"
+    pumpify "^1.3.3"
+    through2 "^2.0.3"
+
 har-schema@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -2666,6 +3132,11 @@ has-flag@^3.0.0:
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
   integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0=
 
+has-flag@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
+  integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
+
 has-glob@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/has-glob/-/has-glob-1.0.0.tgz#9aaa9eedbffb1ba3990a7b0010fb678ee0081207"
@@ -2719,6 +3190,11 @@ has-values@^1.0.0:
     is-number "^3.0.0"
     kind-of "^4.0.0"
 
+has-yarn@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77"
+  integrity sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==
+
 has@^1.0.0, has@^1.0.1, has@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
@@ -2763,6 +3239,13 @@ hosted-git-info@^2.7.1:
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.5.tgz#759cfcf2c4d156ade59b0b2dfabddc42a6b9c70c"
   integrity sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==
 
+hosted-git-info@^3.0.4:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-3.0.5.tgz#bea87905ef7317442e8df3087faa3c842397df03"
+  integrity sha512-i4dpK6xj9BIpVOTboXIlKG9+8HMKggcrMX7WA24xZtKwX0TPelq/rbaS5rCKeNX8sJXZJGdSxpnEGtta+wismQ==
+  dependencies:
+    lru-cache "^6.0.0"
+
 hsl-regex@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e"
@@ -2778,6 +3261,11 @@ html-comment-regex@^1.1.0:
   resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.2.tgz#97d4688aeb5c81886a364faa0cad1dda14d433a7"
   integrity sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ==
 
+http-cache-semantics@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
+  integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==
+
 http-errors@1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f"
@@ -2817,6 +3305,14 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
+http2-wrapper@^1.0.0-beta.4.5, http2-wrapper@^1.0.0-beta.5.0:
+  version "1.0.0-beta.5.2"
+  resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.0-beta.5.2.tgz#8b923deb90144aea65cf834b016a340fc98556f3"
+  integrity sha512-xYz9goEyBnC8XwXDTuC/MZ6t+MrKVQZOk4s7+PaDkwIsQd8IwqvM+0M6bA/2lvG8GHXcPdf+MejTUeO2LCPCeQ==
+  dependencies:
+    quick-lru "^5.1.1"
+    resolve-alpn "^1.0.0"
+
 https-proxy-agent@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz#b8c286433e87602311b01c8ea34413d856a4af81"
@@ -2842,6 +3338,16 @@ icss-replace-symbols@1.1.0, icss-replace-symbols@^1.0.2, icss-replace-symbols@^1
   resolved "https://registry.yarnpkg.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz#06ea6f83679a7749e386cfe1fe812ae5db223ded"
   integrity sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=
 
+ieee754@^1.1.4:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
+  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
+
+ignore@^5.1.1:
+  version "5.1.8"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
+  integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
+
 image-size@~0.5.0:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"
@@ -2919,7 +3425,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -2929,7 +3435,7 @@ inherits@2.0.3:
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
   integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
-ini@^1.3.0, ini@^1.3.4, ini@~1.3.0:
+ini@^1.3.0, ini@^1.3.4, ini@^1.3.5, ini@~1.3.0:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"
   integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==
@@ -3013,13 +3519,20 @@ is-callable@^1.1.5:
   resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.0.tgz#83336560b54a38e35e3a2df7afd0454d691468bb"
   integrity sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==
 
-is-ci@1.2.1, is-ci@^1.0.10:
+is-ci@1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-1.2.1.tgz#e3779c8ee17fccf428488f6e281187f2e632841c"
   integrity sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==
   dependencies:
     ci-info "^1.5.0"
 
+is-ci@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c"
+  integrity sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==
+  dependencies:
+    ci-info "^2.0.0"
+
 is-color-stop@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-color-stop/-/is-color-stop-1.1.0.tgz#cfff471aee4dd5c9e158598fbe12967b5cdad345"
@@ -3056,6 +3569,11 @@ is-date-object@^1.0.2:
   resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e"
   integrity sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==
 
+is-deflate@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-deflate/-/is-deflate-1.0.0.tgz#c862901c3c161fb09dac7cdc7e784f80e98f2f14"
+  integrity sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=
+
 is-descriptor@^0.1.0:
   version "0.1.6"
   resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca"
@@ -3096,7 +3614,7 @@ is-extendable@^1.0.1:
   dependencies:
     is-plain-object "^2.0.4"
 
-is-extglob@^2.1.0:
+is-extglob@^2.1.0, is-extglob@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
   integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
@@ -3118,6 +3636,11 @@ is-fullwidth-code-point@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f"
   integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=
 
+is-fullwidth-code-point@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
+  integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
+
 is-glob@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a"
@@ -3125,7 +3648,19 @@ is-glob@^3.0.0:
   dependencies:
     is-extglob "^2.1.0"
 
-is-installed-globally@0.1.0, is-installed-globally@^0.1.0:
+is-glob@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
+  integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
+  dependencies:
+    is-extglob "^2.1.1"
+
+is-gzip@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-gzip/-/is-gzip-1.0.0.tgz#6ca8b07b99c77998025900e555ced8ed80879a83"
+  integrity sha1-bKiwe5nHeZgCWQDlVc7Y7YCHmoM=
+
+is-installed-globally@0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.1.0.tgz#0dfd98f5a9111716dd535dda6492f67bf3d25a80"
   integrity sha1-Df2Y9akRFxbdU13aZJL2e/PSWoA=
@@ -3133,6 +3668,14 @@ is-installed-globally@0.1.0, is-installed-globally@^0.1.0:
     global-dirs "^0.1.0"
     is-path-inside "^1.0.0"
 
+is-installed-globally@^0.3.1:
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.3.2.tgz#fd3efa79ee670d1187233182d5b0a1dd00313141"
+  integrity sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==
+  dependencies:
+    global-dirs "^2.0.1"
+    is-path-inside "^3.0.1"
+
 is-map@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1"
@@ -3143,10 +3686,10 @@ is-module@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591"
   integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=
 
-is-npm@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"
-  integrity sha1-8vtjpl5JBbQGyGBydloaTceTufQ=
+is-npm@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d"
+  integrity sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==
 
 is-number-object@^1.0.3:
   version "1.0.4"
@@ -3160,6 +3703,11 @@ is-number@^3.0.0:
   dependencies:
     kind-of "^3.0.2"
 
+is-number@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
+  integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
+
 is-obj@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f"
@@ -3177,6 +3725,11 @@ is-path-inside@^1.0.0:
   dependencies:
     path-is-inside "^1.0.1"
 
+is-path-inside@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017"
+  integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==
+
 is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677"
@@ -3189,11 +3742,6 @@ is-promise@^2.1.0:
   resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa"
   integrity sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=
 
-is-redirect@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24"
-  integrity sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=
-
 is-regex@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491"
@@ -3213,24 +3761,12 @@ is-resolvable@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88"
   integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==
 
-is-retry-allowed@^1.0.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4"
-  integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==
-
 is-set@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.1.tgz#d1604afdab1724986d30091575f54945da7e5f43"
   integrity sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA==
 
-is-ssh@^1.3.0:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/is-ssh/-/is-ssh-1.3.1.tgz#f349a8cadd24e65298037a522cf7520f2e81a0f3"
-  integrity sha512-0eRIASHZt1E68/ixClI8bp2YK2wmBPVWEismTs6M+M099jKgrzl/3E976zIbImSIob48N2/XGe9y7ZiYdImSlg==
-  dependencies:
-    protocols "^1.1.0"
-
-is-stream@^1.0.0, is-stream@^1.1.0:
+is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
   integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ=
@@ -3264,7 +3800,7 @@ is-typed-array@^1.1.3:
     foreach "^2.0.5"
     has-symbols "^1.0.1"
 
-is-typedarray@~1.0.0:
+is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
   integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
@@ -3301,6 +3837,16 @@ is-wsl@^2.1.1:
   dependencies:
     is-docker "^2.0.0"
 
+is-yarn-global@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232"
+  integrity sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==
+
+is@^3.2.1:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79"
+  integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==
+
 isarray@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
@@ -3376,6 +3922,14 @@ js-tokens@^4.0.0:
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
   integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
 
+js-yaml@^3.10.0:
+  version "3.14.0"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482"
+  integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==
+  dependencies:
+    argparse "^1.0.7"
+    esprima "^4.0.0"
+
 js-yaml@^3.12.0, js-yaml@^3.9.0:
   version "3.12.2"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.2.tgz#ef1d067c5a9d9cb65bd72f285b5d8105c77f14fc"
@@ -3407,6 +3961,27 @@ jsesc@~0.5.0:
   resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d"
   integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=
 
+json-buffer@3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898"
+  integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=
+
+json-buffer@3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13"
+  integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==
+
+json-file-plus@^3.3.1:
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/json-file-plus/-/json-file-plus-3.3.1.tgz#f4363806b82819ff8803d83d539d6a9edd2a5258"
+  integrity sha512-wo0q1UuiV5NsDPQDup1Km8IwEeqe+olr8tkWxeJq9Bjtcp7DZ0l+yrg28fSC3DEtrE311mhTZ54QGS6oiqnZEA==
+  dependencies:
+    is "^3.2.1"
+    node.extend "^2.0.0"
+    object.assign "^4.1.0"
+    promiseback "^2.0.2"
+    safer-buffer "^2.0.2"
+
 json-parse-better-errors@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -3449,16 +4024,15 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
-jszip@3.1.5:
-  version "3.1.5"
-  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.1.5.tgz#e3c2a6c6d706ac6e603314036d43cd40beefdf37"
-  integrity sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==
+jszip@3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.3.0.tgz#29d72c21a54990fa885b11fc843db320640d5271"
+  integrity sha512-EJ9k766htB1ZWnsV5ZMDkKLgA+201r/ouFF8R2OigVjVdcm2rurcBrrdXaeqBJbqnUVMko512PYmlncBKE1Huw==
   dependencies:
-    core-js "~2.3.0"
-    es6-promise "~3.0.2"
-    lie "~3.1.0"
+    lie "~3.3.0"
     pako "~1.0.2"
-    readable-stream "~2.0.6"
+    readable-stream "~2.3.6"
+    set-immediate-shim "~1.0.1"
 
 jszip@^3.2.2:
   version "3.4.0"
@@ -3470,6 +4044,20 @@ jszip@^3.2.2:
     readable-stream "~2.3.6"
     set-immediate-shim "~1.0.1"
 
+keyv@^3.0.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9"
+  integrity sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==
+  dependencies:
+    json-buffer "3.0.0"
+
+keyv@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.0.1.tgz#9fe703cb4a94d6d11729d320af033307efd02ee6"
+  integrity sha512-xz6Jv6oNkbhrFCvCP7HQa8AaII8y8LRpoSm661NOKLr4uHuBwhX4epXrPQgF3+xdJnN4Esm5X0xwY4bOlALOtw==
+  dependencies:
+    json-buffer "3.0.1"
+
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
@@ -3494,12 +4082,12 @@ kind-of@^6.0.0, kind-of@^6.0.2:
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
   integrity sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==
 
-latest-version@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-3.1.0.tgz#a205383fea322b33b5ae3b18abee0dc2f356ee15"
-  integrity sha1-ogU4P+oyKzO1rjsYq+4NwvNW7hU=
+latest-version@^5.0.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face"
+  integrity sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==
   dependencies:
-    package-json "^4.0.0"
+    package-json "^6.3.0"
 
 lazy-ass@1.6.0:
   version "1.6.0"
@@ -3538,13 +4126,6 @@ levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
-lie@~3.1.0:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
-  integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
-  dependencies:
-    immediate "~3.0.5"
-
 lie@~3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
@@ -3657,16 +4238,81 @@ lodash.clonedeep@^4.3.0, lodash.clonedeep@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
   integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=
 
+lodash.constant@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/lodash.constant/-/lodash.constant-3.0.0.tgz#bfe05cce7e515b3128925d6362138420bd624910"
+  integrity sha1-v+Bczn5RWzEokl1jYhOEIL1iSRA=
+
+lodash.escaperegexp@^4.1.0:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
+  integrity sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=
+
+lodash.filter@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
+  integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=
+
+lodash.flatmap@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatmap/-/lodash.flatmap-4.5.0.tgz#ef8cbf408f6e48268663345305c6acc0b778702e"
+  integrity sha1-74y/QI9uSCaGYzRTBcaswLd4cC4=
+
 lodash.flatten@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
   integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
 
+lodash.foreach@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
+  integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=
+
 lodash.get@^4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
   integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=
 
+lodash.has@^4.5.2:
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862"
+  integrity sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=
+
+lodash.isarray@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-4.0.0.tgz#2aca496b28c4ca6d726715313590c02e6ea34403"
+  integrity sha1-KspJayjEym1yZxUxNZDALm6jRAM=
+
+lodash.isempty@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e"
+  integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4=
+
+lodash.isequal@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
+  integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA=
+
+lodash.isfunction@^3.0.9:
+  version "3.0.9"
+  resolved "https://registry.yarnpkg.com/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz#06de25df4db327ac931981d1bdb067e5af68d051"
+  integrity sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==
+
+lodash.isundefined@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz#23ef3d9535565203a66cefd5b830f848911afb48"
+  integrity sha1-I+89lTVWUgOmbO/VuDD4SJEa+0g=
+
+lodash.keys@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
+  integrity sha1-oIYCrBLk+4P5H8H7ejYKTZujUgU=
+
+lodash.map@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
+  integrity sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=
+
 lodash.memoize@^4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"
@@ -3682,17 +4328,47 @@ lodash.once@^4.1.1:
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
   integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=
 
+lodash.reduce@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
+  integrity sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=
+
 lodash.set@^4.3.2:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23"
   integrity sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=
 
+lodash.size@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.size/-/lodash.size-4.2.0.tgz#71fe75ed3eabdb2bcb73a1b0b4f51c392ee27b86"
+  integrity sha1-cf517T6r2yvLc6GwtPUcOS7ie4Y=
+
+lodash.topairs@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.topairs/-/lodash.topairs-4.3.0.tgz#3b6deaa37d60fb116713c46c5f17ea190ec48d64"
+  integrity sha1-O23qo31g+xFnE8RsXxfqGQ7EjWQ=
+
+lodash.transform@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.transform/-/lodash.transform-4.6.0.tgz#12306422f63324aed8483d3f38332b5f670547a0"
+  integrity sha1-EjBkIvYzJK7YSD0/ODMrX2cFR6A=
+
+lodash.union@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
+  integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
+
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
   integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
 
-lodash@4.17.15, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.10:
+lodash.values@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347"
+  integrity sha1-o6bCsOvsxcLLocF+bmIP6BtT00c=
+
+lodash@4.17.15, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.10:
   version "4.17.15"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
   integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
@@ -3719,6 +4395,13 @@ log-update@^1.0.2:
     ansi-escapes "^1.0.0"
     cli-cursor "^1.0.2"
 
+logic-solver@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/logic-solver/-/logic-solver-2.0.1.tgz#e9fa47002eb5d8cda7616d41639b97552eb674be"
+  integrity sha1-6fpHAC612M2nYW1BY5uXVS62dL4=
+  dependencies:
+    underscore "^1.7.0"
+
 loud-rejection@^1.0.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f"
@@ -3727,11 +4410,16 @@ loud-rejection@^1.0.0:
     currently-unhandled "^0.4.1"
     signal-exit "^3.0.0"
 
-lowercase-keys@^1.0.0:
+lowercase-keys@^1.0.0, lowercase-keys@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f"
   integrity sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==
 
+lowercase-keys@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479"
+  integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==
+
 lru-cache@^4.0.0, lru-cache@^4.0.1, lru-cache@^4.1.2:
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
@@ -3747,6 +4435,13 @@ lru-cache@^5.1.1:
   dependencies:
     yallist "^3.0.2"
 
+lru-cache@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
+  integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==
+  dependencies:
+    yallist "^4.0.0"
+
 macos-release@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-2.3.0.tgz#eb1930b036c0800adebccd5f17bc4c12de8bb71f"
@@ -3766,12 +4461,12 @@ magic-string@^0.25.1:
   dependencies:
     sourcemap-codec "^1.4.4"
 
-make-dir@^1.0.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
-  integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==
+make-dir@^3.0.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
+  integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==
   dependencies:
-    pify "^3.0.0"
+    semver "^6.0.0"
 
 map-cache@^0.2.2:
   version "0.2.2"
@@ -3847,6 +4542,11 @@ merge-stream@^1.0.1:
   dependencies:
     readable-stream "^2.0.1"
 
+merge2@^1.2.3, merge2@^1.3.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
+  integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
+
 methods@^1.1.1, methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
@@ -3871,6 +4571,14 @@ micromatch@^3.1.10:
     snapdragon "^0.8.1"
     to-regex "^3.0.2"
 
+micromatch@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259"
+  integrity sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==
+  dependencies:
+    braces "^3.0.1"
+    picomatch "^2.0.5"
+
 mime-db@1.40.0:
   version "1.40.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32"
@@ -3905,6 +4613,16 @@ mimic-fn@^1.0.0:
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
   integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==
 
+mimic-response@^1.0.0, mimic-response@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
+  integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
+
+mimic-response@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
+  integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==
+
 minimatch@^3.0.4, minimatch@~3.0.2:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@@ -3922,6 +4640,21 @@ minimist@1.2.0, minimist@^1.1.3, minimist@^1.2.0:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
   integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=
 
+minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0:
+  version "2.9.0"
+  resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6"
+  integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==
+  dependencies:
+    safe-buffer "^5.1.2"
+    yallist "^3.0.0"
+
+minizlib@^1.2.1:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d"
+  integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==
+  dependencies:
+    minipass "^2.9.0"
+
 mixin-deep@^1.2.0:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566"
@@ -3937,6 +4670,11 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
   dependencies:
     minimist "0.0.8"
 
+mkdirp@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
+  integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
+
 moment-timezone@^0.5.28:
   version "0.5.28"
   resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.28.tgz#f093d789d091ed7b055d82aa81a82467f72e4338"
@@ -4022,19 +4760,19 @@ ndarray@^1.0.13, ndarray@^1.0.18:
     iota-array "^1.0.0"
     is-buffer "^1.0.2"
 
-needle@^2.2.4, needle@^2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/needle/-/needle-2.4.0.tgz#6833e74975c444642590e15a750288c5f939b57c"
-  integrity sha512-4Hnwzr3mi5L97hMYeNl8wRW/Onhy4nUKR/lVemJ8gJedxxUyBLm9kkrDColJvoSfwi0jCNhD+xCdOtiGDQiRZg==
+needle@^2.3.3:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/needle/-/needle-2.4.1.tgz#14af48732463d7475696f937626b1b993247a56a"
+  integrity sha512-x/gi6ijr4B7fwl6WYL9FwlCvRQKGlUNvnceho8wxkwXqN8jvVmmmATTmZPRRG7b/yC1eode26C2HO9jl78Du9g==
   dependencies:
     debug "^3.2.6"
     iconv-lite "^0.4.4"
     sax "^1.2.4"
 
-needle@^2.3.3:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/needle/-/needle-2.4.1.tgz#14af48732463d7475696f937626b1b993247a56a"
-  integrity sha512-x/gi6ijr4B7fwl6WYL9FwlCvRQKGlUNvnceho8wxkwXqN8jvVmmmATTmZPRRG7b/yC1eode26C2HO9jl78Du9g==
+needle@^2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/needle/-/needle-2.5.0.tgz#e6fc4b3cc6c25caed7554bd613a5cf0bac8c31c0"
+  integrity sha512-o/qITSDR0JCyCKEQ1/1bnUXMmznxabbwi/Y4WwJElf+evwJNFNwIDMCCt5IigFVxgeGBJESLohGtIS9gEzo1fA==
   dependencies:
     debug "^3.2.6"
     iconv-lite "^0.4.4"
@@ -4108,6 +4846,14 @@ node-sass@^4.13.1:
     stdout-stream "^1.4.0"
     "true-case-path" "^1.0.2"
 
+node.extend@^2.0.0:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/node.extend/-/node.extend-2.0.2.tgz#b4404525494acc99740f3703c496b7d5182cc6cc"
+  integrity sha512-pDT4Dchl94/+kkgdwyS2PauDFjZG0Hk0IcHIB+LkW27HLDtdoeMxHTxZh39DYbPP8UflWXWj9JcdDozF+YDOpQ==
+  dependencies:
+    has "^1.0.3"
+    is "^3.2.1"
+
 "nopt@2 || 3":
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9"
@@ -4132,11 +4878,16 @@ normalize-package-data@^2.3.2, normalize-package-data@^2.3.4:
     semver "2 || 3 || 4 || 5"
     validate-npm-package-license "^3.0.1"
 
-normalize-url@^3.0.0, normalize-url@^3.3.0:
+normalize-url@^3.0.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
   integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==
 
+normalize-url@^4.1.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129"
+  integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==
+
 npm-run-path@^2.0.0:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
@@ -4190,11 +4941,6 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
-object-hash@^1.3.1:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
-  integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==
-
 object-hash@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
@@ -4284,6 +5030,13 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0:
   dependencies:
     wrappy "1"
 
+once@~1.3.0:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/once/-/once-1.3.3.tgz#b2e261557ce4c314ec8304f3fa82663e4297ca20"
+  integrity sha1-suJhVXzkwxTsgwTz+oJmPkKXyiA=
+  dependencies:
+    wrappy "1"
+
 onetime@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/onetime/-/onetime-1.1.0.tgz#a1f7838f8314c516f05ecefcbc4ccfe04b4ed789"
@@ -4359,6 +5112,16 @@ osenv@0:
     os-homedir "^1.0.0"
     os-tmpdir "^1.0.0"
 
+p-cancelable@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
+  integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
+
+p-cancelable@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.0.0.tgz#4a3740f5bdaf5ed5d7c3e34882c6fb5d6b266a6e"
+  integrity sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg==
+
 p-finally@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
@@ -4371,6 +5134,13 @@ p-limit@^2.0.0:
   dependencies:
     p-try "^2.0.0"
 
+p-limit@^2.2.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
+  integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==
+  dependencies:
+    p-try "^2.0.0"
+
 p-locate@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
@@ -4423,15 +5193,20 @@ pac-resolver@^3.0.0:
     netmask "^1.0.6"
     thunkify "^2.1.2"
 
-package-json@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/package-json/-/package-json-4.0.1.tgz#8869a0401253661c4c4ca3da6c2121ed555f5eed"
-  integrity sha1-iGmgQBJTZhxMTKPabCEh7VVfXu0=
+package-json@^6.3.0:
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0"
+  integrity sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==
   dependencies:
-    got "^6.7.1"
-    registry-auth-token "^3.0.1"
-    registry-url "^3.0.3"
-    semver "^5.1.0"
+    got "^9.6.0"
+    registry-auth-token "^4.0.0"
+    registry-url "^5.0.0"
+    semver "^6.2.0"
+
+pako@~0.2.0:
+  version "0.2.9"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75"
+  integrity sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=
 
 pako@~1.0.2:
   version "1.0.11"
@@ -4470,29 +5245,18 @@ parse-json@^4.0.0:
     error-ex "^1.3.1"
     json-parse-better-errors "^1.0.1"
 
+parse-link-header@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/parse-link-header/-/parse-link-header-1.0.1.tgz#bedfe0d2118aeb84be75e7b025419ec8a61140a7"
+  integrity sha1-vt/g0hGK64S+deewJUGeyKYRQKc=
+  dependencies:
+    xtend "~4.0.1"
+
 parse-passwd@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
 
-parse-path@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-4.0.1.tgz#0ec769704949778cb3b8eda5e994c32073a1adff"
-  integrity sha512-d7yhga0Oc+PwNXDvQ0Jv1BuWkLVPXcAoQ/WREgd6vNNoKYaW52KI+RdOFjI63wjkmps9yUE8VS4veP+AgpQ/hA==
-  dependencies:
-    is-ssh "^1.3.0"
-    protocols "^1.4.0"
-
-parse-url@^5.0.0:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-5.0.1.tgz#99c4084fc11be14141efa41b3d117a96fcb9527f"
-  integrity sha512-flNUPP27r3vJpROi0/R3/2efgKkyXqnXwyP1KQ2U0SfFRgdizOdWfvrrvJg1LuOoxs7GQhmxJlq23IpQ/BkByg==
-  dependencies:
-    is-ssh "^1.3.0"
-    normalize-url "^3.3.0"
-    parse-path "^4.0.0"
-    protocols "^1.4.0"
-
 parseqs@0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d"
@@ -4544,6 +5308,11 @@ path-key@^2.0.0, path-key@^2.0.1:
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
   integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
 
+path-key@^3.1.0:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
+  integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
+
 path-parse@^1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c"
@@ -4563,6 +5332,20 @@ path-type@^1.0.0:
     pify "^2.0.0"
     pinkie-promise "^2.0.0"
 
+path-type@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
+  integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
+
+peek-stream@^1.1.0:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/peek-stream/-/peek-stream-1.1.3.tgz#3b35d84b7ccbbd262fff31dc10da56856ead6d67"
+  integrity sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==
+  dependencies:
+    buffer-from "^1.0.0"
+    duplexify "^3.5.0"
+    through2 "^2.0.3"
+
 pend@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@@ -4573,6 +5356,11 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
+picomatch@^2.0.5, picomatch@^2.2.1:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
+  integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==
+
 pify@^2.0.0, pify@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
@@ -4595,6 +5383,11 @@ pinkie@^2.0.0:
   resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
   integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA=
 
+pluralize@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777"
+  integrity sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==
+
 pngjs@^3.3.3:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
@@ -4980,10 +5773,10 @@ prelude-ls@~1.1.2:
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
   integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
 
-prepend-http@^1.0.1:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc"
-  integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=
+prepend-http@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
+  integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=
 
 prettier@1.16.3:
   version "1.16.3"
@@ -4995,15 +5788,10 @@ prettier@^1.13.0:
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.16.4.tgz#73e37e73e018ad2db9c76742e2647e21790c9717"
   integrity sha512-ZzWuos7TI5CKUeQAtFd6Zhm2s6EpAD/ZLApIhsF9pRvRtM1RFo61dM/4MSRUA0SuLugA/zgrZD8m0BaY46Og7g==
 
-prettier@^1.19.1:
-  version "1.19.1"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.19.1.tgz#f7d7f5ff8a9cd872a7be4ca142095956a60797cb"
-  integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==
-
-process-nextick-args@~1.0.6:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
-  integrity sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=
+pretty-bytes@^5.1.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.3.0.tgz#f2849e27db79fb4d6cfe24764fc4134f165989f2"
+  integrity sha512-hjGrh+P926p4R4WbaB6OckyRtO0F0/lQBiT+0gnxjV+5kjPBrfVBFCsCLbMqVQeydvIoouYTCmmEURiH3R1Bdg==
 
 process-nextick-args@~2.0.0:
   version "2.0.1"
@@ -5015,22 +5803,32 @@ progress@^2.0.3:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
+promise-deferred@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/promise-deferred/-/promise-deferred-2.0.3.tgz#b99c9588820798501862a593d49cece51d06fd7f"
+  integrity sha512-n10XaoznCzLfyPFOlEE8iurezHpxrYzyjgq/1eW9Wk1gJwur/N7BdBmjJYJpqMeMcXK4wEbzo2EvZQcqjYcKUQ==
+  dependencies:
+    promise "^7.3.1"
+
 promise.series@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/promise.series/-/promise.series-0.2.0.tgz#2cc7ebe959fc3a6619c04ab4dbdc9e452d864bbd"
   integrity sha1-LMfr6Vn8OmYZwEq029yeRS2GS70=
 
-"promise@>=3.2 <8", promise@^7.1.1:
+"promise@>=3.2 <8", promise@^7.1.1, promise@^7.3.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf"
   integrity sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==
   dependencies:
     asap "~2.0.3"
 
-protocols@^1.1.0, protocols@^1.4.0:
-  version "1.4.7"
-  resolved "https://registry.yarnpkg.com/protocols/-/protocols-1.4.7.tgz#95f788a4f0e979b291ffefcf5636ad113d037d32"
-  integrity sha512-Fx65lf9/YDn3hUX08XUc0J8rSux36rEsyiv21ZGUC1mOyeM3lTRpZLcrm8aAolzS4itwVfm7TAPyxC2E5zd6xg==
+promiseback@^2.0.2:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/promiseback/-/promiseback-2.0.3.tgz#bd468d86930e8cd44bfc3292de9a6fbafb6378e6"
+  integrity sha512-VZXdCwS0ppVNTIRfNsCvVwJAaP2b+pxQF7lM8DMWfmpNWyTxB6O5YNbzs+8z0ki/KIBHKHk308NTIl4kJUem3w==
+  dependencies:
+    is-callable "^1.1.5"
+    promise-deferred "^2.0.3"
 
 proxy-addr@~2.0.5:
   version "2.0.6"
@@ -5074,6 +5872,14 @@ psl@^1.1.24:
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
   integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
 
+pump@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909"
+  integrity sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==
+  dependencies:
+    end-of-stream "^1.1.0"
+    once "^1.3.1"
+
 pump@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
@@ -5082,6 +5888,15 @@ pump@^3.0.0:
     end-of-stream "^1.1.0"
     once "^1.3.1"
 
+pumpify@^1.3.3:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/pumpify/-/pumpify-1.5.1.tgz#36513be246ab27570b1a374a5ce278bfd74370ce"
+  integrity sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==
+  dependencies:
+    duplexify "^3.6.0"
+    inherits "^2.0.3"
+    pump "^2.0.0"
+
 punycode@1.3.2:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
@@ -5097,6 +5912,13 @@ punycode@^2.1.0:
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
   integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
 
+pupa@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.0.1.tgz#dbdc9ff48ffbea4a26a069b6f9f7abb051008726"
+  integrity sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==
+  dependencies:
+    escape-goat "^2.0.0"
+
 q@^1.1.2:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
@@ -5130,6 +5952,11 @@ querystring@0.2.0, querystring@^0.2.0:
   resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
   integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
 
+quick-lru@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932"
+  integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==
+
 quill-delta@4.2.1:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/quill-delta/-/quill-delta-4.2.1.tgz#ad4f191cdf3be5079c5dc3991b9603a5cc0db69a"
@@ -5221,7 +6048,7 @@ raw-loader@^0.5.1:
   resolved "https://registry.yarnpkg.com/raw-loader/-/raw-loader-0.5.1.tgz#0c3d0beaed8a01c966d9787bf778281252a979aa"
   integrity sha1-DD0L6u2KAclm2Xh793goElKpeao=
 
-rc@^1.0.1, rc@^1.1.6:
+rc@^1.2.8:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"
   integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==
@@ -5258,7 +6085,7 @@ readable-stream@1.1.x:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@2, readable-stream@^2.2.2, readable-stream@~2.3.6:
+readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@~2.3.6:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -5284,7 +6111,7 @@ readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.3.5:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.0.1, readable-stream@^3.1.1:
+readable-stream@^3.0.1, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -5293,18 +6120,6 @@ readable-stream@^3.0.1, readable-stream@^3.1.1:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
-readable-stream@~2.0.6:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
-  integrity sha1-j5A0HmilPMySh4jaz80Rs265t44=
-  dependencies:
-    core-util-is "~1.0.0"
-    inherits "~2.0.1"
-    isarray "~1.0.0"
-    process-nextick-args "~1.0.6"
-    string_decoder "~0.10.x"
-    util-deprecate "~1.0.1"
-
 redent@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde"
@@ -5391,20 +6206,19 @@ regexpu-core@^4.2.0:
     unicode-match-property-ecmascript "^1.0.4"
     unicode-match-property-value-ecmascript "^1.0.2"
 
-registry-auth-token@^3.0.1:
-  version "3.4.0"
-  resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-3.4.0.tgz#d7446815433f5d5ed6431cd5dca21048f66b397e"
-  integrity sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==
+registry-auth-token@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.0.tgz#1d37dffda72bbecd0f581e4715540213a65eb7da"
+  integrity sha512-P+lWzPrsgfN+UEpDS3U8AQKg/UjZX6mQSJueZj3EK+vNESoqBSpBUD3gmu4sF9lOsjXWjF11dQKUqemf3veq1w==
   dependencies:
-    rc "^1.1.6"
-    safe-buffer "^5.0.1"
+    rc "^1.2.8"
 
-registry-url@^3.0.3:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/registry-url/-/registry-url-3.1.0.tgz#3d4ef870f73dde1d77f0cf9a381432444e174942"
-  integrity sha1-PU74cPc93h138M+aOBQyRE4XSUI=
+registry-url@^5.0.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/registry-url/-/registry-url-5.1.0.tgz#e98334b50d5434b81136b44ec638d9c2009c5009"
+  integrity sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==
   dependencies:
-    rc "^1.0.1"
+    rc "^1.2.8"
 
 regjsgen@^0.2.0:
   version "0.2.0"
@@ -5505,6 +6319,11 @@ reserved-words@^0.1.2:
   resolved "https://registry.yarnpkg.com/reserved-words/-/reserved-words-0.1.2.tgz#00a0940f98cd501aeaaac316411d9adc52b31ab1"
   integrity sha1-AKCUD5jNUBrqqsMWQR2a3FKzGrE=
 
+resolve-alpn@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c"
+  integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA==
+
 resolve-dir@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43"
@@ -5530,6 +6349,20 @@ resolve@^1.10.0, resolve@^1.4.0, resolve@^1.5.0:
   dependencies:
     path-parse "^1.0.6"
 
+responselike@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
+  integrity sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=
+  dependencies:
+    lowercase-keys "^1.0.0"
+
+responselike@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/responselike/-/responselike-2.0.0.tgz#26391bcc3174f750f9a79eacc40a12a5c42d7723"
+  integrity sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==
+  dependencies:
+    lowercase-keys "^2.0.0"
+
 restore-cursor@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541"
@@ -5551,6 +6384,11 @@ ret@~0.1.10:
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
   integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
 
+reusify@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
+  integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
+
 rgb-regex@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1"
@@ -5568,6 +6406,13 @@ rimraf@2, rimraf@^2.6.3:
   dependencies:
     glob "^7.1.3"
 
+rimraf@^3.0.0:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
+  integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==
+  dependencies:
+    glob "^7.1.3"
+
 rollup-plugin-buble@^0.19.2:
   version "0.19.6"
   resolved "https://registry.yarnpkg.com/rollup-plugin-buble/-/rollup-plugin-buble-0.19.6.tgz#55ee0995d8870d536f01f4277c3eef4276e8747e"
@@ -5671,6 +6516,11 @@ run-async@^2.2.0:
   dependencies:
     is-promise "^2.1.0"
 
+run-parallel@^1.1.9:
+  version "1.1.9"
+  resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679"
+  integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==
+
 rxjs@^5.0.0-beta.11:
   version "5.5.12"
   resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-5.5.12.tgz#6fa61b8a77c3d793dbaf270bee2f43f652d741cc"
@@ -5735,24 +6585,24 @@ secure-keys@^1.0.0:
   resolved "https://registry.yarnpkg.com/secure-keys/-/secure-keys-1.0.0.tgz#f0c82d98a3b139a8776a8808050b824431087fca"
   integrity sha1-8MgtmKOxOah3aogIBQuCRDEIf8o=
 
-semver-diff@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36"
-  integrity sha1-S7uEN8jTfksM8aaP1ybsbWRdbTY=
+semver-diff@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-3.1.1.tgz#05f77ce59f325e00e2706afd67bb506ddb1ca32b"
+  integrity sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==
   dependencies:
-    semver "^5.0.3"
+    semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.0.3, semver@^5.1.0, semver@^5.3.0, semver@^5.5.0, semver@^5.5.1:
+"semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.5.0, semver@^5.5.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
-semver@^6.0.0, semver@^6.1.0, semver@^6.1.2:
+semver@^6.0.0, semver@^6.1.0, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
-semver@^7.3.2:
+semver@^7.1.2:
   version "7.3.2"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
   integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
@@ -5838,11 +6688,23 @@ shebang-command@^1.2.0:
   dependencies:
     shebang-regex "^1.0.0"
 
+shebang-command@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
+  integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==
+  dependencies:
+    shebang-regex "^3.0.0"
+
 shebang-regex@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
   integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
 
+shebang-regex@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
+  integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
+
 showdown@^1.9.1:
   version "1.9.1"
   resolved "https://registry.yarnpkg.com/showdown/-/showdown-1.9.1.tgz#134e148e75cd4623e09c21b0511977d79b5ad0ef"
@@ -5875,6 +6737,11 @@ simple-swizzle@^0.2.2:
   dependencies:
     is-arrayish "^0.3.1"
 
+slash@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
+  integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
+
 slice-ansi@0.0.4:
   version "0.0.4"
   resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35"
@@ -5933,52 +6800,75 @@ snyk-config@^3.0.0:
     lodash.merge "^4.6.2"
     nconf "^0.10.0"
 
-snyk-docker-plugin@3.2.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/snyk-docker-plugin/-/snyk-docker-plugin-3.2.0.tgz#5fb1cf045e45b4a3cfcdc9878ab29c80334d9742"
-  integrity sha512-LKsvGcRVBYzyTNT/Z5kImm6uHMX3wAs7gvR4dO8zqBVzCsn3zfi//kmRHWh7zhgvIb6reuhUqY1hMXaz0q/mBw==
+snyk-cpp-plugin@1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/snyk-cpp-plugin/-/snyk-cpp-plugin-1.2.0.tgz#1e554eece61d40288bbc745f040c2ca54ec7e08a"
+  integrity sha512-zc2h0IAAjcxJwxWp4rauliwEUXZahmC3WoCgjTmRi36UhzwlxWJf8QpIeSOKJnc3t0NhlKzUc1YFkaQGJ2dXVQ==
   dependencies:
-    "@snyk/rpm-parser" "^1.1.0"
+    tslib "^2.0.0"
+
+snyk-docker-plugin@3.17.0:
+  version "3.17.0"
+  resolved "https://registry.yarnpkg.com/snyk-docker-plugin/-/snyk-docker-plugin-3.17.0.tgz#9f48f9d4455b15dc15fdfc37137c567e9c9e24e2"
+  integrity sha512-Pq0xr/NQQUss3bLb0rTfVtsot/9ID6TzZCWlF9nMrj9JFrJRrJPQxi/KO/uqfX0Nbwz4VysXDzwdWNtrxpaabg==
+  dependencies:
+    "@snyk/rpm-parser" "^2.0.0"
+    "@snyk/snyk-docker-pull" "^3.2.0"
     debug "^4.1.1"
+    docker-modem "2.1.3"
     dockerfile-ast "0.0.19"
-    event-loop-spinner "^1.1.0"
+    event-loop-spinner "^2.0.0"
+    gunzip-maybe "^1.4.2"
+    mkdirp "^1.0.4"
     semver "^6.1.0"
     snyk-nodejs-lockfile-parser "1.22.0"
     tar-stream "^2.1.0"
+    tmp "^0.2.1"
     tslib "^1"
+    uuid "^8.2.0"
 
-snyk-go-parser@1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/snyk-go-parser/-/snyk-go-parser-1.4.0.tgz#0fa7e4b9f2cf14c95dbc09206416ac4676436c70"
-  integrity sha512-zcLA8u/WreycCjFKBblYfxszg7Fmnemuu9Ug/CE/jqF0yBXsI5DCWMteUvFkoa8DRntfGTlgf98TRl2aTSc2MQ==
+snyk-go-parser@1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/snyk-go-parser/-/snyk-go-parser-1.4.1.tgz#df16a5fbd7a517ee757268ef081abc33506c8857"
+  integrity sha512-StU3uHB85VMEkcgXta63M0Fgd+9cs5sMCjQXTBoYTdE4dxarPn7U67yCuwkRRdZdny1ZXtzfY8LKns9i0+dy9w==
   dependencies:
     toml "^3.0.0"
     tslib "^1.10.0"
 
-snyk-go-plugin@1.14.0:
-  version "1.14.0"
-  resolved "https://registry.yarnpkg.com/snyk-go-plugin/-/snyk-go-plugin-1.14.0.tgz#a96c3c09ea1728807158c52dd2d9a6d650617a82"
-  integrity sha512-9L+76De8F6yXWb+O3DA8QUi7+eDF2mOzCOveEPUJGkqWIDmurIiFcVxHJoj0EStjcxb3dX367KKlDlfFx+HiyA==
+snyk-go-plugin@1.16.0:
+  version "1.16.0"
+  resolved "https://registry.yarnpkg.com/snyk-go-plugin/-/snyk-go-plugin-1.16.0.tgz#4688e45e59d1673256f645fccca2fa6a7ab0f55a"
+  integrity sha512-XNGHEFyP+pCzcqmXnj5T/1Oy6AZzm2WkTSuUpohWQ/09ecMRCCv2yrr/kwMQemrKN4+7CoJS/9xfm3GnNlzVHA==
   dependencies:
+    "@snyk/dep-graph" "1.19.3"
     "@snyk/graphlib" "2.1.9-patch"
     debug "^4.1.1"
-    snyk-go-parser "1.4.0"
-    tmp "0.1.0"
+    snyk-go-parser "1.4.1"
+    tmp "0.2.0"
     tslib "^1.10.0"
 
-snyk-gradle-plugin@3.2.7:
-  version "3.2.7"
-  resolved "https://registry.yarnpkg.com/snyk-gradle-plugin/-/snyk-gradle-plugin-3.2.7.tgz#3a4f37afeac00407d7051891a0e3ee4d69f6866f"
-  integrity sha512-fBgQpRwfuHGuPvYGEjgNBtuwnkNAV9sv17lG+eWdQO3ntkAKnt1RyzQydAdffK2e8339XM6Mg/1EPYuIGIY3TA==
+snyk-gradle-plugin@3.5.1:
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/snyk-gradle-plugin/-/snyk-gradle-plugin-3.5.1.tgz#a424a7b8136471296e971e7b25f4776d1a08b9ff"
+  integrity sha512-8tZwQCqRbjp1azvc+bBRXSbn2AjbUpFAM6qoSpM/IZpfGl1RaOtz4/JTkGFxj+iBhTFoAkGxEunT66eO0pHZZw==
   dependencies:
-    "@snyk/cli-interface" "2.3.2"
+    "@snyk/cli-interface" "2.8.0"
+    "@snyk/dep-graph" "^1.17.0"
     "@types/debug" "^4.1.4"
-    chalk "^2.4.2"
+    chalk "^3.0.0"
     debug "^4.1.1"
-    tmp "0.0.33"
-    tslib "^1.9.3"
+    tmp "0.2.1"
+    tslib "^2.0.0"
 
-snyk-module@1.9.1, snyk-module@^1.6.0:
+snyk-module@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/snyk-module/-/snyk-module-3.1.0.tgz#3e088ff473ddf0d4e253a46ea6749d76d8e6e7ba"
+  integrity sha512-HHuOYEAACpUpkFgU8HT57mmxmonaJ4O3YADoSkVhnhkmJ+AowqZyJOau703dYHNrq2DvQ7qYw81H7yyxS1Nfjw==
+  dependencies:
+    debug "^4.1.1"
+    hosted-git-info "^3.0.4"
+
+snyk-module@^1.6.0:
   version "1.9.1"
   resolved "https://registry.yarnpkg.com/snyk-module/-/snyk-module-1.9.1.tgz#b2a78f736600b0ab680f1703466ed7309c980804"
   integrity sha512-A+CCyBSa4IKok5uEhqT+hV/35RO6APFNLqk9DRRHg7xW2/j//nPX8wTSZUPF8QeRNEk/sX+6df7M1y6PBHGSHA==
@@ -5996,15 +6886,15 @@ snyk-module@^2.0.2:
     debug "^3.1.0"
     hosted-git-info "^2.7.1"
 
-snyk-mvn-plugin@2.15.1:
-  version "2.15.1"
-  resolved "https://registry.yarnpkg.com/snyk-mvn-plugin/-/snyk-mvn-plugin-2.15.1.tgz#df3652456ec887007417943b1f88f38c28b04f99"
-  integrity sha512-QxgS+DpPrGZVI41rNXWC2vUHy1pAC1n7lLts5+ospQZtIWAUo2JAo+M6Qp4L/CD4tuG9bMXNXaCrAQNgemHGLA==
+snyk-mvn-plugin@2.19.1:
+  version "2.19.1"
+  resolved "https://registry.yarnpkg.com/snyk-mvn-plugin/-/snyk-mvn-plugin-2.19.1.tgz#00b68a459a4da9e39b0d8c416ae05496b3b03ee3"
+  integrity sha512-VXYJSdhUmOQAyxdsv5frAKbi3UOcHPabWEQxQ9wxhVBEEmx2lP5ajv1a+ntxwWwL7u3jdc+rnCIKHpLlQJ5nyw==
   dependencies:
-    "@snyk/cli-interface" "2.5.0"
-    "@snyk/java-call-graph-builder" "1.8.1"
+    "@snyk/cli-interface" "2.8.1"
+    "@snyk/java-call-graph-builder" "1.13.1"
     debug "^4.1.1"
-    needle "^2.4.0"
+    needle "^2.5.0"
     tmp "^0.1.0"
     tslib "1.11.1"
 
@@ -6023,17 +6913,39 @@ snyk-nodejs-lockfile-parser@1.22.0:
     tslib "^1.9.3"
     uuid "^3.3.2"
 
-snyk-nuget-plugin@1.17.0:
-  version "1.17.0"
-  resolved "https://registry.yarnpkg.com/snyk-nuget-plugin/-/snyk-nuget-plugin-1.17.0.tgz#20c18d962d725b2217ab2c94a49a1a6524162c53"
-  integrity sha512-t7iZ87LBhCK6P2/mJsQh7Dmk3J9zd+IHL4yoSK95Iyk/gP8r++DZijoRHEXy8BlS+eOtSAj1vgCYvv2eAmG28w==
+snyk-nodejs-lockfile-parser@1.27.0:
+  version "1.27.0"
+  resolved "https://registry.yarnpkg.com/snyk-nodejs-lockfile-parser/-/snyk-nodejs-lockfile-parser-1.27.0.tgz#cde4d052ac429c5fa2844c981578f5ff5dea3022"
+  integrity sha512-s1yEqEC8uxELzEXqXmnWKpoOIn0rM8StMjuy0X5U+FmxGvO3jRtFDU5hcVw9cAULLfrQYBHRowrO/5FaR79fxA==
+  dependencies:
+    "@snyk/graphlib" "2.1.9-patch"
+    "@yarnpkg/core" "^2.0.0-rc.29"
+    "@yarnpkg/lockfile" "^1.1.0"
+    event-loop-spinner "^2.0.0"
+    got "11.4.0"
+    lodash.clonedeep "^4.5.0"
+    lodash.flatmap "^4.5.0"
+    lodash.isempty "^4.4.0"
+    lodash.set "^4.3.2"
+    lodash.topairs "^4.3.0"
+    p-map "2.1.0"
+    snyk-config "^3.0.0"
+    source-map-support "^0.5.7"
+    tslib "^1.9.3"
+    uuid "^3.3.2"
+    yaml "^1.9.2"
+
+snyk-nuget-plugin@1.18.1:
+  version "1.18.1"
+  resolved "https://registry.yarnpkg.com/snyk-nuget-plugin/-/snyk-nuget-plugin-1.18.1.tgz#5296ee7b3ee96fc8186fa157f71697b804173a60"
+  integrity sha512-Bq+IzbyewxIrUhgdFaDKS5wCNixERC7QBitKsZGM3uCOr9fJM8rr5qg5SS9UIU7eyeKvzuVO/V1yDzjo1cKvUw==
   dependencies:
     "@snyk/lodash" "4.17.15-patch"
-    debug "^3.1.0"
+    debug "^4.1.1"
     dotnet-deps-parser "4.10.0"
-    jszip "3.1.5"
+    jszip "3.3.0"
     snyk-paket-parser "1.6.0"
-    tslib "^1.9.3"
+    tslib "^1.11.2"
     xml2js "^0.4.17"
 
 snyk-paket-parser@1.6.0:
@@ -6067,10 +6979,10 @@ snyk-policy@1.14.1:
     snyk-try-require "^1.3.1"
     then-fs "^2.0.0"
 
-snyk-python-plugin@1.17.0:
-  version "1.17.0"
-  resolved "https://registry.yarnpkg.com/snyk-python-plugin/-/snyk-python-plugin-1.17.0.tgz#9bc38ba3c799c3cbef7676a1081f52608690d254"
-  integrity sha512-EKdVOUlvhiVpXA5TeW8vyxYVqbITAfT+2AbL2ZRiiUNLP5ae+WiNYaPy7aB5HAS9IKBKih+IH8Ag65Xu1IYSYA==
+snyk-python-plugin@1.17.1:
+  version "1.17.1"
+  resolved "https://registry.yarnpkg.com/snyk-python-plugin/-/snyk-python-plugin-1.17.1.tgz#303ec2885ef748634d89f22f3099ef1febdc3325"
+  integrity sha512-KKklat9Hfbj4hw2y63LRhgmziYzmyRt+cSuzN5KDmBSAGYck0EAoPDtNpJXjrIs1kPNz28EXnE6NDnadXnOjiQ==
   dependencies:
     "@snyk/cli-interface" "^2.0.3"
     tmp "0.0.33"
@@ -6134,47 +7046,45 @@ snyk-try-require@1.3.1, snyk-try-require@^1.1.1, snyk-try-require@^1.3.1:
     lru-cache "^4.0.0"
     then-fs "^2.0.0"
 
-snyk@^1.316.1:
-  version "1.320.2"
-  resolved "https://registry.yarnpkg.com/snyk/-/snyk-1.320.2.tgz#ba61c8d1cd5cf67314a51d218340a2909364f9c2"
-  integrity sha512-PfXY83G/3j+gBOt2EbMFWL/MVb6QJK+XE1l2CL5hlqzyeQp/tVVCI4RNNg0h+itzHfffVQCuAPRxbh1ijU8BEg==
+snyk@1.382.0, snyk@^1.382.0:
+  version "1.382.0"
+  resolved "https://registry.yarnpkg.com/snyk/-/snyk-1.382.0.tgz#5625931dd066fca406f2d35663eb7280d57565bd"
+  integrity sha512-lZsuJ6HUVNO7ZYDQm6B2gPdcXiRtd8bCQUre8xYMZu1VakKwrCCNsJbxo91z3gK5aYqempjmTe7+qH2CksXZ+Q==
   dependencies:
-    "@snyk/cli-interface" "2.6.0"
-    "@snyk/configstore" "^3.2.0-rc1"
-    "@snyk/dep-graph" "1.18.3"
+    "@snyk/cli-interface" "2.8.1"
+    "@snyk/dep-graph" "1.19.3"
     "@snyk/gemfile" "1.2.0"
     "@snyk/graphlib" "2.1.9-patch"
     "@snyk/inquirer" "6.2.2-patch"
     "@snyk/lodash" "^4.17.15-patch"
     "@snyk/ruby-semver" "2.2.0"
-    "@snyk/snyk-cocoapods-plugin" "2.2.0"
-    "@snyk/update-notifier" "^2.5.1-rc2"
-    "@types/agent-base" "^4.2.0"
+    "@snyk/snyk-cocoapods-plugin" "2.5.0"
     abbrev "^1.1.1"
     ansi-escapes "3.2.0"
     chalk "^2.4.2"
     cli-spinner "0.2.10"
-    debug "^3.1.0"
+    configstore "^5.0.1"
+    debug "^4.1.1"
     diff "^4.0.1"
-    git-url-parse "11.1.2"
     glob "^7.1.3"
-    needle "^2.2.4"
+    needle "^2.5.0"
     open "^7.0.3"
     os-name "^3.0.0"
     proxy-agent "^3.1.1"
     proxy-from-env "^1.0.0"
     semver "^6.0.0"
     snyk-config "3.1.0"
-    snyk-docker-plugin "3.2.0"
-    snyk-go-plugin "1.14.0"
-    snyk-gradle-plugin "3.2.7"
-    snyk-module "1.9.1"
-    snyk-mvn-plugin "2.15.1"
-    snyk-nodejs-lockfile-parser "1.22.0"
-    snyk-nuget-plugin "1.17.0"
+    snyk-cpp-plugin "1.2.0"
+    snyk-docker-plugin "3.17.0"
+    snyk-go-plugin "1.16.0"
+    snyk-gradle-plugin "3.5.1"
+    snyk-module "3.1.0"
+    snyk-mvn-plugin "2.19.1"
+    snyk-nodejs-lockfile-parser "1.27.0"
+    snyk-nuget-plugin "1.18.1"
     snyk-php-plugin "1.9.0"
     snyk-policy "1.14.1"
-    snyk-python-plugin "1.17.0"
+    snyk-python-plugin "1.17.1"
     snyk-resolve "1.0.1"
     snyk-resolve-deps "4.4.0"
     snyk-sbt-plugin "2.11.0"
@@ -6183,7 +7093,7 @@ snyk@^1.316.1:
     source-map-support "^0.5.11"
     strip-ansi "^5.2.0"
     tempfile "^2.0.0"
-    then-fs "^2.0.0"
+    update-notifier "^4.1.0"
     uuid "^3.3.2"
     wrap-ansi "^5.1.0"
 
@@ -6343,6 +7253,11 @@ spdx-license-ids@^3.0.0:
   resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz#81c0ce8f21474756148bbb5f3bfc0f36bf15d76e"
   integrity sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==
 
+split-ca@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6"
+  integrity sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=
+
 split-string@^3.0.1, split-string@^3.0.2:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@@ -6355,6 +7270,22 @@ sprintf-js@~1.0.2:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
   integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
 
+ssh2-streams@~0.4.10:
+  version "0.4.10"
+  resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34"
+  integrity sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==
+  dependencies:
+    asn1 "~0.2.0"
+    bcrypt-pbkdf "^1.0.2"
+    streamsearch "~0.1.2"
+
+ssh2@^0.8.7:
+  version "0.8.9"
+  resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3"
+  integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==
+  dependencies:
+    ssh2-streams "~0.4.10"
+
 sshpk@^1.7.0:
   version "1.16.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
@@ -6395,11 +7326,42 @@ stdout-stream@^1.4.0:
   dependencies:
     readable-stream "^2.0.1"
 
+stream-buffers@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/stream-buffers/-/stream-buffers-3.0.2.tgz#5249005a8d5c2d00b3a32e6e0a6ea209dc4f3521"
+  integrity sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==
+
+stream-shift@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
+  integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
+
+stream-to-array@~2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353"
+  integrity sha1-u/azn19D7DC8cbq8s3VXrOzzQ1M=
+  dependencies:
+    any-promise "^1.1.0"
+
 stream-to-observable@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/stream-to-observable/-/stream-to-observable-0.1.0.tgz#45bf1d9f2d7dc09bed81f1c307c430e68b84cffe"
   integrity sha1-Rb8dny19wJvtgfHDB8Qw5ouEz/4=
 
+stream-to-promise@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/stream-to-promise/-/stream-to-promise-2.2.0.tgz#b1edb2e1c8cb11289d1b503c08d3f2aef51e650f"
+  integrity sha1-se2y4cjLESidG1A8CNPyrvUeZQ8=
+  dependencies:
+    any-promise "~1.3.0"
+    end-of-stream "~1.1.0"
+    stream-to-array "~2.3.0"
+
+streamsearch@~0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
+  integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
+
 string-hash@^1.1.0, string-hash@^1.1.1:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/string-hash/-/string-hash-1.1.3.tgz#e8aafc0ac1855b4666929ed7dd1275df5d6c811b"
@@ -6414,7 +7376,7 @@ string-width@^1.0.1, string-width@^1.0.2:
     is-fullwidth-code-point "^1.0.0"
     strip-ansi "^3.0.0"
 
-"string-width@^1.0.2 || 2", string-width@^2.0.0, string-width@^2.1.0, string-width@^2.1.1:
+"string-width@^1.0.2 || 2", string-width@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
   integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==
@@ -6431,6 +7393,15 @@ string-width@^3.0.0, string-width@^3.1.0:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^5.1.0"
 
+string-width@^4.0.0, string-width@^4.1.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
+  integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.0"
+
 string.prototype.trimend@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz#85812a6b847ac002270f5808146064c995fb6913"
@@ -6505,6 +7476,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
+  integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
+  dependencies:
+    ansi-regex "^5.0.0"
+
 strip-bom@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e"
@@ -6585,6 +7563,13 @@ supports-color@^6.1.0:
   dependencies:
     has-flag "^3.0.0"
 
+supports-color@^7.1.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.1.0.tgz#68e32591df73e25ad1c4b49108a2ec507962bfd1"
+  integrity sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==
+  dependencies:
+    has-flag "^4.0.0"
+
 svgo@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.2.0.tgz#305a8fc0f4f9710828c65039bb93d5793225ffc3"
@@ -6621,6 +7606,17 @@ tar-stream@^2.1.0:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+tar-stream@^2.1.2:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.1.3.tgz#1e2022559221b7866161660f118255e20fa79e41"
+  integrity sha512-Z9yri56Dih8IaK8gncVPx4Wqt86NDmQTSh49XLZgjWpGZL9GK9HKParS2scqHCC4w6X9Gh2jwaU45V47XTKwVA==
+  dependencies:
+    bl "^4.0.1"
+    end-of-stream "^1.4.1"
+    fs-constants "^1.0.0"
+    inherits "^2.0.3"
+    readable-stream "^3.1.1"
+
 tar@^2.0.0:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40"
@@ -6630,6 +7626,19 @@ tar@^2.0.0:
     fstream "^1.0.12"
     inherits "2"
 
+tar@^4.4.6:
+  version "4.4.13"
+  resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525"
+  integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==
+  dependencies:
+    chownr "^1.1.1"
+    fs-minipass "^1.2.5"
+    minipass "^2.8.6"
+    minizlib "^1.2.1"
+    mkdirp "^0.5.0"
+    safe-buffer "^5.1.2"
+    yallist "^3.0.3"
+
 temp-dir@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d"
@@ -6648,12 +7657,10 @@ tempfile@^2.0.0:
     temp-dir "^1.0.0"
     uuid "^3.0.1"
 
-term-size@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/term-size/-/term-size-1.2.0.tgz#458b83887f288fc56d6fffbfad262e26638efa69"
-  integrity sha1-RYuDiH8oj8Vtb/+/rSYuJmOO+mk=
-  dependencies:
-    execa "^0.7.0"
+term-size@^2.1.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753"
+  integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==
 
 terser@^3.14.1:
   version "3.17.0"
@@ -6676,6 +7683,14 @@ throttleit@^1.0.0:
   resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"
   integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=
 
+through2@^2.0.3:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
+  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
+  dependencies:
+    readable-stream "~2.3.6"
+    xtend "~4.0.1"
+
 through@^2.3.4, through@^2.3.6:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -6686,11 +7701,6 @@ thunkify@^2.1.2:
   resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d"
   integrity sha1-+qDp0jDFGsyVyhOjYawFyn4EVT0=
 
-timed-out@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
-  integrity sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=
-
 timsort@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4"
@@ -6710,6 +7720,20 @@ tmp@0.1.0, tmp@^0.1.0:
   dependencies:
     rimraf "^2.6.3"
 
+tmp@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.0.tgz#fdc6a78d2a77bf7e426ca476d8a4f82eefcf648c"
+  integrity sha512-spsb5g6EiPmteS5TcOAECU3rltCMDMp4VMU2Sb0+WttN4qGobEkMAd+dkr1cubscN08JGNDX765dPbGImbG7MQ==
+  dependencies:
+    rimraf "^3.0.0"
+
+tmp@0.2.1, tmp@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14"
+  integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==
+  dependencies:
+    rimraf "^3.0.0"
+
 to-array@0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890"
@@ -6722,6 +7746,11 @@ to-object-path@^0.3.0:
   dependencies:
     kind-of "^3.0.2"
 
+to-readable-stream@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771"
+  integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==
+
 to-regex-range@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38"
@@ -6730,6 +7759,13 @@ to-regex-range@^2.1.0:
     is-number "^3.0.0"
     repeat-string "^1.6.1"
 
+to-regex-range@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
+  integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==
+  dependencies:
+    is-number "^7.0.0"
+
 to-regex@^3.0.1, to-regex@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.2.tgz#13cfdd9b336552f30b51f33a8ae1b42a7a7599ce"
@@ -6792,10 +7828,15 @@ tslib@^1, tslib@^1.10.0, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a"
   integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==
 
-tslib@^1.11.1:
-  version "1.11.2"
-  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.11.2.tgz#9c79d83272c9a7aaf166f73915c9667ecdde3cc9"
-  integrity sha512-tTSkux6IGPnUGUd1XAZHcpu85MOkIl5zX49pO+jfsie3eP0B6pyhOlLXm3cAC6T7s+euSDDUUV+Acop5WmtkVg==
+tslib@^1.11.2, tslib@^1.13.0:
+  version "1.13.0"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043"
+  integrity sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==
+
+tslib@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.1.tgz#410eb0d113e5b6356490eec749603725b021b43e"
+  integrity sha512-SgIkNheinmEBgx1IUNirK0TUD4X9yjjBRTqqjggWCU3pUEqIk3/Uwl3yRixYKT6WjQuGiwDv4NomL3wqRCj+CQ==
 
 tunnel-agent@^0.6.0:
   version "0.6.0"
@@ -6804,6 +7845,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+tunnel@^0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
+  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
@@ -6816,6 +7862,11 @@ type-check@~0.3.2:
   dependencies:
     prelude-ls "~1.1.2"
 
+type-fest@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
+  integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
+
 type-is@~1.6.17, type-is@~1.6.18:
   version "1.6.18"
   resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
@@ -6824,15 +7875,22 @@ type-is@~1.6.17, type-is@~1.6.18:
     media-typer "0.3.0"
     mime-types "~2.1.24"
 
+typedarray-to-buffer@^3.1.5:
+  version "3.1.5"
+  resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"
+  integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==
+  dependencies:
+    is-typedarray "^1.0.0"
+
 typedarray@^0.0.6:
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
-typescript@3.8.3:
-  version "3.8.3"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.8.3.tgz#409eb8544ea0335711205869ec458ab109ee1061"
-  integrity sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==
+underscore@^1.7.0:
+  version "1.10.2"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf"
+  integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==
 
 unicode-canonical-property-names-ecmascript@^1.0.4:
   version "1.0.4"
@@ -6877,12 +7935,12 @@ uniqs@^2.0.0:
   resolved "https://registry.yarnpkg.com/uniqs/-/uniqs-2.0.0.tgz#ffede4b36b25290696e6e165d4a59edb998e6b02"
   integrity sha1-/+3ks2slKQaW5uFl1KWe25mOawI=
 
-unique-string@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-1.0.0.tgz#9e1057cca851abb93398f8b33ae187b99caec11a"
-  integrity sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=
+unique-string@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-2.0.0.tgz#39c6451f81afb2749de2b233e3f7c5e8843bd89d"
+  integrity sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==
   dependencies:
-    crypto-random-string "^1.0.0"
+    crypto-random-string "^2.0.0"
 
 universalify@^0.1.0:
   version "0.1.2"
@@ -6912,10 +7970,24 @@ untildify@3.0.3:
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-3.0.3.tgz#1e7b42b140bcfd922b22e70ca1265bfe3634c7c9"
   integrity sha512-iSk/J8efr8uPT/Z4eSUywnqyrQU7DSdMfdqK4iWEaUVVmcP5JcnpRqmVMwcwcnmI1ATFNgC5V90u09tBynNFKA==
 
-unzip-response@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97"
-  integrity sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=
+update-notifier@^4.1.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.1.tgz#895fc8562bbe666179500f9f2cebac4f26323746"
+  integrity sha512-9y+Kds0+LoLG6yN802wVXoIfxYEwh3FlZwzMwpCZp62S2i1/Jzeqb9Eeeju3NSHccGGasfGlK5/vEHbAifYRDg==
+  dependencies:
+    boxen "^4.2.0"
+    chalk "^3.0.0"
+    configstore "^5.0.1"
+    has-yarn "^2.1.0"
+    import-lazy "^2.1.0"
+    is-ci "^2.0.0"
+    is-installed-globally "^0.3.1"
+    is-npm "^4.0.0"
+    is-yarn-global "^0.3.0"
+    latest-version "^5.0.0"
+    pupa "^2.0.1"
+    semver-diff "^3.1.1"
+    xdg-basedir "^4.0.0"
 
 uri-js@^4.2.2:
   version "4.2.2"
@@ -6929,12 +8001,12 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
-url-parse-lax@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
-  integrity sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=
+url-parse-lax@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c"
+  integrity sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=
   dependencies:
-    prepend-http "^1.0.1"
+    prepend-http "^2.0.0"
 
 url@0.11.0:
   version "0.11.0"
@@ -6972,6 +8044,11 @@ uuid@^3.0.1, uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
+uuid@^8.2.0:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.0.tgz#ab738085ca22dc9a8c92725e459b1d507df5d6ea"
+  integrity sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ==
+
 validate-npm-package-license@^3.0.1:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"
@@ -7087,6 +8164,13 @@ which@1, which@^1.2.14, which@^1.2.9:
   dependencies:
     isexe "^2.0.0"
 
+which@^2.0.1:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
+  integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
+  dependencies:
+    isexe "^2.0.0"
+
 wide-align@^1.1.0:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457"
@@ -7094,12 +8178,12 @@ wide-align@^1.1.0:
   dependencies:
     string-width "^1.0.2 || 2"
 
-widest-line@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-2.0.1.tgz#7438764730ec7ef4381ce4df82fb98a53142a3fc"
-  integrity sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==
+widest-line@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca"
+  integrity sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==
   dependencies:
-    string-width "^2.1.1"
+    string-width "^4.0.0"
 
 window-size@^0.1.4:
   version "0.1.4"
@@ -7140,14 +8224,15 @@ wrappy@1:
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
 
-write-file-atomic@^2.0.0:
-  version "2.4.3"
-  resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-2.4.3.tgz#1fd2e9ae1df3e75b8d8c367443c692d4ca81f481"
-  integrity sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==
+write-file-atomic@^3.0.0:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8"
+  integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==
   dependencies:
-    graceful-fs "^4.1.11"
     imurmurhash "^0.1.4"
+    is-typedarray "^1.0.0"
     signal-exit "^3.0.2"
+    typedarray-to-buffer "^3.1.5"
 
 ws@^7.1.2:
   version "7.2.1"
@@ -7161,10 +8246,10 @@ ws@~6.1.0:
   dependencies:
     async-limiter "~1.0.0"
 
-xdg-basedir@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
-  integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
+xdg-basedir@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
+  integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
 
 xml2js@0.4.23, xml2js@^0.4.17:
   version "0.4.23"
@@ -7189,6 +8274,11 @@ xregexp@2.0.0:
   resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-2.0.0.tgz#52a63e56ca0b84a7f3a5f3d61872f126ad7a5943"
   integrity sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=
 
+xtend@~4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
+  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
+
 y18n@^3.2.0, y18n@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41"
@@ -7204,11 +8294,21 @@ yallist@^2.1.2:
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
   integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
 
-yallist@^3.0.2:
+yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
   integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
 
+yallist@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
+  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
+
+yaml@^1.9.2:
+  version "1.10.0"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e"
+  integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==
+
 yargs-parser@^15.0.0:
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-15.0.0.tgz#cdd7a97490ec836195f59f3f4dbe5ea9e8f75f08"

From 7ce40130687ee2b7e86769b918676aa45ce36d01 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2020 13:05:49 +0530
Subject: [PATCH 349/361] fix: if db_type is not defined then set default to
 'mariadb'  (bp #11346) (#11349)

(cherry picked from commit 927cd03341c33ffb82e5db373066a7ba239c1491)

Co-authored-by: Saurabh <saurabh@erpnext.com>
---
 frappe/utils/backups.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py
index a64ba8c3a7..bae341ce87 100644
--- a/frappe/utils/backups.py
+++ b/frappe/utils/backups.py
@@ -41,6 +41,9 @@ class BackupGenerator:
 		self.backup_path_db = backup_path_db
 		self.backup_path_private_files = backup_path_private_files
 
+		if not self.db_type:
+			self.db_type = 'mariadb'
+
 		if not self.db_port and self.db_type == 'mariadb':
 			self.db_port = 3306
 		elif not self.db_port and self.db_type == 'postgres':

From 737c033803caa6f6165be4c882d01050274db787 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 13:56:48 +0530
Subject: [PATCH 350/361] fix(minor): fix patch that breaks in erpnext

---
 frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py
index 1c4a571b01..60599066e6 100644
--- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py
+++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py
@@ -1,6 +1,8 @@
 import frappe
 
 def execute():
+	frappe.reload_doc('core', 'doctype', 'doctype_link')
+	frappe.reload_doc('core', 'doctype', 'doctype_action')
 	frappe.reload_doc('core', 'doctype', 'doctype')
 	frappe.model.delete_fields({
 		'DocType': ['hide_heading', 'image_view', 'read_only_onload']

From ebc961cf842885f2ba7b18320311d997670df4f9 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 14:12:30 +0530
Subject: [PATCH 351/361] fix(debug): print error before it breaks

---
 frappe/model/base_document.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index 09d303bec7..e94ce18e6b 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -299,7 +299,11 @@ class BaseDocument(object):
 		return frappe.as_json(self.as_dict())
 
 	def get_table_field_doctype(self, fieldname):
-		return self.meta.get_field(fieldname).options
+		try:
+			return self.meta.get_field(fieldname).options
+		except AttributeError:
+			print('Unable to find options for {}'.format(fieldname))
+			raise
 
 	def get_parentfield_of_doctype(self, doctype):
 		fieldname = [df.fieldname for df in self.meta.get_table_fields() if df.options==doctype]

From 5d2a1ea47a2be007b56a658dbbfe4450f3051d3a Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 14:29:08 +0530
Subject: [PATCH 352/361] fix(minor): handle importing of doctype when links
 are missing

---
 frappe/model/base_document.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index e94ce18e6b..db2b4eff85 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -302,7 +302,8 @@ class BaseDocument(object):
 		try:
 			return self.meta.get_field(fieldname).options
 		except AttributeError:
-			print('Unable to find options for {}'.format(fieldname))
+			if self.doctype == 'DocType':
+				return dict(links='DocType Link', actions='DocType Action').get(fieldname)
 			raise
 
 	def get_parentfield_of_doctype(self, doctype):

From 58e9c074c9b822a6a6ba4db6776c990128157b5c Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 14:43:50 +0530
Subject: [PATCH 353/361] fix(rename_field): skip if old fieldname does not
 exist

---
 frappe/model/utils/rename_field.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py
index 0cb974df1e..428fafd4c8 100644
--- a/frappe/model/utils/rename_field.py
+++ b/frappe/model/utils/rename_field.py
@@ -19,11 +19,16 @@ def rename_field(doctype, old_fieldname, new_fieldname):
 		print("rename_field: " + (new_fieldname) + " not found in " + doctype)
 		return
 
+	if not frappe.db.has_column(doctype, old_fieldname):
+		# never had the field?
+		return
+
 	if new_field.fieldtype in table_fields:
 		# change parentfield of table mentioned in options
 		frappe.db.sql("""update `tab%s` set parentfield=%s
 			where parentfield=%s""" % (new_field.options.split("\n")[0], "%s", "%s"),
 			(new_fieldname, old_fieldname))
+
 	elif new_field.fieldtype not in no_value_fields:
 		if meta.issingle:
 			frappe.db.sql("""update `tabSingles` set field=%s

From 394088008a782d5056ce5aeed183f1d542abe6bf Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 15:20:12 +0530
Subject: [PATCH 354/361] fix(minor): rename_field - show message before
 ignoring

---
 frappe/model/utils/rename_field.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py
index 428fafd4c8..2a734cb1dc 100644
--- a/frappe/model/utils/rename_field.py
+++ b/frappe/model/utils/rename_field.py
@@ -20,6 +20,7 @@ def rename_field(doctype, old_fieldname, new_fieldname):
 		return
 
 	if not frappe.db.has_column(doctype, old_fieldname):
+		print("rename_field: " + (old_fieldname) + " not found in table for: " + doctype)
 		# never had the field?
 		return
 

From 4fe9eb7b13e535075c72a2114d878f917cbfba86 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 15:32:00 +0530
Subject: [PATCH 355/361] fix(patch): reload report

---
 frappe/patches.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/patches.txt b/frappe/patches.txt
index afbd007abc..985911314e 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -40,7 +40,9 @@ execute:frappe.reload_doc('core', 'doctype', 'role') #2017-05-23
 execute:frappe.reload_doc('core', 'doctype', 'user') #2017-10-27
 execute:frappe.reload_doc('custom', 'doctype', 'custom_field') #2015-10-19
 execute:frappe.reload_doc('core', 'doctype', 'page') #2013-13-26
-execute:frappe.reload_doc('core', 'doctype', 'report') #2014-06-03
+execute:frappe.reload_doc('core', 'doctype', 'report_column')
+execute:frappe.reload_doc('core', 'doctype', 'report_filter')
+execute:frappe.reload_doc('core', 'doctype', 'report') #2020-08-25
 execute:frappe.reload_doc('core', 'doctype', 'translation') #2016-03-03
 execute:frappe.reload_doc('email', 'doctype', 'email_alert') #2014-07-15
 execute:frappe.reload_doc('desk', 'doctype', 'todo') #2014-12-31-1

From 938b87ee466e9d3dd81494ee26c31e64fbbb7a90 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Tue, 25 Aug 2020 15:44:08 +0530
Subject: [PATCH 356/361] fix(minor): rename field, ignore for single doctype

---
 frappe/model/utils/rename_field.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py
index 2a734cb1dc..778f623092 100644
--- a/frappe/model/utils/rename_field.py
+++ b/frappe/model/utils/rename_field.py
@@ -19,7 +19,7 @@ def rename_field(doctype, old_fieldname, new_fieldname):
 		print("rename_field: " + (new_fieldname) + " not found in " + doctype)
 		return
 
-	if not frappe.db.has_column(doctype, old_fieldname):
+	if not meta.issingle and not frappe.db.has_column(doctype, old_fieldname):
 		print("rename_field: " + (old_fieldname) + " not found in table for: " + doctype)
 		# never had the field?
 		return

From 8b35a5f5bf17e52f72a951176b35011751365859 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Tue, 25 Aug 2020 15:57:40 +0530
Subject: [PATCH 357/361] fix: Update yarn.lock

---
 yarn.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn.lock b/yarn.lock
index 9e85a639aa..c3808f680a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7046,7 +7046,7 @@ snyk-try-require@1.3.1, snyk-try-require@^1.1.1, snyk-try-require@^1.3.1:
     lru-cache "^4.0.0"
     then-fs "^2.0.0"
 
-snyk@1.382.0, snyk@^1.382.0:
+snyk@^1.382.0:
   version "1.382.0"
   resolved "https://registry.yarnpkg.com/snyk/-/snyk-1.382.0.tgz#5625931dd066fca406f2d35663eb7280d57565bd"
   integrity sha512-lZsuJ6HUVNO7ZYDQm6B2gPdcXiRtd8bCQUre8xYMZu1VakKwrCCNsJbxo91z3gK5aYqempjmTe7+qH2CksXZ+Q==

From 5a6bacf43f9427ce524f0c546c47ffc4da7c3ea7 Mon Sep 17 00:00:00 2001
From: Shivam Mishra <scm.mymail@gmail.com>
Date: Tue, 25 Aug 2020 18:31:54 +0530
Subject: [PATCH 358/361] refactor: use this.doc if this.frm is not available

---
 frappe/public/js/frappe/form/layout.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js
index d6106255a0..1348203bc4 100644
--- a/frappe/public/js/frappe/form/layout.js
+++ b/frappe/public/js/frappe/form/layout.js
@@ -525,7 +525,7 @@ frappe.ui.form.Layout = Class.extend({
 			return;
 		}
 
-		var parent = this.frm ? this.frm.doc : null;
+		var parent = this.frm ? this.frm.doc : this.doc || null;
 
 		if(typeof(expression) === 'boolean') {
 			out = expression;

From 76292f14e9935ebade2ea3d4dae325236fdb55dd Mon Sep 17 00:00:00 2001
From: Faris Ansari <netchampfaris@users.noreply.github.com>
Date: Wed, 26 Aug 2020 12:28:05 +0530
Subject: [PATCH 359/361] Update frappe-framework-logo.svg

---
 .github/frappe-framework-logo.svg | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/frappe-framework-logo.svg b/.github/frappe-framework-logo.svg
index ba04ebf264..12dc55b736 100644
--- a/.github/frappe-framework-logo.svg
+++ b/.github/frappe-framework-logo.svg
@@ -1,4 +1,5 @@
-<svg viewBox="0 0 1082 125" fill="none" xmlns="http://www.w3.org/2000/svg">
-	<path d="M1 56.284v67.508h26.904V79.368l47.902-.04V56.248L1 56.284zM82.994 1H1.034v23.043h81.96V1z" fill="#3C88F7"/>
-	<path d="M216.735 75.269V53.233h-52.897V22.505h61.849V.469H138v122.509h25.838v-47.71h52.897zM285.477 59.096c2.645 0 5.29.404 7.731 1.415l3.052-26.281c-2.441-.607-4.883-1.01-7.934-1.01-11.8.201-19.125 6.67-25.635 22.439.61-3.639.814-7.48.814-11.321v-9.3h-23.193v87.94h25.227V72.64c5.697-12.13 15.056-13.544 19.938-13.544zM380.301 125c4.476 0 9.359-1.011 13.428-4.043l-2.442-17.184c-.813.404-1.627.607-2.645.607-2.644 0-4.272-2.426-4.272-6.672v-32.75c0-23.652-16.276-31.739-39.266-31.739-23.193 0-38.655 10.715-41.3 30.728l22.583 1.213c1.22-8.49 7.12-12.533 17.09-12.533 9.358 0 15.869 4.65 15.869 13.949v5.458h-16.683c-26.245 0-41.097 8.895-41.097 27.898 0 18.397 14.648 25.068 31.128 25.068 15.462 0 24.007-6.671 28.89-15.364C363.211 121.563 372.163 125 380.301 125zm-20.752-39.22c0 14.759-10.579 21.228-19.735 21.228-7.324 0-12.41-3.033-12.41-9.704 0-8.086 6.917-11.523 18.107-11.523h14.038zM449.522 55.255c9.562 0 15.055 5.862 15.055 19.003v48.72h25.228V69.001c3.458-8.895 9.358-13.746 16.683-13.746 10.172 0 15.055 5.862 15.055 19.003v48.72h25.228v-54.38c0-24.462-11.8-35.379-32.145-35.379-11.8 0-22.38 5.054-29.297 15.97-4.476-11.118-13.021-15.97-26.856-15.97-13.02 0-22.379 5.66-28.076 17.184.204-2.022.204-3.841.204-5.863V35.04h-23.193v87.939h25.227V69.406c3.459-9.097 8.952-14.151 16.887-14.151zM606.579 125c22.583 0 39.062-8.491 44.148-29.111l-24.21-2.224c-2.442 7.48-8.749 11.725-18.311 11.725-12.614 0-19.531-7.277-20.548-21.429l64.087-.202c.203-2.224.203-4.245.203-6.469 0-28.505-15.462-44.07-43.131-44.07-29.297 0-47.201 18.8-47.201 46.496 0 28.909 17.09 45.284 44.963 45.284zm1.831-72.98c10.783 0 17.09 6.065 17.903 17.184h-37.841c2.441-11.928 9.765-17.184 19.938-17.184zM751.095 122.978h34.18l24.414-87.94H782.63l-15.056 63.479h-1.831l-17.089-63.478h-28.89l-17.497 63.478h-2.035l-15.055-63.478h-27.669l24.617 87.939h34.383l16.48-61.456h1.831l16.276 61.456zM814.935 79.918c0 28.303 16.276 45.082 45.98 45.082 31.535 0 48.828-17.992 48.828-46.093 0-27.898-16.073-45.688-45.573-45.688-31.738 0-49.235 18.599-49.235 46.7zm26.652-.606c0-16.173 8.545-26.079 21.566-26.079 12.614 0 19.938 9.906 19.938 26.079 0 15.768-8.341 25.876-20.955 25.876-12.818 0-20.549-10.108-20.549-25.876zM971.128 59.096c2.645 0 5.289.404 7.731 1.415l3.052-26.281c-2.442-.607-4.883-1.01-7.935-1.01-11.8.201-19.124 6.67-25.635 22.439.611-3.639.814-7.48.814-11.321v-9.3h-23.193v87.94h25.228V72.64c5.696-12.13 15.055-13.544 19.938-13.544zM1070.4 125c3.46 0 7.33-.404 10.79-1.819l-1.83-21.227c-.82.202-1.63.404-3.26.404-4.27 0-7.12-2.224-11.39-7.682l-16.48-21.833L1082 35.039h-27.67l-34.18 36.793h-.61V1.5h-25.227v121.478h25.227v-20.62l10.78-12.534 14.45 19.61c8.95 12.938 15.87 15.566 25.63 15.566z" fill="#000"/>
+<svg width="406" height="82" viewBox="0 0 406 82" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M33.4319 16H0V24.794H33.4319V16Z" fill="#0089FF"/>
+    <path d="M0 38.4357V66H10.671V47.2445H31.2149V38.4357H0Z" fill="#0089FF"/>
+    <path d="M54.8778 66V16.5455H86.5597V24.0554H63.8366V37.4815H84.3864V44.9915H63.8366V66H54.8778ZM92.104 66V28.9091H100.58V35.0909H100.966C101.642 32.9337 102.801 31.2756 104.443 30.1165C106.102 28.9574 107.993 28.3778 110.118 28.3778C110.601 28.3778 111.14 28.402 111.736 28.4503C112.348 28.4825 112.855 28.5388 113.257 28.6193V36.6605C112.887 36.5317 112.3 36.419 111.495 36.3224C110.706 36.2098 109.941 36.1534 109.201 36.1534C106.802 36.1534 104.806 36.91 103.212 38.4233C101.634 39.9205 100.845 41.8442 100.845 44.1946V66H92.104ZM127.308 66.7486C123.766 66.7486 120.812 65.7988 118.446 63.8991C116.079 61.9995 114.896 59.2306 114.896 55.5923C114.896 52.8073 115.564 50.6501 116.9 49.1207C118.252 47.5753 119.999 46.4564 122.14 45.7642C124.297 45.0559 126.575 44.5729 128.974 44.3153C132.242 43.9612 134.584 43.6312 136.001 43.3253C137.434 43.0194 138.15 42.2386 138.15 40.983V40.8381C138.15 39.0189 137.611 37.6103 136.532 36.6122C135.454 35.6141 133.9 35.1151 131.872 35.1151C129.731 35.1151 128.032 35.5819 126.776 36.5156C125.537 37.4493 124.7 38.5521 124.265 39.8239L116.103 38.6648C117.069 35.2841 118.953 32.7325 121.754 31.0099C124.571 29.2874 127.928 28.4261 131.823 28.4261C133.594 28.4261 135.365 28.6354 137.136 29.054C138.907 29.4725 140.525 30.1648 141.99 31.1307C143.471 32.0966 144.654 33.4006 145.539 35.0426C146.441 36.6847 146.892 38.7292 146.892 41.1761V66H138.488V60.9048H138.198C137.41 62.4664 136.122 63.8348 134.335 65.0099C132.564 66.169 130.222 66.7486 127.308 66.7486ZM129.578 60.3253C132.218 60.3253 134.311 59.5687 135.856 58.0554C137.401 56.5421 138.174 54.7552 138.174 52.6946V48.3239C137.804 48.6297 137.12 48.9034 136.122 49.1449C135.14 49.3703 134.093 49.5634 132.982 49.7244C131.872 49.8854 130.93 50.0223 130.157 50.1349C128.209 50.4086 126.583 50.9479 125.279 51.7528C123.975 52.5417 123.323 53.7732 123.323 55.4474C123.323 57.0573 123.911 58.2727 125.086 59.0938C126.261 59.9148 127.758 60.3253 129.578 60.3253ZM153.981 66V28.9091H162.336V35.2116H162.771C163.544 33.1027 164.824 31.4446 166.611 30.2372C168.398 29.0298 170.531 28.4261 173.01 28.4261C175.505 28.4261 177.622 29.0298 179.361 30.2372C181.099 31.4446 182.323 33.1027 183.031 35.2116H183.417C184.238 33.151 185.623 31.509 187.571 30.2855C189.535 29.0459 191.861 28.4261 194.55 28.4261C197.962 28.4261 200.747 29.5047 202.905 31.6619C205.062 33.8191 206.14 36.9664 206.14 41.1037V66H197.375V42.456C197.375 40.1539 196.763 38.4716 195.54 37.4091C194.316 36.3305 192.819 35.7912 191.048 35.7912C188.939 35.7912 187.289 36.4512 186.098 37.7713C184.923 39.0753 184.335 40.7737 184.335 42.8665V66H175.763V42.0938C175.763 40.178 175.183 38.6487 174.024 37.5057C172.881 36.3627 171.384 35.7912 169.532 35.7912C167.649 35.7912 166.039 36.4995 164.703 37.9162C163.383 39.3168 162.723 41.16 162.723 43.446V66H153.981ZM229.729 66.7244C226.01 66.7244 222.799 65.9437 220.094 64.3821C217.406 62.8205 215.337 60.6151 213.888 57.7656C212.439 54.9162 211.715 51.5516 211.715 47.6719C211.715 43.8565 212.439 40.508 213.888 37.6264C215.337 34.7287 217.374 32.4749 219.998 30.8651C222.638 29.2391 225.729 28.4261 229.27 28.4261C232.313 28.4261 235.122 29.0862 237.698 30.4062C240.274 31.7102 242.334 33.7467 243.88 36.5156C245.441 39.2685 246.222 42.8182 246.222 47.1648V49.8452H220.384C220.432 53.0166 221.31 55.4877 223.016 57.2585C224.739 59.0294 227.017 59.9148 229.85 59.9148C231.733 59.9148 233.359 59.5123 234.728 58.7074C236.112 57.8864 237.102 56.6951 237.698 55.1335L245.86 56.0511C245.087 59.2708 243.284 61.8546 240.451 63.8026C237.633 65.7505 234.06 66.7244 229.729 66.7244ZM220.408 43.9531H237.843C237.827 41.4257 237.038 39.3409 235.476 37.6989C233.931 36.0568 231.902 35.2358 229.391 35.2358C226.783 35.2358 224.666 36.0971 223.04 37.8196C221.414 39.5421 220.537 41.5866 220.408 43.9531ZM259.373 66L248.893 28.9091H257.803L264.323 54.9886H264.661L271.326 28.9091H280.14L286.805 54.8438H287.167L293.59 28.9091H302.525L292.021 66H282.917L275.962 40.9347H275.455L268.501 66H259.373ZM322.111 66.7244C318.489 66.7244 315.35 65.9276 312.693 64.3338C310.037 62.7401 307.977 60.5104 306.512 57.6449C305.063 54.7633 304.338 51.4148 304.338 47.5994C304.338 43.768 305.063 40.4195 306.512 37.554C307.977 34.6723 310.037 32.4347 312.693 30.8409C315.35 29.2311 318.489 28.4261 322.111 28.4261C325.733 28.4261 328.872 29.2311 331.529 30.8409C334.201 32.4347 336.262 34.6723 337.711 37.554C339.159 40.4195 339.884 43.768 339.884 47.5994C339.884 51.4148 339.159 54.7633 337.711 57.6449C336.262 60.5104 334.201 62.7401 331.529 64.3338C328.872 65.9276 325.733 66.7244 322.111 66.7244ZM322.159 59.7216C324.139 59.7216 325.782 59.1823 327.086 58.1037C328.406 57.0251 329.388 55.5682 330.032 53.733C330.692 51.8816 331.022 49.8291 331.022 47.5753C331.022 45.3054 330.692 43.2528 330.032 41.4176C329.388 39.5663 328.406 38.0933 327.086 36.9986C325.782 35.9039 324.139 35.3565 322.159 35.3565C320.163 35.3565 318.497 35.9039 317.161 36.9986C315.841 38.0933 314.851 39.5663 314.191 41.4176C313.531 43.2528 313.201 45.3054 313.201 47.5753C313.201 49.8291 313.531 51.8816 314.191 53.733C314.851 55.5682 315.841 57.0251 317.161 58.1037C318.497 59.1823 320.163 59.7216 322.159 59.7216ZM345.603 66V28.9091H354.079V35.0909H354.465C355.142 32.9337 356.301 31.2756 357.943 30.1165C359.601 28.9574 361.492 28.3778 363.617 28.3778C364.1 28.3778 364.64 28.402 365.235 28.4503C365.847 28.4825 366.354 28.5388 366.757 28.6193V36.6605C366.386 36.5317 365.799 36.419 364.994 36.3224C364.205 36.2098 363.44 36.1534 362.7 36.1534C360.301 36.1534 358.305 36.91 356.711 38.4233C355.134 39.9205 354.345 41.8442 354.345 44.1946V66H345.603ZM370.864 66V16.5455H379.606V43.8082H380.209L393.539 28.9091H403.753L389.385 44.919L404.599 66H394.143L382.793 50.1349L379.606 53.5398V66H370.864Z" fill="black"/>
 </svg>

From f8dcc60bb6cce9f81c28559dfe9e5d513dc5b2dd Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 26 Aug 2020 17:32:58 +0530
Subject: [PATCH 360/361] chore: frappe.io => frappeframework.com

---
 .github/ISSUE_TEMPLATE/bug_report.md                  | 2 +-
 .github/ISSUE_TEMPLATE/feature_request.md             | 2 +-
 .github/ISSUE_TEMPLATE/question-about-using-frappe.md | 2 +-
 .github/frappe_linter/translation.py                  | 2 +-
 README.md                                             | 2 +-
 frappe/public/js/frappe/ui/toolbar/about.js           | 2 +-
 frappe/templates/base.html                            | 2 +-
 frappe/templates/includes/footer/footer_powered.html  | 2 +-
 frappe/website/doctype/web_form/web_form.json         | 2 +-
 package.json                                          | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index a1c4b0a633..e4cae95711 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -9,7 +9,7 @@ Welcome to the Frappe Framework issue tracker! Before creating an issue, please
 
 1. This tracker should only be used to report bugs and request features / enhancements to Frappe
     - For questions and general support, use https://stackoverflow.com/questions/tagged/frappe
-    - For documentation issues, refer to https://frappe.io/docs/user/en or the developer cheetsheet https://github.com/frappe/frappe/wiki/Developer-Cheatsheet
+    - For documentation issues, refer to https://frappeframework.com/docs/user/en or the developer cheetsheet https://github.com/frappe/frappe/wiki/Developer-Cheatsheet
 2. Use the search function before creating a new issue. Duplicates will be closed and directed to
    the original discussion.
 3. When making a bug report, make sure you provide all required information. The easier it is for
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index ed5e2169a3..6a37517f7a 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -9,7 +9,7 @@ Welcome to the Frappe Framework issue tracker! Before creating an issue, please
 
 1. This tracker should only be used to report bugs and request features / enhancements to Frappe
     - For questions and general support, refer to https://stackoverflow.com/questions/tagged/frappe
-    - For documentation issues, use https://frappe.io/docs/user/en or the developer cheetsheet https://github.com/frappe/frappe/wiki/Developer-Cheatsheet
+    - For documentation issues, use https://frappeframework.com/docs/user/en or the developer cheetsheet https://frappeframework.com/docs/user/en/bench/resources/bench-commands-cheatsheet
 2. Use the search function before creating a new issue. Duplicates will be closed and directed to
    the original discussion.
 3. When making a feature request, make sure to be as verbose as possible. The better you convey your message, the     greater the drive to make it happen.
diff --git a/.github/ISSUE_TEMPLATE/question-about-using-frappe.md b/.github/ISSUE_TEMPLATE/question-about-using-frappe.md
index 43d987ee2c..ab5d023e1c 100644
--- a/.github/ISSUE_TEMPLATE/question-about-using-frappe.md
+++ b/.github/ISSUE_TEMPLATE/question-about-using-frappe.md
@@ -12,7 +12,7 @@ for questions about using `ERPNext`: https://discuss.erpnext.com
 
 for questions about using `bench`, probably the best place to start is the [bench repo](https://github.com/frappe/bench)
 
-For documentation issues, use the [Frappe Framework Documentation](https://frappe.io/docs/user/en) or the [developer cheetsheet](https://github.com/frappe/frappe/wiki/Developer-Cheatsheet)
+For documentation issues, use the [Frappe Framework Documentation](https://frappeframework.com/docs) or the [developer cheetsheet](https://github.com/frappe/frappe/wiki/Developer-Cheatsheet)
 
 For a slightly outdated yet informative developer guide: https://www.youtube.com/playlist?list=PL3lFfCEoMxvzHtsZHFJ4T3n5yMM3nGJ1W
 
diff --git a/.github/frappe_linter/translation.py b/.github/frappe_linter/translation.py
index bb81e848f1..d9fc98c76e 100644
--- a/.github/frappe_linter/translation.py
+++ b/.github/frappe_linter/translation.py
@@ -22,7 +22,7 @@ for _file in files:
 						print(f'A syntax error has been discovered at line number: {num}')
 						print(f'Syntax error occurred with: {line}')
 if errors_encounter > 0:
-	print('You can visit "https://frappe.io/docs/user/en/translations" to resolve this error.')
+	print('You can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
 	assert 1+1 == 3
 else:
 	print('Good To Go!')
diff --git a/README.md b/README.md
index 1f59376f48..57d4ca243d 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@
     <a href="https://travis-ci.com/frappe/frappe">
         <img src="https://travis-ci.com/frappe/frappe.svg?branch=develop">
     </a>
-    <a href='https://frappe.io/docs'>
+    <a href='https://frappeframework.com/docs'>
         <img src='https://img.shields.io/badge/docs-📖-7575FF.svg?style=flat-square'/>
     </a>
 	<a href='https://www.codetriage.com/frappe/frappe'>
diff --git a/frappe/public/js/frappe/ui/toolbar/about.js b/frappe/public/js/frappe/ui/toolbar/about.js
index 87462916c2..47dc7ee851 100644
--- a/frappe/public/js/frappe/ui/toolbar/about.js
+++ b/frappe/public/js/frappe/ui/toolbar/about.js
@@ -6,7 +6,7 @@ frappe.ui.misc.about = function() {
 		$(d.body).html(repl("<div>\
 		<p>"+__("Open Source Applications for the Web")+"</p>  \
 		<p><i class='fa fa-globe fa-fw'></i>\
-			Website: <a href='https://frappe.io' target='_blank'>https://frappe.io</a></p>\
+			Website: <a href='https://frappeframework.com' target='_blank'>https://frappeframework.com</a></p>\
 		<p><i class='fa fa-github fa-fw'></i>\
 			Source: <a href='https://github.com/frappe' target='_blank'>https://github.com/frappe</a></p>\
 		<p><i class='fa fa-linkedin fa-fw'></i>\
diff --git a/frappe/templates/base.html b/frappe/templates/base.html
index 8c843a44a4..cf55a29b54 100644
--- a/frappe/templates/base.html
+++ b/frappe/templates/base.html
@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<!-- Built on Frappe. https://frappe.io/ -->
+<!-- Built on Frappe. https://frappeframework.com/ -->
 <html lang="en">
 <head>
 	<meta charset="utf-8">
diff --git a/frappe/templates/includes/footer/footer_powered.html b/frappe/templates/includes/footer/footer_powered.html
index 8d4e6e0b16..d71c537fb0 100644
--- a/frappe/templates/includes/footer/footer_powered.html
+++ b/frappe/templates/includes/footer/footer_powered.html
@@ -1 +1 @@
-<a href="https://frappe.io?source=website_footer" target="_blank" class="text-muted">Built on Frappe</a>
\ No newline at end of file
+<a href="https://frappeframework.com?source=website_footer" target="_blank" class="text-muted">Built on Frappe</a>
\ No newline at end of file
diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json
index 77dbbb9734..9199183a65 100644
--- a/frappe/website/doctype/web_form/web_form.json
+++ b/frappe/website/doctype/web_form/web_form.json
@@ -209,7 +209,7 @@
    "label": "Client Script"
   },
   {
-   "description": "For help see <a href=\"https://frappe.io/docs/user/en/guides/portal-development/web-forms\" target=\"_blank\">Client Script API and Examples</a>",
+   "description": "For help see <a href=\"https://frappeframework.com/docs/user/en/guides/portal-development/web-forms\" target=\"_blank\">Client Script API and Examples</a>",
    "fieldname": "client_script",
    "fieldtype": "Code",
    "label": "Client Script"
diff --git a/package.json b/package.json
index fe2ef2a694..f893d03ad3 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,7 @@
   "bugs": {
     "url": "https://github.com/frappe/frappe/issues"
   },
-  "homepage": "https://frappe.io",
+  "homepage": "https://frappeframework.com",
   "dependencies": {
     "ace-builds": "^1.4.8",
     "air-datepicker": "http://github.com/frappe/air-datepicker",

From ce161b37f8883437db20aefa4710157b0c9ab656 Mon Sep 17 00:00:00 2001
From: Afshan <afshan13k@gmail.com>
Date: Wed, 26 Aug 2020 22:32:46 +0530
Subject: [PATCH 361/361] fix: contact not showing in form

---
 frappe/contacts/doctype/contact/contact.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py
index f82946dc5e..987ba7d3d6 100644
--- a/frappe/contacts/doctype/contact/contact.py
+++ b/frappe/contacts/doctype/contact/contact.py
@@ -188,7 +188,7 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters):
 	from frappe.desk.reportview import get_match_cond
 
 	if not frappe.get_meta("Contact").get_field(searchfield)\
-		or searchfield not in frappe.db.DEFAULT_COLUMNS:
+		and searchfield not in frappe.db.DEFAULT_COLUMNS:
 		return []
 
 	link_doctype = filters.pop('link_doctype')