Merge branch 'develop' of https://github.com/frappe/frappe into rebrand-ui

.snyk

@@ -1,5 +1,5 @@
 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.13.5
+version: v1.14.1
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
   SNYK-JS-AWESOMPLETE-174474:
@@ -22,3 +22,44 @@ patch:
   SNYK-JS-LODASH-450202:
     - frappe-datatable > lodash:
         patched: '2020-01-31T01:33:09.889Z'
+  SNYK-JS-LODASH-567746:
+    - frappe-datatable > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - quagga > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - tailwindcss > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - '@tailwindcss/ui > @tailwindcss/custom-forms > lodash':
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/dep-graph > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > inquirer > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-config > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-mvn-plugin > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-nodejs-lockfile-parser > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-nuget-plugin > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-go-plugin > graphlib > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-nodejs-lockfile-parser > graphlib > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-nuget-plugin > dotnet-deps-parser > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > snyk-php-plugin > @snyk/composer-lockfile-parser > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/ruby-semver > lodash:
+        patched: '2020-04-30T23:02:32.330Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-04-30T23:02:32.330Z'

.travis.yml

@@ -102,7 +102,13 @@ install:
   - if [ $TYPE == "server" ]; then sed -i 's/socketio:/# socketio:/g' Procfile; fi
   - if [ $TYPE == "server" ]; then sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile; fi
 
-  - bench setup requirements --node
+  - if [ $TYPE == "ui" ]; then bench setup requirements --node; fi
+
+  # install node-sass which is required for website theme test
+  - cd ./apps/frappe
+  - yarn add node-sass@4.13.1
+  - cd ../..
+
   - bench start &
   - bench --site test_site reinstall --yes
   - bench --site test_site_producer reinstall --yes

cypress.json

@@ -2,6 +2,6 @@
   "baseUrl": "http://test_site_ui:8000",
   "projectId": "92odwv",
   "adminPassword": "admin",
-  "defaultCommandTimeout": 10000,
+  "defaultCommandTimeout": 20000,
   "pageLoadTimeout": 15000
 }

cypress/integration/control_link.js

@@ -1,7 +1,11 @@
 context('Control Link', () => {
-	beforeEach(() => {
+	before(() => {
 		cy.login();
 		cy.visit('/desk#workspace/Website');
+	});
+
+	beforeEach(() => {
+		cy.visit('/desk#workspace/Website');
 		cy.create_records({
 			doctype: 'ToDo',
 			description: 'this is a test todo for link'
@@ -30,7 +34,7 @@ context('Control Link', () => {
 
 		cy.get('.frappe-control[data-fieldname=link] input').focus().as('input');
 		cy.wait('@search_link');
-		cy.get('@input').type('todo for link');
+		cy.get('@input').type('todo for link', { delay: 200 });
 		cy.wait('@search_link');
 		cy.get('.frappe-control[data-fieldname=link] ul').should('be.visible');
 		cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 });

cypress/integration/list_view_settings.js

@@ -9,7 +9,9 @@ context('List View Settings', () => {
 		cy.get('.sidebar-stat').should('contain', "Tags");
 	});
 	it('disable count and sidebar stats then verify', () => {
+		cy.wait(300);
 		cy.visit('/desk#List/DocType/List');
+		cy.wait(300);
 		cy.get('.list-count').should('contain', "20 of");
 		cy.get('button').contains('Menu').click();
 		cy.get('.dropdown-menu li').filter(':visible').contains('Settings').click();

cypress/integration/login.js

@@ -21,6 +21,15 @@ context('Login', () => {
 		cy.location('pathname').should('eq', '/login');
 	});
 
+	it('shows invalid login if incorrect credentials', () => {
+		cy.get('#login_email').type('Administrator');
+		cy.get('#login_password').type('qwer');
+
+		cy.get('.btn-login').click();
+		cy.get('.page-card-head').contains('Invalid Login. Try again.');
+		cy.location('pathname').should('eq', '/login');
+	});
+
 	it('logs in using correct credentials', () => {
 		cy.get('#login_email').type('Administrator');
 		cy.get('#login_password').type(Cypress.config('adminPassword'));
@@ -30,12 +39,30 @@ context('Login', () => {
 		cy.window().its('frappe.session.user').should('eq', 'Administrator');
 	});
 
-	it('shows invalid login if incorrect credentials', () => {
+	it('check redirect after login', () => {
+
+		// mock for OAuth 2.0 client_id, redirect_uri, scope and state
+		const payload = new URLSearchParams({
+			uuid: '6fed1519-cfd8-4a2d-84a6-9a1799c7c741',
+			encoded_string: 'hello all',
+			encoded_url: 'http://test.localhost/callback',
+			base64_string: 'aGVsbG8gYWxs'
+		});
+
+		cy.request('/api/method/logout');
+
+		// redirect-to /me page with params to mock OAuth 2.0 like request
+		cy.visit(
+			'/login?redirect-to=/me?' +
+				encodeURIComponent(payload.toString().replace("+", " "))
+		);
+
 		cy.get('#login_email').type('Administrator');
-		cy.get('#login_password').type('qwer');
+		cy.get('#login_password').type(Cypress.config('adminPassword'));
 
 		cy.get('.btn-login').click();
-		cy.get('.page-card-head').contains('Invalid Login. Try again.');
-		cy.location('pathname').should('eq', '/login');
+
+		// verify redirected location and url params after login
+		cy.url().should('include', '/me?' + payload.toString().replace('+', '%20'));
 	});
 });

cypress/integration/relative_filters.js

@@ -1,7 +1,6 @@
 context('Relative Timeframe', () => {
 	beforeEach(() => {
 		cy.login();
-		cy.visit('/desk#workspace/Website');
 	});
 	before(() => {
 		cy.login();

frappe/__init__.py

@@ -345,7 +345,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None,
 			style="margin: 0;">{}</table>'''.format(table_rows)
 
 	if flags.print_messages and out.message:
-		print("Message: " + repr(out.message).encode("utf-8"))
+		print(f"Message: {repr(out.message).encode('utf-8')}")
 
 	if title:
 		out.title = title

frappe/app.py

@@ -26,6 +26,7 @@ from frappe.core.doctype.comment.comment import update_comments_in_parent_after_
 from frappe import _
 import frappe.recorder
 import frappe.monitor
+import frappe.rate_limiter
 
 local_manager = LocalManager([frappe.local])
 
@@ -54,6 +55,7 @@ def application(request):
 
 		frappe.recorder.record()
 		frappe.monitor.start()
+		frappe.rate_limiter.apply()
 
 		if frappe.local.form_dict.cmd:
 			response = frappe.handler.handle()
@@ -93,9 +95,13 @@ def application(request):
 		if response and hasattr(frappe.local, 'cookie_manager'):
 			frappe.local.cookie_manager.flush_cookies(response=response)
 
+		frappe.rate_limiter.update()
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
+		if response and hasattr(frappe.local, 'rate_limiter'):
+			response.headers.extend(frappe.local.rate_limiter.headers())
+
 		frappe.destroy()
 
 	return response
@@ -171,6 +177,9 @@ def handle_exception(e):
 			http_status_code=http_status_code,  indicator_color='red')
 		return_as_message = True
 
+	elif http_status_code == 429:
+		response = frappe.rate_limiter.respond()
+
 	else:
 		traceback = "<pre>" + sanitize_html(frappe.get_traceback()) + "</pre>"
 		if frappe.local.flags.disable_traceback:

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -44,7 +44,7 @@ class AssignmentRule(Document):
 		user = self.get_user()
 
 		assign_to.add(dict(
-			assign_to = user,
+			assign_to = [user],
 			doctype = doc.get('doctype'),
 			name = doc.get('name'),
 			description = frappe.render_template(self.description, doc),

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -299,17 +299,20 @@ def get_next_schedule_date(schedule_date, frequency, start_date, repeat_on_day=N
 		next_date = get_next_date(start_date, month_count)
 	else:
 		days = 7 if frequency == 'Weekly' else 1
-		next_date = add_days(start_date, days)
+		next_date = add_days(schedule_date, days)
 
 	# next schedule date should be after or on current date
 	if not for_full_schedule:
 		while getdate(next_date) < getdate(today()):
 			if month_count:
 				month_count += month_map.get(frequency)
-			next_date = get_next_date(start_date, month_count, day_count)
+				next_date = get_next_date(start_date, month_count, day_count)
+			elif days:
+				next_date = add_days(next_date, days)
 
 	return next_date
 
+
 def get_next_date(dt, mcount, day=None):
 	dt = getdate(dt)
 	dt += relativedelta(months=mcount, day=day)

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -99,13 +99,18 @@ class TestAutoRepeat(unittest.TestCase):
 	def test_next_schedule_date(self):
 		current_date = getdate(today())
 		todo = frappe.get_doc(
-			dict(doctype='ToDo', description='test next schedule date todo', assigned_by='Administrator')).insert()
+			dict(doctype='ToDo', description='test next schedule date for monthly', assigned_by='Administrator')).insert()
 		doc = make_auto_repeat(frequency='Monthly',	reference_document=todo.name, start_date=add_months(today(), -2))
 
 		# next_schedule_date is set as on or after current date
 		# it should not be a previous month's date
 		self.assertTrue((doc.next_schedule_date >= current_date))
 
+		todo = frappe.get_doc(
+			dict(doctype='ToDo', description='test next schedule date for daily', assigned_by='Administrator')).insert()
+		doc = make_auto_repeat(frequency='Daily', reference_document=todo.name, start_date=add_days(today(), -2))
+		self.assertEqual(getdate(doc.next_schedule_date), current_date)
+
 
 def make_auto_repeat(**args):
 	args = frappe._dict(args)

frappe/boot.py

@@ -106,14 +106,22 @@ def load_desktop_data(bootinfo):
 	from frappe.desk.desktop import get_desk_sidebar_items
 	bootinfo.allowed_modules = get_modules_from_all_apps_for_user()
 	bootinfo.allowed_workspaces = get_desk_sidebar_items(True)
+	bootinfo.dashboards = frappe.get_all("Dashboard")
 
-def get_allowed_pages():
-	return get_user_pages_or_reports('Page')
+def get_allowed_pages(cache=False):
+	return get_user_pages_or_reports('Page', cache=cache)
 
-def get_allowed_reports():
-	return get_user_pages_or_reports('Report')
+def get_allowed_reports(cache=False):
+	return get_user_pages_or_reports('Report', cache=cache)
+
+def get_user_pages_or_reports(parent, cache=False):
+	_cache = frappe.cache()
+
+	if cache:
+		has_role = _cache.get_value('has_role:' + parent, user=frappe.session.user)
+		if has_role:
+			return has_role
 
-def get_user_pages_or_reports(parent):
 	roles = frappe.get_roles()
 	has_role = {}
 	column = get_column(parent)
@@ -184,6 +192,8 @@ def get_user_pages_or_reports(parent):
 		for report in reports:
 			has_role[report.name]["report_type"] = report.report_type
 
+	# Expire every six hours
+	_cache.set_value('has_role:' + parent, has_role, frappe.session.user, 21600)
 	return has_role
 
 def get_column(doctype):

frappe/cache_manager.py

@@ -20,7 +20,8 @@ global_cache_keys = ("app_hooks", "installed_apps",
 
 user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
 		"defaults", "user_permissions", "home_page", "linked_with",
-		"desktop_icons", 'portal_menu_items')
+		"desktop_icons", 'portal_menu_items', 'user_perm_can_read',
+		"has_role:Page", "has_role:Report")
 
 doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified",
 		"linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map')

frappe/commands/site.py

@@ -256,6 +256,15 @@ def migrate(context, rebuild_website=False, skip_failing=False):
 	print("Compiling Python Files...")
 	compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*'))
 
+@click.command('migrate-to')
+@click.argument('frappe_provider')
+@pass_context
+def migrate_to(context, frappe_provider):
+	"Migrates site to the specified provider"
+	from frappe.integrations.frappe_providers import migrate_to
+	for site in context.sites:
+		migrate_to(site, frappe_provider)
+
 @click.command('run-patch')
 @click.argument('module')
 @pass_context
@@ -317,23 +326,25 @@ def use(site, sites_path='.'):
 	if os.path.exists(os.path.join(sites_path, site)):
 		with open(os.path.join(sites_path,  "currentsite.txt"), "w") as sitefile:
 			sitefile.write(site)
+		print("Current Site set to {}".format(site))
 	else:
 		print("{} does not exist".format(site))
 
 @click.command('backup')
 @click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
+@click.option('--verbose', default=False, is_flag=True)
 @pass_context
 def backup(context, with_files=False, backup_path_db=None, backup_path_files=None,
-	backup_path_private_files=None, quiet=False):
+	backup_path_private_files=None, quiet=False, verbose=False):
 	"Backup"
 	from frappe.utils.backups import scheduled_backup
-	verbose = context.verbose
+	verbose = verbose or context.verbose
 	exit_code = 0
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, force=True)
+			odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, force=True, verbose=verbose)
 		except Exception as e:
 			if verbose:
 				print("Backup failed for {0}. Database or site_config.json may be corrupted".format(site))
@@ -342,10 +353,12 @@ def backup(context, with_files=False, backup_path_db=None, backup_path_files=Non
 
 		if verbose:
 			from frappe.utils import now
-			print("database backup taken -", odb.backup_path_db, "- on", now())
+			summary_title = "Backup Summary at {0}".format(now())
+			print(summary_title + "\n" + "-" * len(summary_title))
+			print("Database backup:", odb.backup_path_db)
 			if with_files:
-				print("files backup taken -", odb.backup_path_files, "- on", now())
-				print("private files backup taken -", odb.backup_path_private_files, "- on", now())
+				print("Public files:   ", odb.backup_path_files)
+				print("Private files:  ", odb.backup_path_private_files)
 
 		frappe.destroy()
 	sys.exit(exit_code)
@@ -559,6 +572,7 @@ commands = [
 	install_app,
 	list_apps,
 	migrate,
+	migrate_to,
 	new_site,
 	reinstall,
 	reload_doc,

frappe/config/customization.py

@@ -3,7 +3,7 @@ from frappe import _
 
 def get_data():
 	return [
-        {
+		{
 			"label": _("Form Customization"),
 			"icon": "fa fa-glass",
 			"items": [
@@ -57,9 +57,9 @@ def get_data():
 				},
 				{
 					"type": "doctype",
-					"label": _("Custom Tags"),
-					"name": "Tag Category",
-					"description": _("Add your own Tag Categories")
+					"label": _("Package"),
+					"name": "Package",
+					"description": _("Import and Export Packages.")
 				}
 			]
 		}

frappe/contacts/doctype/contact/contact.py

@@ -3,7 +3,7 @@
 
 from __future__ import unicode_literals
 import frappe
-from frappe.utils import cstr, has_gravatar
+from frappe.utils import cstr, has_gravatar, cint
 from frappe import _
 from frappe.model.document import Document
 from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
@@ -133,7 +133,7 @@ def get_default_contact(doctype, name):
 			dl.parenttype = "Contact"''', (doctype, name))
 
 	if out:
-		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(y[1], x[1])))[0][0]
+		return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0]
 	else:
 		return None
 

frappe/core/doctype/data_import/importer_new.py

@@ -74,7 +74,6 @@ class Importer:
 			self.read_content(content, extension)
 
 		self.validate_template_content()
-		self.remove_empty_rows_and_columns()
 
 	def read_file(self, file_path):
 		extn = file_path.split(".")[1]
@@ -99,6 +98,8 @@ class Importer:
 		elif extension == "xls":
 			data = read_xls_file_from_attached_file(content)
 
+		data = self.remove_empty_rows_and_columns(data)
+
 		if len(data) <= 1:
 			frappe.throw(
 				_("Import template should contain a Header and atleast one row."), title=error_title
@@ -114,42 +115,41 @@ class Importer:
 				_("Number of columns does not match with data"), title=_("Invalid Template")
 			)
 
-	def remove_empty_rows_and_columns(self):
+	def remove_empty_rows_and_columns(self, raw_data):
 		self.row_index_map = []
 		removed_rows = []
 		removed_columns = []
 
 		# remove empty rows
-		data = []
-		for i, row in enumerate(self.data):
+		data_without_empty_rows = []
+		for i, row in enumerate(raw_data):
 			if all(v in INVALID_VALUES for v in row):
 				# empty row
 				removed_rows.append(i)
 			else:
-				data.append(row)
+				data_without_empty_rows.append(row)
 				self.row_index_map.append(i)
 
 		# remove empty columns
 		# a column with a header and no data is a valid column
 		# a column with no header and no data will be removed
-		header_row = []
-		for i, column in enumerate(self.header_row):
-			column_values = [row[i] for row in data]
-			values = [column] + column_values
-			if all(v in INVALID_VALUES for v in values):
+		first_row = data_without_empty_rows[0]
+		for i, column in enumerate(first_row):
+			column_values = [row[i] for row in data_without_empty_rows]
+			if all(v in INVALID_VALUES for v in column_values):
 				# empty column
 				removed_columns.append(i)
-			else:
-				header_row.append(column)
 
-		data_without_empty_columns = []
-		# remove empty columns from data
-		for i, row in enumerate(data):
-			new_row = [v for j, v in enumerate(row) if j not in removed_columns]
-			data_without_empty_columns.append(new_row)
+		if removed_columns:
+			data_without_empty_rows_and_columns = []
+			# remove empty columns from data
+			for i, row in enumerate(data_without_empty_rows):
+				new_row = [v for j, v in enumerate(row) if j not in removed_columns]
+				data_without_empty_rows_and_columns.append(new_row)
+		else:
+			data_without_empty_rows_and_columns = data_without_empty_rows
 
-		self.data = data_without_empty_columns
-		self.header_row = header_row
+		return data_without_empty_rows_and_columns
 
 	def get_data_for_import_preview(self):
 		out = frappe._dict()
@@ -325,7 +325,7 @@ class Importer:
 
 	def detect_date_formats(self, columns):
 		for col in columns:
-			if col.df and col.df.fieldtype in ['Date', 'Time', 'Datetime']:
+			if col.df and col.df.fieldtype in ["Date", "Time", "Datetime"]:
 				col.date_format = self.guess_date_format_for_column(col, columns)
 		return columns
 
@@ -351,7 +351,16 @@ class Importer:
 		value = cstr(value)
 
 		# convert boolean values to 0 or 1
-		if df.fieldtype == "Check" and value.lower().strip() in ["t", "f", "true", "false", "yes", "no", "y", "n"]:
+		if df.fieldtype == "Check" and value.lower().strip() in [
+			"t",
+			"f",
+			"true",
+			"false",
+			"yes",
+			"no",
+			"y",
+			"n",
+		]:
 			value = value.lower().strip()
 			value = 1 if value in ["t", "true", "y", "yes"] else 0
 
@@ -398,8 +407,9 @@ class Importer:
 			date_values = [
 				row[column_index] for row in self.data[:PARSE_ROW_COUNT] if row[column_index]
 			]
-			date_formats = [guess_date_format(d) if isinstance(d, str) else None
-							for d in date_values]
+			date_formats = [
+				guess_date_format(d) if isinstance(d, str) else None for d in date_values
+			]
 			if not date_formats:
 				return
 			max_occurred_date_format = max(set(date_formats), key=date_formats.count)
@@ -827,9 +837,9 @@ class Importer:
 		id_value = doc[id_fieldname]
 		existing_doc = frappe.get_doc(self.doctype, id_value)
 		existing_doc.flags.updater_reference = {
-			'doctype': self.data_import.doctype,
-			'docname': self.data_import.name,
-			'label': _('via Data Import')
+			"doctype": self.data_import.doctype,
+			"docname": self.data_import.name,
+			"label": _("via Data Import"),
 		}
 		existing_doc.update(doc)
 		existing_doc.save()

frappe/core/doctype/data_import/test_exporter_new.py

@@ -20,7 +20,7 @@ class TestExporter(unittest.TestCase):
 		e = Exporter('Web Page', export_fields='All')
 		csv_array = e.get_csv_array()
 		header = csv_array[0]
-		self.assertEqual(len(header), 36)
+		self.assertEqual(len(header), 37)
 
 
 	def test_exports_selected_fields(self):

frappe/core/doctype/data_import_beta/data_import_beta.js

@@ -177,8 +177,8 @@ frappe.ui.form.on('Data Import Beta', {
 	start_import(frm) {
 		frm
 			.call({
-				doc: frm.doc,
-				method: 'start_import',
+				method: 'form_start_import',
+				args: { data_import: frm.doc.name },
 				btn: frm.page.btn_primary
 			})
 			.then(r => {
@@ -252,8 +252,8 @@ frappe.ui.form.on('Data Import Beta', {
 
 		frm
 			.call({
-				doc: frm.doc,
 				method: 'get_preview_from_template',
+				args: { data_import: frm.doc.name },
 				error_handlers: {
 					TimestampMismatchError() {
 						// ignore this error

frappe/core/doctype/data_import_beta/data_import_beta.py

@@ -61,6 +61,16 @@ class DataImportBeta(Document):
 		return Importer(self.reference_doctype, data_import=self)
 
 
+@frappe.whitelist()
+def get_preview_from_template(data_import):
+	return frappe.get_doc("Data Import Beta", data_import).get_preview_from_template()
+
+
+@frappe.whitelist()
+def form_start_import(data_import):
+	return frappe.get_doc("Data Import Beta", data_import).start_import()
+
+
 def start_import(data_import):
 	"""This method runs in background job"""
 	data_import = frappe.get_doc("Data Import Beta", data_import)
@@ -69,12 +79,11 @@ def start_import(data_import):
 		i.import_data()
 	except:
 		frappe.db.rollback()
-		data_import.db_set('status', 'Error')
+		data_import.db_set("status", "Error")
 		frappe.log_error(title=data_import.name)
 		frappe.db.commit()
-		frappe.publish_realtime(
-			"data_import_refresh", {"data_import": data_import.name}
-		)
+		frappe.publish_realtime("data_import_refresh", {"data_import": data_import.name})
+
 
 @frappe.whitelist()
 def download_template(

frappe/core/doctype/docfield/docfield.json

@@ -43,6 +43,7 @@
   "report_hide",
   "remember_last_selected_value",
   "ignore_xss_filter",
+  "hide_border",
   "property_depends_on_section",
   "mandatory_depends_on",
   "column_break_38",
@@ -448,12 +449,19 @@
   {
    "fieldname": "column_break_38",
    "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.fieldtype=='Section Break'",
+   "fieldname": "hide_border",
+   "fieldtype": "Check",
+   "label": "Hide Border"
   }
  ],
  "idx": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-04-19 21:54:13.783908",
+ "modified": "2020-04-27 11:38:21.223185",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocField",

frappe/core/doctype/installed_application/installed_application.json

@@ -0,0 +1,49 @@
+{
+ "actions": [],
+ "creation": "2020-05-11 17:44:54.674657",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "app_name",
+  "app_version",
+  "git_branch"
+ ],
+ "fields": [
+  {
+   "fieldname": "git_branch",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Git Branch",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "app_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Application Name",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "app_version",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Application Version",
+   "read_only": 1,
+   "reqd": 1
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-05-12 10:09:49.148087",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Installed Application",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/installed_application/installed_application.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class InstalledApplication(Document):
+	pass

frappe/core/doctype/installed_applications/installed_applications.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Installed Applications', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/installed_applications/installed_applications.json

@@ -0,0 +1,42 @@
+{
+ "actions": [],
+ "creation": "2020-05-11 17:45:41.587750",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "installed_applications"
+ ],
+ "fields": [
+  {
+   "fieldname": "installed_applications",
+   "fieldtype": "Table",
+   "label": "Installed Applications",
+   "options": "Installed Application",
+   "read_only": 1
+  }
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-05-12 10:09:14.310622",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Installed Applications",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/installed_applications/installed_applications.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class InstalledApplications(Document):
+	def update_versions(self):
+		self.delete_key("installed_applications")
+		for app in frappe.utils.get_installed_apps_info():
+			self.append("installed_applications", {
+				"app_name": app.get("app_name"),
+				"app_version": app.get("version"),
+				"git_branch": app.get("branch")
+			})
+		self.save()

frappe/core/doctype/installed_applications/test_installed_applications.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestInstalledApplications(unittest.TestCase):
+	pass

frappe/core/page/dashboard/dashboard.css

@@ -0,0 +1,5 @@
+.restricted-button {
+	cursor: default;
+	position: relative;
+	right: -5px;
+}

frappe/core/page/dashboard/dashboard.js

@@ -6,7 +6,7 @@ frappe.provide('frappe.dashboards.chart_sources');
 
 
 frappe.pages['dashboard'].on_page_load = function(wrapper) {
-	var page = frappe.ui.make_app_page({
+	frappe.ui.make_app_page({
 		parent: wrapper,
 		title: __("Dashboard"),
 		single_column: true
@@ -21,11 +21,18 @@ frappe.pages['dashboard'].on_page_load = function(wrapper) {
 class Dashboard {
 	constructor(wrapper) {
 		this.wrapper = $(wrapper);
-		$(`<div class="dashboard">
+		$(`<div class="dashboard" style="overflow-y: hidden">
 			<div class="dashboard-graph"></div>
 		</div>`).appendTo(this.wrapper.find(".page-content").empty());
 		this.container = this.wrapper.find(".dashboard-graph");
 		this.page = wrapper.page;
+
+		this.page.set_title_sub(
+			$(`<button class="restricted-button">
+				<span class="octicon octicon-lock"></span>
+				<span>${__('Restricted')}</span>
+			</button>`)
+		);
 	}
 
 	show() {

frappe/custom/doctype/custom_field/custom_field.json

@@ -48,6 +48,7 @@
   "allow_in_quick_entry",
   "ignore_xss_filter",
   "translatable",
+  "hide_border",
   "description",
   "permlevel",
   "width",
@@ -378,12 +379,19 @@
    "fieldname": "in_preview",
    "fieldtype": "Check",
    "label": "In Preview"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.fieldtype=='Section Break'",
+   "fieldname": "hide_border",
+   "fieldtype": "Check",
+   "label": "Hide Border"
   }
  ],
  "icon": "fa fa-glass",
  "idx": 1,
  "links": [],
- "modified": "2020-04-10 11:57:10.392218",
+ "modified": "2020-04-27 11:40:48.325481",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",

frappe/custom/doctype/custom_link/custom_link.js

@@ -0,0 +1,20 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Custom Link', {
+	refresh: function(frm) {
+		frm.set_query("document_type", function () {
+			return {
+				filters: {
+					custom: 0,
+					istable: 0,
+					module: ['not in', ["Email", "Core", "Custom", "Event Streaming", "Social", "Data Migration", "Geo", "Desk"]]
+				}
+			};
+		});
+
+		frm.add_custom_button(__('Go to {0} List', [frm.doc.document_type]), function() {
+			frappe.set_route('List', frm.doc.document_type);
+		});
+	}
+});

frappe/custom/doctype/custom_link/custom_link.json

@@ -0,0 +1,52 @@
+{
+ "actions": [],
+ "autoname": "field:document_type",
+ "creation": "2020-04-08 15:16:44.342509",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "document_type",
+  "links"
+ ],
+ "fields": [
+  {
+   "fieldname": "document_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1,
+   "unique": 1
+  },
+  {
+   "fieldname": "links",
+   "fieldtype": "Table",
+   "label": "Links",
+   "options": "DocType Link"
+  }
+ ],
+ "links": [],
+ "modified": "2020-04-08 16:42:59.402671",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Custom Link",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/custom/doctype/custom_link/custom_link.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class CustomLink(Document):
+	pass

frappe/custom/doctype/custom_link/test_custom_link.py

@@ -6,5 +6,5 @@ from __future__ import unicode_literals
 # import frappe
 import unittest
 
-class TestOnboarding(unittest.TestCase):
+class TestCustomLink(unittest.TestCase):
 	pass

frappe/custom/doctype/customize_form/customize_form.py

@@ -76,7 +76,8 @@ docfield_properties = {
 	'remember_last_selected_value': 'Check',
 	'allow_bulk_edit': 'Check',
 	'auto_repeat': 'Link',
-	'allow_in_quick_entry': 'Check'
+	'allow_in_quick_entry': 'Check',
+	'hide_border': 'Check'
 }
 
 allowed_fieldtype_change = (('Currency', 'Float', 'Percent'), ('Small Text', 'Data'),

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -39,6 +39,7 @@
   "allow_on_submit",
   "report_hide",
   "remember_last_selected_value",
+  "hide_border",
   "property_depends_on_section",
   "mandatory_depends_on",
   "column_break_33",
@@ -388,12 +389,19 @@
    "fieldname": "in_preview",
    "fieldtype": "Check",
    "label": "In Preview"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.fieldtype=='Section Break'",
+   "fieldname": "hide_border",
+   "fieldtype": "Check",
+   "label": "Hide Border"
   }
  ],
  "idx": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-04-10 11:58:44.573537",
+ "modified": "2020-04-27 11:39:26.389300",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/custom/doctype/package_document_type/package_document_type.json

@@ -0,0 +1,65 @@
+{
+ "actions": [],
+ "creation": "2020-05-14 16:45:47.196395",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "document_type",
+  "column_break_2",
+  "attachments",
+  "overwrite",
+  "section_break_4",
+  "filters_json"
+ ],
+ "fields": [
+  {
+   "fieldname": "document_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Document Type",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_2",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "attachments",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Include Attachments"
+  },
+  {
+   "default": "0",
+   "fieldname": "overwrite",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Overwrite"
+  },
+  {
+   "fieldname": "section_break_4",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "filters_json",
+   "fieldtype": "Code",
+   "label": "Filters",
+   "options": "JSON"
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-05-14 16:45:47.196395",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Package Document Type",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/custom/doctype/package_document_type/package_document_type.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class PackageDocumentType(Document):
+	pass

frappe/custom/doctype/package_publish_target/package_publish_target.json

@@ -0,0 +1,47 @@
+{
+ "actions": [],
+ "creation": "2020-05-13 16:04:32.724663",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "instance_url",
+  "username",
+  "password"
+ ],
+ "fields": [
+  {
+   "fieldname": "instance_url",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Site URL",
+   "reqd": 1
+  },
+  {
+   "fieldname": "username",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Username",
+   "reqd": 1
+  },
+  {
+   "fieldname": "password",
+   "fieldtype": "Password",
+   "in_list_view": 1,
+   "label": "Password",
+   "reqd": 1
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2020-05-15 17:35:16.282235",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Package Publish Target",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/custom/doctype/package_publish_target/package_publish_target.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class PackagePublishTarget(Document):
+	pass

frappe/custom/doctype/package_publish_tool/package_publish_tool.js

@@ -0,0 +1,159 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Package Publish Tool', {
+	refresh: function(frm) {
+		frm.set_query("document_type", "package_details", function () {
+			return {
+				filters: {
+					"istable": 0,
+				}
+			};
+		});
+
+		frappe.realtime.on("package", (data) => {
+			frm.dashboard.show_progress(data.prefix, data.progress / data.total * 100, __("{0}", [data.message]));
+			if ((data.progress+1) != data.total) {
+				frm.dashboard.show_progress(data.prefix, data.progress / data.total * 100, __("{0}", [data.message]));
+			} else {
+				frm.dashboard.hide_progress();
+			}
+		});
+
+		frm.trigger("show_instructions");
+		frm.trigger("last_deployed_on");
+		frm.trigger("set_dirty_trigger");
+		frm.trigger("set_deploy_primary_action");
+	},
+	last_deployed_on: function(frm) {
+		if (frm.doc.last_deployed_on) {
+			frm.trigger("show_indicator");
+		}
+	},
+	show_indicator: function(frm) {
+		let pretty_date = frappe.datetime.prettyDate(frm.doc.last_deployed_on);
+		frm.page.set_indicator(__("Last published {0}", [pretty_date]), "blue");
+	},
+	set_dirty_trigger: function(frm) {
+		$(frm.wrapper).on("dirty", function() {
+			frm.page.set_primary_action(__('Save'), () => frm.save());
+		});
+	},
+	set_deploy_primary_action: function(frm) {
+		if (frm.doc.package_details.length && frm.doc.instances.length) {
+			frm.page.set_primary_action(__("Publish"), function () {
+				frappe.show_alert({
+					message: __("Publishing documents..."),
+					indicator: "green"
+				});
+
+				frappe.call({
+					method: "frappe.custom.doctype.package_publish_tool.package_publish_tool.deploy_package",
+					callback: function() {
+						frm.reload_doc();
+						frappe.msgprint(__("Documents have been published."));
+					}
+				});
+			});
+		}
+	},
+	show_instructions: function(frm) {
+		let field = frm.get_field("html_info");
+		field.html(`
+			<p class="text-muted text-medium">
+				Package Publish Tool let's you copy documents from your site to any other remote site.
+				Follow the steps below to publish.
+			</p>
+			<ol class="text-muted small">
+				<li>Add Document Types that you want to copy from the table below. You can also add filters by expanding the row.</li>
+				<li>Add the Sites URL where you want to copy these documents, and enter the Username and Password.</li>
+				<li>Click on Save. Now, you can click on Publish and the documents will be copied.</li>
+			</ol>
+		`);
+	}
+});
+
+frappe.ui.form.on('Package Document Type', {
+	form_render: function (frm, cdt, cdn) {
+		function _show_filters(filters, table) {
+			table.find('tbody').empty();
+
+			if (filters.length > 0) {
+				filters.forEach(filter => {
+					const filter_row =
+						$(`<tr>
+							<td>${filter[1]}</td>
+							<td>${filter[2] || ""}</td>
+							<td>${filter[3]}</td>
+						</tr>`);
+
+					table.find('tbody').append(filter_row);
+				});
+			} else {
+				const filter_row = $(`<tr><td colspan="3" class="text-muted text-center">
+					${__("Click to Set Filters")}</td></tr>`);
+				table.find('tbody').append(filter_row);
+			}
+		}
+
+		let row = frappe.get_doc(cdt, cdn);
+
+		let wrapper = $(`[data-fieldname="filters_json"]`).empty();
+		let table = $(`<table class="table table-bordered" style="cursor:pointer; margin:0px;">
+				<thead>
+					<tr>
+						<th style="width: 33%">${__('Filter')}</th>
+						<th style="width: 33%">${__('Condition')}</th>
+						<th>${__('Value')}</th>
+					</tr>
+				</thead>
+				<tbody>
+				</tbody>
+			</table>`).appendTo(wrapper);
+		$(`<p class="text-muted small">${__("Click table to edit")}</p>`).appendTo(wrapper);
+
+		let filters = JSON.parse(row.filters_json || '[]');
+		_show_filters(filters, table);
+
+		table.on('click', () => {
+			if (!row.document_type) {
+				frappe.msgprint(__("Select Document Type."));
+				return;
+			}
+
+			frappe.model.with_doctype(row.document_type, function() {
+				let dialog = new frappe.ui.Dialog({
+					title: __('Set Filters'),
+					fields: [
+						{
+							fieldtype: 'HTML',
+							label: 'Filters',
+							fieldname: 'filter_area',
+						}
+					],
+					primary_action: function() {
+						let values = filter_group.get_filters();
+						let flt = [];
+						if (values) {
+							values.forEach(function(value) {
+								flt.push([value[0], value[1], value[2], value[3]]);
+							});
+						}
+						row.filters_json = JSON.stringify(flt);
+						_show_filters(flt, table);
+						dialog.hide();
+					},
+					primary_action_label: "Set"
+				});
+
+				let filter_group = new frappe.ui.FilterGroup({
+					parent: dialog.get_field('filter_area').$wrapper,
+					doctype: row.document_type,
+					on_change: () => {},
+				});
+				filter_group.add_filters_to_filter_group(filters);
+				dialog.show();
+			});
+		});
+	},
+});

frappe/custom/doctype/package_publish_tool/package_publish_tool.json

@@ -0,0 +1,84 @@
+{
+ "actions": [],
+ "creation": "2020-05-13 15:54:38.082657",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "html_info",
+  "sb_00",
+  "package_details",
+  "sb_01",
+  "instances",
+  "last_deployed_on"
+ ],
+ "fields": [
+  {
+   "description": "Click on the row for accessing filters.",
+   "fieldname": "package_details",
+   "fieldtype": "Table",
+   "label": "Document Types",
+   "options": "Package Document Type",
+   "reqd": 1
+  },
+  {
+   "fieldname": "instances",
+   "fieldtype": "Table",
+   "label": "Sites",
+   "options": "Package Publish Target",
+   "reqd": 1
+  },
+  {
+   "fieldname": "html_info",
+   "fieldtype": "HTML"
+  },
+  {
+   "fieldname": "last_deployed_on",
+   "fieldtype": "Datetime",
+   "hidden": 1,
+   "label": "Last Deployed On",
+   "read_only": 1
+  },
+  {
+   "fieldname": "sb_00",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "sb_01",
+   "fieldtype": "Section Break"
+  }
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2020-05-15 17:31:37.060199",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Package Publish Tool",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "All",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/custom/doctype/package_publish_tool/package_publish_tool.py

@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+import json
+import datetime
+import base64
+from frappe.model.document import Document
+from frappe.utils.file_manager import save_file, get_file
+from frappe import _
+from six import string_types
+from frappe.frappeclient import FrappeClient
+from frappe.utils import get_datetime_str, get_datetime
+from frappe.utils.password import get_decrypted_password
+
+class PackagePublishTool(Document):
+	pass
+
+@frappe.whitelist()
+def deploy_package():
+	package, doc = export_package()
+
+	file_name = "Package-" + get_datetime_str(get_datetime())
+
+	length = len(doc.instances)
+	for idx, instance in enumerate(doc.instances):
+		frappe.publish_realtime("package",  {"progress": idx, "total": length, "message": instance.instance_url, "prefix": _("Deploying")},
+			user=frappe.session.user)
+
+		install_package_to_remote(package, instance)
+
+	frappe.db.set_value("Package Publish Tool", "Package Publish Tool", "last_deployed_on", frappe.utils.now_datetime())
+
+def install_package_to_remote(package, instance):
+	try:
+		connection = FrappeClient(instance.instance_url, instance.username, get_decrypted_password(instance.doctype, instance.name))
+	except Exception:
+		frappe.log_error(frappe.get_traceback())
+		frappe.throw(_("Couldn't connect to site {0}. Please check Error Logs.").format(instance.instance_url))
+
+	try:
+		connection.post_request({
+			"cmd": "frappe.custom.doctype.package_publish_tool.package_publish_tool.import_package",
+			"package": json.dumps(package)
+		})
+	except Exception:
+		frappe.log_error(frappe.get_traceback())
+		frappe.throw(_("Error while installing package to site {0}. Please check Error Logs.").format(instance.instance_url))
+
+@frappe.whitelist()
+def export_package():
+	"""Export package as JSON."""
+	package_doc = frappe.get_single("Package Publish Tool")
+	package = []
+
+	for doctype in package_doc.package_details:
+		filters = []
+
+		if doctype.get("filters_json"):
+			filters = json.loads(doctype.get("filters_json"))
+
+		docs = frappe.get_all(doctype.get("document_type"), filters=filters)
+		length = len(docs)
+
+		for idx, doc in enumerate(docs):
+			frappe.publish_realtime("package", {
+					"progress":idx, "total":length,
+					"message":doctype.get("document_type"),
+					"prefix": _("Exporting")
+				},
+				user=frappe.session.user)
+
+			document = frappe.get_doc(doctype.get("document_type"), doc.name).as_dict()
+			attachments = []
+
+			if doctype.attachments:
+				filters = {
+					"attached_to_doctype": document.get("doctype"),
+					"attached_to_name": document.get("name")
+				}
+
+				for f in frappe.get_list("File", filters=filters):
+					fname, fcontents = get_file(f.name)
+					attachments.append({
+						"fname": fname,
+						"content": base64.b64encode(fcontents).decode('ascii')
+					})
+
+			document.update({
+				"__attachments": attachments,
+				"__overwrite": True if doctype.overwrite else False
+			})
+
+			package.append(document)
+
+	return post_process(package), package_doc
+
+@frappe.whitelist()
+def import_package(package=None):
+	"""Import package from JSON."""
+	if isinstance(package, string_types):
+		package = json.loads(package)
+
+	for doc in package:
+		modified = doc.pop("modified")
+		overwrite = doc.pop("__overwrite")
+		attachments = doc.pop("__attachments")
+		exists = frappe.db.exists(doc.get("doctype"), doc.get("name"))
+
+		if not exists:
+			d = frappe.get_doc(doc).insert(ignore_permissions=True, ignore_if_duplicate=True)
+			if attachments:
+				add_attachment(attachments, d)
+		else:
+			docname = doc.pop("name")
+			document = frappe.get_doc(doc.get("doctype"), docname)
+
+			if overwrite:
+				update_document(document, doc, attachments)
+
+			else:
+				if frappe.utils.get_datetime(document.modified) < frappe.utils.get_datetime(modified):
+					update_document(document, doc, attachments)
+
+def update_document(document, doc, attachments):
+	document.update(doc)
+	document.save()
+	if attachments:
+		add_attachment(attachments, document)
+
+def add_attachment(attachments, doc):
+	for attachment in attachments:
+		save_file(attachment.get("fname"), base64.b64decode(attachment.get("content")), doc.get("doctype"), doc.get("name"))
+
+def post_process(package):
+	"""Remove the keys from Document and Child Document. Convert datetime, date, time to str."""
+	del_keys = ('modified_by', 'creation', 'owner', 'idx', 'docstatus')
+	child_del_keys = ('modified_by', 'creation', 'owner', 'idx', 'docstatus', 'name')
+
+	for doc in package:
+		for key in del_keys:
+			if key in doc:
+				del doc[key]
+
+		for key, value in doc.items():
+			stringified_value = get_stringified_value(value)
+			if stringified_value:
+				doc[key] = stringified_value
+
+			if not isinstance(value, list):
+				continue
+
+			for child in value:
+				for child_key in child_del_keys:
+					if child_key in child:
+						del child[child_key]
+
+				for child_key, child_value in child.items():
+					stringified_value = get_stringified_value(child_value)
+					if stringified_value:
+						child[child_key] = stringified_value
+
+	return package
+
+def get_stringified_value(value):
+	if isinstance(value, datetime.datetime):
+		return frappe.utils.get_datetime_str(value)
+
+	if isinstance(value, datetime.date):
+		return frappe.utils.get_date_str(value)
+
+	if isinstance(value, datetime.timedelta):
+		return frappe.utils.get_time_str(value)
+
+	return None

frappe/custom/doctype/package_publish_tool/test_package_publish_tool.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestPackagePublishTool(unittest.TestCase):
+	pass

frappe/database/mariadb/framework_mariadb.sql

@@ -63,6 +63,7 @@ CREATE TABLE `tabDocField` (
   `precision` varchar(255) DEFAULT NULL,
   `length` int(11) NOT NULL DEFAULT 0,
   `translatable` int(1) NOT NULL DEFAULT 0,
+  `hide_border` int(1) NOT NULL DEFAULT 0,
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`),
   KEY `label` (`label`),

frappe/database/postgres/framework_postgres.sql

@@ -63,6 +63,7 @@ CREATE TABLE "tabDocField" (
   "precision" varchar(255) DEFAULT NULL,
   "length" bigint NOT NULL DEFAULT 0,
   "translatable" smallint NOT NULL DEFAULT 0,
+  "hide_border" smallint NOT NULL DEFAULT 0,
   PRIMARY KEY ("name")
 ) ;
 

frappe/desk/desktop.py

@@ -21,19 +21,17 @@ class Workspace:
 		self.extended_charts = []
 		self.extended_shortcuts = []
 
-		user = frappe.get_user()
-		user.build_permissions()
-
-		user_doc = frappe.get_doc('User', frappe.session.user)
-		self.blocked_modules = user_doc.get_blocked_modules()
+		self.user = frappe.get_user()
+		self.allowed_modules = self.get_cached_value('user_allowed_modules', self.get_allowed_modules)
 		self.doc = self.get_page_for_user()
 
-		if self.doc.module in self.blocked_modules:
+		if self.doc.module not in self.allowed_modules:
 			raise frappe.PermissionError
 
-		self.user = user
-		self.allowed_pages = get_allowed_pages()
-		self.allowed_reports = get_allowed_reports()
+		self.can_read = self.get_cached_value('user_perm_can_read', self.get_can_read_items)
+
+		self.allowed_pages = get_allowed_pages(cache=True)
+		self.allowed_reports = get_allowed_reports(cache=True)
 		self.onboarding_doc = self.get_onboarding_doc()
 		self.onboarding = None
 
@@ -41,6 +39,31 @@ class Workspace:
 		self.restricted_doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache()
 		self.restricted_pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache()
 
+	def get_cached_value(self, cache_key, fallback_fn):
+		_cache = frappe.cache()
+
+		value = _cache.get_value(cache_key, user=frappe.session.user)
+		if value:
+			return value
+
+		value = fallback_fn()
+
+		# Expire every six hour
+		_cache.set_value(cache_key, value, frappe.session.user, 21600)
+		return value
+
+	def get_can_read_items(self):
+		if not self.user.can_read:
+			self.user.build_permissions()
+
+		return self.user.can_read
+
+	def get_allowed_modules(self):
+		if not self.user.allow_modules:
+			self.user.build_permissions()
+
+		return self.user.allow_modules
+
 	def get_page_for_user(self):
 		filters = {
 			'extends': self.page_name,
@@ -61,14 +84,14 @@ class Workspace:
 		if not self.doc.onboarding:
 			return None
 
-		if frappe.db.get_value("Onboarding", self.doc.onboarding, "is_complete"):
+		if frappe.db.get_value("Module Onboarding", self.doc.onboarding, "is_complete"):
 			return None
 
-		doc = frappe.get_doc("Onboarding", self.doc.onboarding)
+		doc = frappe.get_doc("Module Onboarding", self.doc.onboarding)
 
 		# Check if user is allowed
 		allowed_roles = set(doc.get_allowed_roles())
-		user_roles = set(self.user.get_roles())
+		user_roles = set(frappe.get_roles())
 		if not allowed_roles & user_roles:
 			return None
 
@@ -83,7 +106,7 @@ class Workspace:
 			"extends": self.page_name,
 			'restrict_to_domain': ['in', frappe.get_active_domains()],
 			'for_user': '',
-			'module': ['not in', self.blocked_modules]
+			'module': ['in', self.allowed_modules]
 		})
 
 		pages = [frappe.get_doc("Desk Page", page['name']) for page in pages]
@@ -97,13 +120,15 @@ class Workspace:
 		item_type = item_type.lower()
 
 		if item_type == "doctype":
-			return (name in self.user.can_read and name in self.restricted_doctypes)
+			return (name in self.can_read and name in self.restricted_doctypes)
 		if item_type == "page":
 			return (name in self.allowed_pages and name in self.restricted_pages)
 		if item_type == "report":
 			return name in self.allowed_reports
 		if item_type == "help":
 			return True
+		if item_type == "dashboard":
+			return True
 
 		return False
 
@@ -134,15 +159,18 @@ class Workspace:
 			}
 
 	def get_cards(self):
-		cards = self.doc.cards + get_custom_reports_and_doctypes(self.doc.module)
+		cards = self.doc.cards
+		if not self.doc.hide_custom:
+			cards = cards + get_custom_reports_and_doctypes(self.doc.module)
+
 		if len(self.extended_cards):
 			cards = cards + self.extended_cards
 		default_country = frappe.db.get_default("country")
 
 		def _doctype_contains_a_record(name):
-			exists = self.table_counts.get(name)
-			if not exists:
-				if not frappe.db.get_value('DocType', name, 'issingle'):
+			exists = self.table_counts.get(name, None)
+			if exists is None:
+				if not frappe.db.get_value('DocType', name, 'issingle', cache=True):
 					exists = frappe.db.count(name)
 				else:
 					exists = True
@@ -249,6 +277,8 @@ class Workspace:
 		for doc in self.onboarding_doc.get_steps():
 			step = doc.as_dict().copy()
 			step.label = _(doc.title)
+			if step.action == "Create Entry":
+				step.is_submittable = frappe.db.get_value("DocType", step.reference_document, 'is_submittable', cache=True)
 			steps.append(step)
 
 		return steps
@@ -292,7 +322,6 @@ def get_desk_sidebar_items(flatten=False):
 	filters = {
 		'restrict_to_domain': ['in', frappe.get_active_domains()],
 		'extends_another_page': 0,
-		'is_standard': 1,
 		'for_user': '',
 		'module': ['not in', blocked_modules]
 	}

frappe/desk/doctype/dashboard/dashboard.json

@@ -9,6 +9,7 @@
   "dashboard_name",
   "is_default",
   "charts",
+  "chart_options",
   "cards"
  ],
  "fields": [
@@ -33,6 +34,13 @@
    "options": "Dashboard Chart Link",
    "reqd": 1
   },
+  {
+    "description": "Set Default Options for all charts on this Dashboard (Ex: \"colors\": [\"#d1d8dd\", \"#ff5858\"])",
+    "fieldname": "chart_options",
+    "fieldtype": "Code",
+    "label": "Chart Options",
+    "options": "JSON"
+  },
   {
    "fieldname": "cards",
    "fieldtype": "Table",
@@ -41,7 +49,7 @@
   }
  ],
  "links": [],
- "modified": "2020-04-19 17:44:36.237163",
+ "modified": "2020-04-29 13:26:37.362482",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard",

frappe/desk/doctype/dashboard/dashboard.py

@@ -5,6 +5,8 @@
 from __future__ import unicode_literals
 from frappe.model.document import Document
 import frappe
+from frappe import _
+import json
 
 class Dashboard(Document):
 	def on_update(self):
@@ -13,13 +15,29 @@ class Dashboard(Document):
 			frappe.db.sql('''update
 				tabDashboard set is_default = 0 where name != %s''', self.name)
 
+	def validate(self):
+		self.validate_custom_options()
+
+	def validate_custom_options(self):
+		if self.chart_options:
+			try:
+				json.loads(self.chart_options)
+			except ValueError as error:
+				frappe.throw(_("Invalid json added in the custom options: {0}").format(error))
+
 @frappe.whitelist()
 def get_permitted_charts(dashboard_name):
 	permitted_charts = []
 	dashboard = frappe.get_doc('Dashboard', dashboard_name)
 	for chart in dashboard.charts:
 		if frappe.has_permission('Dashboard Chart', doc=chart.chart):
-			permitted_charts.append(chart)
+			chart_dict = frappe._dict()
+			chart_dict.update(chart.as_dict())
+
+			if dashboard.get('chart_options'):
+				chart_dict.custom_options = dashboard.get('chart_options')
+			permitted_charts.append(chart_dict)
+
 	return permitted_charts
 
 @frappe.whitelist()

frappe/desk/doctype/dashboard_chart/dashboard_chart.js

@@ -49,6 +49,7 @@ frappe.ui.form.on('Dashboard Chart', {
 		});
 
 		frm.set_df_property("filters_section", "hidden", 1);
+		frm.trigger('set_time_series');
 		frm.set_query('document_type', function() {
 			return {
 				filters: {
@@ -57,6 +58,7 @@ frappe.ui.form.on('Dashboard Chart', {
 			}
 		});
 		frm.trigger('update_options');
+		frm.trigger('set_heatmap_year_options');
 		if (frm.doc.report_name) {
 			frm.trigger('set_chart_report_filters');
 		}
@@ -70,7 +72,17 @@ frappe.ui.form.on('Dashboard Chart', {
 		frm.trigger("show_filters");
 	},
 
+	set_heatmap_year_options: function(frm) {
+		if (frm.doc.type == 'Heatmap') {
+			frappe.db.get_doc('System Settings').then(doc => {
+				const creation_date = doc.creation;
+				frm.set_df_property('heatmap_year', 'options', frappe.dashboard_utils.get_years_since_creation(creation_date));
+			});
+		}
+	},
+
 	chart_type: function(frm) {
+		frm.trigger('set_time_series');
 		if (frm.doc.chart_type == 'Report') {
 			frm.set_query('report_name', () => {
 				return {
@@ -80,23 +92,19 @@ frappe.ui.form.on('Dashboard Chart', {
 				}
 			});
 		} else {
-			// set timeseries based on chart type
-			if (['Count', 'Average', 'Sum'].includes(frm.doc.chart_type)) {
-				frm.set_value('timeseries', 1);
-			} else {
-				frm.set_value('timeseries', 0);
-			}
-
-			if (frm.doc.chart_type == 'Group By') {
-				frm.set_df_property('type', 'options', ['Line', 'Bar', 'Percentage', 'Pie']);
-			} else {
-				frm.set_df_property('type', 'options', ['Line', 'Bar']);
-			}
-
 			frm.set_value('document_type', '');
 		}
 	},
 
+	set_time_series: function(frm) {
+		// set timeseries based on chart type
+		if (['Count', 'Average', 'Sum'].includes(frm.doc.chart_type)) {
+			frm.set_value('timeseries', 1);
+		} else {
+			frm.set_value('timeseries', 0);
+		}
+	},
+
 	document_type: function(frm) {
 		// update `based_on` options based on date / datetime fields
 		frm.set_value('source', '');
@@ -283,17 +291,7 @@ frappe.ui.form.on('Dashboard Chart', {
 				});
 			}
 		} else if (frm.chart_filters.length) {
-			fields = frm.chart_filters.filter(f => {
-				if (f.on_change && !f.reqd) {
-					return false;
-				}
-				if (f.get_query || f.get_data) {
-					f.read_only = 1;
-				}
-
-				return f.fieldname;
-			});
-
+			fields = frm.chart_filters.filter(f => f.fieldname);
 			fields.map( f => {
 				if (filters[f.fieldname]) {
 					let condition = '=';
@@ -353,10 +351,12 @@ frappe.ui.form.on('Dashboard Chart', {
 			}
 
 			dialog.show();
+			//Set query report object so that it can be used while fetching filter values in the report
+			frappe.query_report = new frappe.views.QueryReport({'filters': dialog.fields_list});
+			frappe.query_reports[frm.doc.report_name].onload
+				&& frappe.query_reports[frm.doc.report_name].onload(frappe.query_report);
 			dialog.set_values(filters);
 		});
 	},
 
 });
-
-

frappe/desk/doctype/dashboard_chart/dashboard_chart.json

@@ -23,17 +23,18 @@
   "number_of_groups",
   "column_break_6",
   "is_public",
+  "heatmap_year",
   "timespan",
   "from_date",
   "to_date",
   "time_interval",
   "timeseries",
+  "type",
   "filters_section",
   "filters_json",
   "chart_options_section",
-  "type",
-  "column_break_2",
   "color",
+  "column_break_2",
   "custom_options",
   "section_break_10",
   "last_synced_on"
@@ -85,14 +86,14 @@
    "fieldtype": "Column Break"
   },
   {
-   "depends_on": "timeseries",
+   "depends_on": "eval: doc.timeseries && doc.type !== 'Heatmap'",
    "fieldname": "timespan",
    "fieldtype": "Select",
    "label": "Timespan",
    "options": "Last Year\nLast Quarter\nLast Month\nLast Week\nSelect Date Range"
   },
   {
-   "depends_on": "timeseries",
+   "depends_on": "eval: doc.timeseries && doc.type !== 'Heatmap'",
    "fieldname": "time_interval",
    "fieldtype": "Select",
    "label": "Time Interval",
@@ -100,7 +101,7 @@
   },
   {
    "default": "0",
-   "depends_on": "eval: ['Count', 'Sum', 'Average'].includes(doc.chart_type)",
+   "depends_on": "eval: !['Group By', 'Report'].includes(doc.chart_type)\n",
    "fieldname": "timeseries",
    "fieldtype": "Check",
    "label": "Time Series"
@@ -123,18 +124,18 @@
    "label": "Chart Options"
   },
   {
+   "default": "Line",
    "fieldname": "type",
    "fieldtype": "Select",
    "label": "Type",
-   "options": "Line\nBar\nPercentage\nPie\nDonut",
-   "reqd": 1
+   "options": "Line\nBar\nPercentage\nPie\nDonut\nHeatmap"
   },
   {
    "fieldname": "column_break_2",
    "fieldtype": "Column Break"
   },
   {
-   "depends_on": "eval:doc.chart_type !== 'Report'",
+   "depends_on": "eval: doc.chart_type !== 'Report' && doc.type !== 'Heatmap'",
    "fieldname": "color",
    "fieldtype": "Color",
    "label": "Color"
@@ -228,10 +229,16 @@
    "fieldname": "is_public",
    "fieldtype": "Check",
    "label": "Is Public"
+  },
+  {
+   "depends_on": "eval: doc.type == 'Heatmap'",
+   "fieldname": "heatmap_year",
+   "fieldtype": "Select",
+   "label": "Year"
   }
  ],
  "links": [],
- "modified": "2020-05-01 15:22:59.119341",
+ "modified": "2020-05-16 15:03:02.455395",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Dashboard Chart",

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -8,7 +8,7 @@ from frappe import _
 import datetime
 import json
 from frappe.utils.dashboard import cache_source, get_from_date_from_timespan
-from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate, get_datetime
+from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate, get_datetime, cint
 from frappe.model.naming import append_number_if_name_exists
 from frappe.boot import get_allowed_reports
 from frappe.model.document import Document
@@ -58,13 +58,13 @@ def has_permission(doc, ptype, user):
 @frappe.whitelist()
 @cache_source
 def get(chart_name = None, chart = None, no_cache = None, filters = None, from_date = None,
-	to_date = None, timespan = None, time_interval = None, refresh = None):
+	to_date = None, timespan = None, time_interval = None, heatmap_year=None, refresh = None):
 	if chart_name:
 		chart = frappe.get_doc('Dashboard Chart', chart_name)
 	else:
 		chart = frappe._dict(frappe.parse_json(chart))
 
-
+	heatmap_year = heatmap_year or chart.heatmap_year
 	timespan = timespan or chart.timespan
 
 	if timespan == 'Select Date Range':
@@ -87,7 +87,10 @@ def get(chart_name = None, chart = None, no_cache = None, filters = None, from_d
 	if chart.chart_type == 'Group By':
 		chart_config = get_group_by_chart_config(chart, filters)
 	else:
-		chart_config =  get_chart_config(chart, filters, timespan, timegrain, from_date, to_date)
+		if chart.type == 'Heatmap':
+			chart_config = get_heatmap_chart_config(chart, filters, heatmap_year)
+		else:
+			chart_config =  get_chart_config(chart, filters, timespan, timegrain, from_date, to_date)
 
 	return chart_config
 
@@ -107,11 +110,11 @@ def create_dashboard_chart(args):
 	doc.insert(ignore_permissions=True)
 	return doc
 
-
 @frappe.whitelist()
 def create_report_chart(args):
-	create_dashboard_chart(args)
+	doc = create_dashboard_chart(args)
 	args = frappe.parse_json(args)
+	args.chart_name = doc.chart_name
 	if args.dashboard:
 		add_chart_to_dashboard(json.dumps(args))
 
@@ -134,7 +137,6 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 		to_date = datetime.datetime.now()
 
 	doctype = chart.document_type
-	unit_function = get_unit_function(doctype, chart.based_on, timegrain)
 	datefield = chart.based_on
 	aggregate_function = get_aggregate_function(chart.chart_type)
 	value_field = chart.value_based_on or '1'
@@ -144,26 +146,21 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 	filters.append([doctype, datefield, '>=', from_date, False])
 	filters.append([doctype, datefield, '<=', to_date, False])
 
-	data = frappe.db.get_all(
+	data = frappe.db.get_list(
 		doctype,
 		fields = [
-			'extract(year from `tab{doctype}`.{datefield}) as _year'.format(doctype=doctype, datefield=datefield),
-			'{} as _unit'.format(unit_function),
+			'{} as _unit'.format(datefield),
 			'{aggregate_function}({value_field})'.format(aggregate_function=aggregate_function, value_field=value_field),
 		],
 		filters = filters,
-		group_by = '_year, _unit',
-		order_by = '_year asc, _unit asc',
+		group_by = '_unit',
+		order_by = '_unit asc',
 		as_list = True,
 		ignore_ifnull = True
 	)
 
+	result = get_result(data, timegrain, from_date, to_date)
 
-	# result given as year, unit -> convert it to end of period of that unit
-	result = convert_to_dates(data, timegrain)
-
-	# add missing data points for periods where there was no result
-	result = add_missing_values(result, timegrain, timespan, from_date, to_date)
 	chart_config = {
 		"labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
 		"datasets": [{
@@ -174,6 +171,41 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 
 	return chart_config
 
+def get_heatmap_chart_config(chart, filters, heatmap_year):
+	aggregate_function = get_aggregate_function(chart.chart_type)
+	value_field = chart.value_based_on or '1'
+	doctype = chart.document_type
+	datefield = chart.based_on
+	year = cint(heatmap_year) if heatmap_year else getdate(nowdate()).year
+	year_start_date = datetime.date(year, 1, 1).strftime('%Y-%m-%d')
+	next_year_start_date = datetime.date(year + 1, 1, 1).strftime('%Y-%m-%d')
+
+	filters.append([doctype, datefield, '>', "{date}".format(date=year_start_date), False])
+	filters.append([doctype, datefield, '<', "{date}".format(date=next_year_start_date), False])
+
+	if frappe.db.db_type == 'mariadb':
+		timestamp_field = 'unix_timestamp({datefield})'.format(datefield=datefield)
+	else:
+		timestamp_field = 'extract(epoch from timestamp {datefield})'.format(datefield=datefield)
+
+	data = dict(frappe.db.get_all(
+		doctype,
+		fields = [
+			timestamp_field,
+			'{aggregate_function}({value_field})'.format(aggregate_function=aggregate_function, value_field=value_field),
+		],
+		filters = filters,
+		group_by = 'date({datefield})'.format(datefield=datefield),
+		as_list = 1,
+		order_by = '{datefield} asc'.format(datefield=datefield),
+		ignore_ifnull = True
+	))
+
+	chart_config = {
+		'labels': [],
+		'dataPoints': data,
+	}
+	return chart_config
 
 def get_group_by_chart_config(chart, filters):
 
@@ -182,7 +214,7 @@ def get_group_by_chart_config(chart, filters):
 	group_by_field = chart.group_by_based_on
 	doctype = chart.document_type
 
-	data = frappe.db.get_all(
+	data = frappe.db.get_list(
 		doctype,
 		fields = [
 			'{} as name'.format(group_by_field),
@@ -223,75 +255,22 @@ def get_aggregate_function(chart_type):
 	}[chart_type]
 
 
-def convert_to_dates(data, timegrain):
-	""" Converts individual dates within data to the end of period """
-	result = []
-	for d in data:
-		if d[2] != 0:
-			if timegrain == 'Daily':
-				result.append([add_to_date('{:d}-01-01'.format(int(d[0])), days = d[1] - 1), d[2]])
-			elif timegrain == 'Weekly':
-				result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), weeks = d[1] + 1), days = -1), d[2]])
-			elif timegrain == 'Monthly':
-				result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=d[1]), days = -1), d[2]])
-			elif timegrain == 'Quarterly':
-				result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=d[1] * 3), days = -1), d[2]])
-			elif timegrain == 'Yearly':
-				result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=12), days = -1), d[2]])
-			result[-1][0] = getdate(result[-1][0])
-
-	return result
-
-def get_unit_function(doctype, datefield, timegrain):
-	unit_function = ''
-	if timegrain=='Daily':
-		if frappe.db.db_type == 'mariadb':
-			unit_function = 'dayofyear(`tab{doctype}`.{datefield})'.format(
-				doctype=doctype, datefield=datefield)
-		else:
-			unit_function = 'extract(doy from `tab{doctype}`.{datefield})'.format(
-				doctype=doctype, datefield=datefield)
-
-	else:
-		unit_function = 'extract({unit} from `tab{doctype}`.{datefield})'.format(
-			unit = timegrain[:-2].lower(), doctype=doctype, datefield=datefield)
-
-	return unit_function
-
-def add_missing_values(data, timegrain, timespan, from_date, to_date):
-	# add missing intervals
+def get_result(data, timegrain, from_date, to_date):
+	start_date = getdate(from_date)
+	end_date = getdate(to_date)
 	result = []
 
-	if timespan != 'All Time':
-		first_expected_date = get_period_ending(from_date, timegrain)
-		# fill out data before the first data point
-		first_data_point_date = data[0][0] if data else getdate(add_to_date(to_date, days=1))
-		while first_data_point_date > first_expected_date:
-			result.append([first_expected_date, 0.0])
-			first_expected_date = get_next_expected_date(first_expected_date, timegrain)
+	while start_date <= end_date:
+		next_date = get_next_expected_date(start_date, timegrain)
+		result.append([next_date, 0.0])
+		start_date = next_date
 
-	# fill data points and missing points
-	for i, d in enumerate(data):
-		result.append(d)
-
-		next_expected_date = get_next_expected_date(d[0], timegrain)
-
-		if i < len(data)-1:
-			next_date = data[i+1][0]
-		else:
-			# already reached at end of data, see if we need any more dates
-			next_date = getdate(nowdate())
-
-		# if next data point is earler than the expected date
-		# need to fill out missing data points
-		while next_date > next_expected_date:
-			# fill missing value
-			result.append([next_expected_date, 0.0])
-			next_expected_date = get_next_expected_date(next_expected_date, timegrain)
-
-	# add date for the last period (if missing)
-	if result and get_period_ending(to_date, timegrain) > result[-1][0]:
-		result.append([get_period_ending(to_date, timegrain), 0.0])
+	data_index = 0
+	if data:
+		for i, d in enumerate(result):
+			while data_index < len(data) and getdate(data[data_index][0]) <= d[0]:
+				d[1] += data[data_index][1]
+				data_index += 1
 
 	return result
 
@@ -320,17 +299,12 @@ def get_period_ending(date, timegrain):
 	return getdate(date)
 
 def get_week_ending(date):
-	# fun fact: week ends on the day before 1st Jan of the year.
-	# for 2019 it is Monday
+	# week starts on monday
+	from datetime import timedelta
+	start = date - timedelta(days = date.weekday())
+	end = start + timedelta(days=6)
 
-	week_of_the_year = int(date.strftime('%U'))
-
-	if week_of_the_year == 52:
-		date = add_to_date(date, years=1)
-	# first day of next week
-	date = add_to_date('{}-01-01'.format(date.year), weeks = (week_of_the_year%52) + 1)
-	# last day of this week
-	return add_to_date(date, days=-1)
+	return end
 
 def get_month_ending(date):
 	month_of_the_year = int(date.strftime('%m'))
@@ -397,11 +371,11 @@ class DashboardChart(Document):
 
 	def check_document_type(self):
 		if frappe.get_meta(self.document_type).issingle:
-			frappe.throw("You cannot create a dashboard chart from single DocTypes")
+			frappe.throw(_("You cannot create a dashboard chart from single DocTypes"))
 
 	def validate_custom_options(self):
 		if self.custom_options:
 			try:
 				json.loads(self.custom_options)
 			except ValueError as error:
-				frappe.throw("Invalid json added in the custom options: %s" % error)
+				frappe.throw(_("Invalid json added in the custom options: {0}").format(error))

frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py

@@ -17,10 +17,9 @@ class TestDashboardChart(unittest.TestCase):
 		self.assertEqual(get_period_ending('2019-04-10', 'Daily'),
 			getdate('2019-04-10'))
 
-		# fun fact: week ends on the day before 1st Jan of the year.
-		# for 2019 it is Monday
+		# week starts on monday
 		self.assertEqual(get_period_ending('2019-04-10', 'Weekly'),
-			getdate('2019-04-15'))
+			getdate('2019-04-14'))
 
 		self.assertEqual(get_period_ending('2019-04-10', 'Monthly'),
 			getdate('2019-04-30'))
@@ -133,6 +132,34 @@ class TestDashboardChart(unittest.TestCase):
 
 		frappe.db.rollback()
 
+	def test_weekly_dashboard_chart(self):
+		insert_test_records()
+
+		if frappe.db.exists('Dashboard Chart', 'Test Weekly Dashboard Chart'):
+			frappe.delete_doc('Dashboard Chart', 'Test Weekly Dashboard Chart')
+
+		frappe.get_doc(dict(
+			doctype = 'Dashboard Chart',
+			chart_name = 'Test Weekly Dashboard Chart',
+			chart_type = 'Sum',
+			document_type = 'Communication',
+			based_on = 'communication_date',
+			value_based_on = 'rating',
+			timespan = 'Select Date Range',
+			time_interval = 'Weekly',
+			from_date = datetime(2018, 12, 30),
+			to_date = datetime(2019, 1, 15),
+			filters_json = '[]',
+			timeseries = 1
+		)).insert()
+
+		result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
+
+		self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 0.0])
+		self.assertEqual(result.get('labels'), [formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')])
+
+		frappe.db.rollback()
+
 	def test_group_by_chart_type(self):
 		if frappe.db.exists('Dashboard Chart', 'Test Group By Dashboard Chart'):
 			frappe.delete_doc('Dashboard Chart', 'Test Group By Dashboard Chart')
@@ -155,17 +182,16 @@ class TestDashboardChart(unittest.TestCase):
 
 		frappe.db.rollback()
 
-	def test_dashboard_with_single_doctype(self):
-		if frappe.db.exists('Dashboard Chart', 'Test Single DocType In Dashboard Chart'):
-			frappe.delete_doc('Dashboard Chart', 'Test Single DocType In Dashboard Chart')
+def insert_test_records():
+	create_new_communication(datetime(2019, 1, 10), 100)
+	create_new_communication(datetime(2019, 1, 6), 200)
+	create_new_communication(datetime(2019, 1, 8), 300)
 
-		chart_doc = frappe.get_doc(dict(
-			doctype = 'Dashboard Chart',
-			chart_name = 'Test Single DocType In Dashboard Chart',
-			chart_type = 'Count',
-			document_type = 'System Settings',
-			group_by_based_on = 'Created On',
-			filters_json = '{}',
-		))
-
-		self.assertRaises(frappe.ValidationError, chart_doc.insert)
+def create_new_communication(date, rating):
+	communication = {
+		'doctype': 'Communication',
+		'subject': 'Test Communication',
+		'rating': rating,
+		'communication_date': date
+	}
+	frappe.get_doc(communication).insert()

frappe/desk/doctype/desk_page/desk_page.js

@@ -2,16 +2,22 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Desk Page', {
-	setup: function(frm) {
+	refresh: function(frm) {
+		frm.enable_save();
 		frm.get_field("is_standard").toggle(frappe.boot.developer_mode);
 		frm.get_field("extends_another_page").toggle(frappe.boot.developer_mode);
-		if (!frappe.boot.developer_mode || frm.doc.for_user) {
+		frm.get_field("developer_mode_only").toggle(frappe.boot.developer_mode);
+
+		if (frm.doc.for_user) {
+			frm.set_df_property("extends", "read_only", true);
+		}
+
+		if (frm.doc.for_user || (frm.doc.is_standard && !frappe.boot.developer_mode)) {
 			frm.trigger('disable_form');
 		}
 	},
 
 	disable_form: function(frm) {
-		frm.set_read_only();
 		frm.fields
 			.filter(field => field.has_input)
 			.forEach(field => {

frappe/desk/doctype/desk_page/desk_page.json

@@ -8,8 +8,8 @@
  "engine": "InnoDB",
  "field_order": [
   "label",
-  "extends",
   "for_user",
+  "extends",
   "module",
   "category",
   "restrict_to_domain",
@@ -21,6 +21,7 @@
   "disable_user_customization",
   "pin_to_top",
   "pin_to_bottom",
+  "hide_custom",
   "section_break_2",
   "charts_label",
   "charts",
@@ -170,7 +171,7 @@
    "search_index": 1
   },
   {
-   "depends_on": "eval:doc.extends_another_page == 1",
+   "depends_on": "eval:doc.extends_another_page == 1 || doc.for_user",
    "fieldname": "extends",
    "fieldtype": "Link",
    "in_standard_filter": 1,
@@ -188,11 +189,18 @@
    "fieldname": "onboarding",
    "fieldtype": "Link",
    "label": "Onboarding",
-   "options": "Onboarding"
+   "options": "Module Onboarding"
+  },
+  {
+   "default": "0",
+   "description": "Checking this will hide custom doctypes and reports cards in Links section",
+   "fieldname": "hide_custom",
+   "fieldtype": "Check",
+   "label": "Hide Custom DocTypes and Reports"
   }
  ],
  "links": [],
- "modified": "2020-04-26 12:21:46.205079",
+ "modified": "2020-05-18 19:17:27.206646",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Desk Page",

frappe/desk/doctype/desk_shortcut/desk_shortcut.json

@@ -6,9 +6,9 @@
  "engine": "InnoDB",
  "field_order": [
   "type",
-  "label",
-  "column_break_4",
   "link_to",
+  "column_break_4",
+  "label",
   "icon",
   "restrict_to_domain",
   "section_break_5",
@@ -23,7 +23,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Type",
-   "options": "DocType\nReport\nPage",
+   "options": "DocType\nReport\nPage\nDashboard",
    "reqd": 1
   },
   {
@@ -81,13 +81,14 @@
   {
    "fieldname": "label",
    "fieldtype": "Data",
+   "in_list_view": 1,
    "label": "Label",
    "reqd": 1
   }
  ],
  "istable": 1,
  "links": [],
- "modified": "2020-04-07 19:04:23.645198",
+ "modified": "2020-05-14 16:02:15.420993",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Desk Shortcut",

frappe/desk/doctype/event/test_event.py

@@ -71,7 +71,7 @@ class TestEvent(unittest.TestCase):
 		ev = frappe.get_doc(self.test_records[0]).insert()
 
 		add({
-			"assign_to": "test@example.com",
+			"assign_to": ["test@example.com"],
 			"doctype": "Event",
 			"name": ev.name,
 			"description": "Test Assignment"
@@ -83,7 +83,7 @@ class TestEvent(unittest.TestCase):
 
 		# add another one
 		add({
-			"assign_to": self.test_user,
+			"assign_to": [self.test_user],
 			"doctype": "Event",
 			"name": ev.name,
 			"description": "Test Assignment"

frappe/desk/doctype/module_onboarding/module_onboarding.js

@@ -1,7 +1,7 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Onboarding", {
+frappe.ui.form.on("Module Onboarding", {
 	refresh: function(frm) {
 		frappe.boot.developer_mode &&
 			frm.set_intro(

frappe/desk/doctype/module_onboarding/module_onboarding.json

@@ -90,10 +90,10 @@
   }
  ],
  "links": [],
- "modified": "2020-05-01 19:37:21.492405",
+ "modified": "2020-05-18 19:42:39.738869",
  "modified_by": "Administrator",
  "module": "Desk",
- "name": "Onboarding",
+ "name": "Module Onboarding",
  "owner": "Administrator",
  "permissions": [
   {
@@ -118,6 +118,7 @@
    "share": 1
   }
  ],
+ "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1

frappe/desk/doctype/module_onboarding/module_onboarding.py

@@ -8,10 +8,10 @@ from frappe.model.document import Document
 from frappe.modules.export_file import export_to_files
 
 
-class Onboarding(Document):
+class ModuleOnboarding(Document):
 	def on_update(self):
 		if frappe.conf.developer_mode:
-			export_to_files(record_list=[['Onboarding', self.name]], record_module=self.module)
+			export_to_files(record_list=[['Module Onboarding', self.name]], record_module=self.module)
 
 			for step in self.steps:
 				export_to_files(record_list=[['Onboarding Step', step.step]], record_module=self.module)

frappe/desk/doctype/module_onboarding/test_module_onboarding.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestModuleOnboarding(unittest.TestCase):
+	pass

frappe/desk/doctype/note/note.js

@@ -1,9 +1,9 @@
 frappe.ui.form.on("Note", {
 	refresh: function(frm) {
-		if(frm.doc.__islocal) {
+		if (frm.doc.__islocal) {
 			frm.events.set_editable(frm, true);
 		} else {
-			if(!frm.doc.content) {
+			if (!frm.doc.content) {
 				frm.doc.content = "<span></span>";
 			}
 
@@ -18,16 +18,15 @@ frappe.ui.form.on("Note", {
 		// hide all fields other than content
 
 		// no permission
-		if(editable && !frm.perm[0].write) return;
+		if (editable && !frm.perm[0].write) return;
 
 		// content read_only
-		frm.set_df_property("content", "read_only", editable ? 0: 1);
+		frm.set_df_property("content", "read_only", editable ? 0 : 1);
 
 		// hide all other fields
 		$.each(frm.fields_dict, function(fieldname) {
-
-			if(fieldname !== "content") {
-				frm.set_df_property(fieldname, "hidden", editable ? 0: 1);
+			if (fieldname !== "content") {
+				frm.set_df_property(fieldname, "hidden", editable ? 0 : 1);
 			}
 		});
 
@@ -39,3 +38,16 @@ frappe.ui.form.on("Note", {
 		frm.is_note_editable = editable;
 	}
 });
+
+frappe.tour['Note'] = [
+	{
+		fieldname: "title",
+		title: "Title of the Note",
+		description: "This is the name by which the note will be saved, you can change this later",
+	},
+	{
+		fieldname: "public",
+		title: "Sets the Note to Public",
+		description: "You can change the visibility of the note with this, setting it to public will allow other users to view it.",
+	},
+];

frappe/desk/doctype/note/test_note.py

@@ -20,7 +20,7 @@ class TestNote(unittest.TestCase):
 		note = self.insert_note()
 		note.title = 'test note 1'
 		note.content = '1'
-		note.save()
+		note.save(ignore_version=False)
 
 		version = frappe.get_doc('Version', dict(docname=note.name))
 		data = version.get_data()
@@ -33,7 +33,7 @@ class TestNote(unittest.TestCase):
 
 		# test add
 		note.append('seen_by', {'user': 'Administrator'})
-		note.save()
+		note.save(ignore_version=False)
 
 		version = frappe.get_doc('Version', dict(docname=note.name))
 		data = version.get_data()
@@ -48,7 +48,7 @@ class TestNote(unittest.TestCase):
 
 		# test row change
 		note.seen_by[0].user = 'Guest'
-		note.save()
+		note.save(ignore_version=False)
 
 		version = frappe.get_doc('Version', dict(docname=note.name))
 		data = version.get_data()
@@ -62,7 +62,7 @@ class TestNote(unittest.TestCase):
 
 		# test remove
 		note.seen_by = []
-		note.save()
+		note.save(ignore_version=False)
 
 		version = frappe.get_doc('Version', dict(docname=note.name))
 		data = version.get_data()

frappe/desk/doctype/notification_log/test_notification_log.py

@@ -13,7 +13,7 @@ class TestNotificationLog(unittest.TestCase):
 		user = get_user()
 
 		assign_task({
-			"assign_to": user,
+			"assign_to": [user],
 			"doctype": 'ToDo',
 			"name": todo.name,
 			"description": todo.description

frappe/desk/doctype/number_card/number_card.py

@@ -70,7 +70,7 @@ def get_result(doc, to_date=None):
 	if to_date:
 		filters.append([doc.document_type, 'creation', '<', to_date, False])
 
-	res = frappe.db.get_all(doc.document_type, fields=fields, filters=filters)
+	res = frappe.db.get_list(doc.document_type, fields=fields, filters=filters)
 	number = res[0]['result'] if res else 0
 
 	return cint(number)

frappe/desk/doctype/onboarding_step/onboarding_step.js

@@ -25,6 +25,24 @@ frappe.ui.form.on("Onboarding Step", {
 		}
 	},
 
+	action: function(frm) {
+		if (frm.doc.action == "Show Form Tour") {
+			frm.fields_dict.reference_document.set_description(`You need to add the steps in the contoller JS file. For example: <code>note.js</code>
+<pre class="small text-muted"><code>
+frappe.tour['Note'] = [
+	{
+		fieldname: "title",
+		title: "Title of the Note",
+		description: "...",
+	}
+];
+</code></pre>
+				`);
+		} else {
+			frm.fields_dict.reference_document.set_description(null);
+		}
+	},
+
 	disable_form: function(frm) {
 		frm.set_read_only();
 		frm.fields

frappe/desk/doctype/onboarding_step/onboarding_step.json

@@ -15,10 +15,16 @@
   "action",
   "column_break_7",
   "reference_document",
+  "show_full_form",
+  "is_single",
   "reference_report",
   "report_reference_doctype",
   "report_type",
   "report_description",
+  "path",
+  "callback_title",
+  "callback_message",
+  "validate_action",
   "field",
   "value_to_validate",
   "video_url"
@@ -57,7 +63,7 @@
    "fieldname": "action",
    "fieldtype": "Select",
    "label": "Action",
-   "options": "Create Entry\nUpdate Settings\nView Report\nWatch Video",
+   "options": "Create Entry\nUpdate Settings\nShow Form Tour\nView Report\nGo to Page\nWatch Video",
    "reqd": 1
   },
   {
@@ -65,10 +71,11 @@
    "fieldtype": "Column Break"
   },
   {
-   "depends_on": "eval:doc.action == \"Create Entry\" || doc.action == \"Update Settings\"",
+   "depends_on": "eval:doc.action == \"Create Entry\" || doc.action == \"Update Settings\" || doc.action == \"Create Entry\" || doc.action == \"Show Form Tour\"",
    "fieldname": "reference_document",
    "fieldtype": "Link",
    "label": "Reference Document",
+   "mandatory_depends_on": "eval:doc.action == \"Create Entry\" || doc.action == \"Update Settings\" || doc.action == \"Create Entry\" || doc.action == \"Show Form Tour\"",
    "options": "DocType"
   },
   {
@@ -83,7 +90,8 @@
    "depends_on": "eval:doc.action == \"Watch Video\"",
    "fieldname": "video_url",
    "fieldtype": "Data",
-   "label": "Video URL"
+   "label": "Video URL",
+   "mandatory_depends_on": "eval:doc.action == \"Watch Video\""
   },
   {
    "depends_on": "eval:doc.action == \"View Report\"",
@@ -101,17 +109,19 @@
    "label": "Is Skipped"
   },
   {
-   "depends_on": "eval:doc.action == \"Update Settings\"",
+   "depends_on": "eval:doc.action == \"Update Settings\" && doc.validate_action",
    "fieldname": "field",
    "fieldtype": "Select",
-   "label": "Field"
+   "label": "Field",
+   "mandatory_depends_on": "eval:doc.action == \"Update Settings\" && doc.validate_action"
   },
   {
-   "depends_on": "eval:doc.action == \"Update Settings\"",
+   "depends_on": "eval:doc.action == \"Update Settings\" && doc.validate_action",
    "description": "Use % for any non empty value.",
    "fieldname": "value_to_validate",
    "fieldtype": "Data",
-   "label": "Value to Validate"
+   "label": "Value to Validate",
+   "mandatory_depends_on": "eval:doc.action == \"Update Settings\" && doc.validate_action"
   },
   {
    "depends_on": "eval:doc.action == \"View Report\"",
@@ -127,10 +137,54 @@
    "fieldtype": "Data",
    "label": "Report Reference Doctype",
    "read_only": 1
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.action == \"Create Entry\" || doc.action == \"Update Settings\" || doc.action == \"Create Entry\" || doc.action == \"Show Form Tour\"",
+   "fetch_from": "reference_document.issingle",
+   "fieldname": "is_single",
+   "fieldtype": "Check",
+   "label": "Is Single"
+  },
+  {
+   "depends_on": "eval:doc.action == \"Go to Page\"",
+   "description": "Example: #Tree/Account",
+   "fieldname": "path",
+   "fieldtype": "Data",
+   "label": "Path",
+   "mandatory_depends_on": "eval:doc.action == \"Go to Page\""
+  },
+  {
+   "depends_on": "eval:doc.action == \"Go to Page\"",
+   "fieldname": "callback_title",
+   "fieldtype": "Data",
+   "label": "Callback Title"
+  },
+  {
+   "depends_on": "eval:doc.action == \"Go to Page\"",
+   "description": "This will be shown in a modal after routing",
+   "fieldname": "callback_message",
+   "fieldtype": "Small Text",
+   "label": "Callback Message"
+  },
+  {
+   "default": "1",
+   "depends_on": "eval:doc.action == \"Update Settings\"",
+   "fieldname": "validate_action",
+   "fieldtype": "Check",
+   "label": "Validate Field"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.action == \"Create Entry\"",
+   "description": "Show full form instead of a quick entry modal",
+   "fieldname": "show_full_form",
+   "fieldtype": "Check",
+   "label": "Show Full Form?"
   }
  ],
  "links": [],
- "modified": "2020-05-04 12:53:19.276952",
+ "modified": "2020-05-18 19:42:30.435604",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Onboarding Step",
@@ -159,6 +213,7 @@
   }
  ],
  "quick_entry": 1,
+ "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1

frappe/desk/doctype/onboarding_step/onboarding_step.py

@@ -10,3 +10,7 @@ class OnboardingStep(Document):
 	def before_export(self, doc):
 		doc.is_complete = 0
 		doc.is_skipped = 0
+
+	def validate(self):
+		if self.action == "Go to Page":
+			self.is_mandatory = 0

frappe/desk/form/assign_to.py

@@ -11,6 +11,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create
 	get_title, get_title_html
 import frappe.utils
 import frappe.share
+import json
 
 class DuplicateToDoError(frappe.ValidationError): pass
 
@@ -19,17 +20,17 @@ def get(args=None):
 	if not args:
 		args = frappe.local.form_dict
 
-	return frappe.get_all('ToDo', fields = ['owner', 'description'], filters = dict(
+	return frappe.get_all('ToDo', fields=['owner', 'name'], filters=dict(
 		reference_type = args.get('doctype'),
 		reference_name = args.get('name'),
 		status = ('!=', 'Cancelled')
-	), limit = 5)
+	), limit=5)
 
 @frappe.whitelist()
 def add(args=None):
 	"""add in someone's to do list
 		args = {
-			"assign_to": ,
+			"assign_to": [],
 			"doctype": ,
 			"name": ,
 			"description": ,
@@ -40,56 +41,68 @@ def add(args=None):
 	if not args:
 		args = frappe.local.form_dict
 
-	if frappe.db.sql("""SELECT `owner`
-		FROM `tabToDo`
-		WHERE `reference_type`=%(doctype)s
-		AND `reference_name`=%(name)s
-		AND `status`='Open'
-		AND `owner`=%(assign_to)s""", args):
-		frappe.throw(_("Already in user's To Do list"), DuplicateToDoError)
-	else:
-		from frappe.utils import nowdate
+	users_with_duplicate_todo = []
+	shared_with_users = []
 
-		if not args.get('description'):
-			args['description'] = _('Assignment for {0} {1}').format(args['doctype'], args['name'])
-
-		d = frappe.get_doc({
-			"doctype":"ToDo",
-			"owner": args['assign_to'],
+	for assign_to in frappe.parse_json(args.get("assign_to")):
+		filters = {
 			"reference_type": args['doctype'],
 			"reference_name": args['name'],
-			"description": args.get('description'),
-			"priority": args.get("priority", "Medium"),
 			"status": "Open",
-			"date": args.get('date', nowdate()),
-			"assigned_by": args.get('assigned_by', frappe.session.user),
-			'assignment_rule': args.get('assignment_rule')
-		}).insert(ignore_permissions=True)
+			"owner": assign_to
+		}
 
-		# set assigned_to if field exists
-		if frappe.get_meta(args['doctype']).get_field("assigned_to"):
-			frappe.db.set_value(args['doctype'], args['name'], "assigned_to", args['assign_to'])
+		if frappe.get_all("ToDo", filters=filters):
+			users_with_duplicate_todo.append(assign_to)
+		else:
+			from frappe.utils import nowdate
 
-		doc = frappe.get_doc(args['doctype'], args['name'])
+			if not args.get('description'):
+				args['description'] = _('Assignment for {0} {1}').format(args['doctype'], args['name'])
 
-		# if assignee does not have permissions, share
-		if not frappe.has_permission(doc=doc, user=args['assign_to']):
-			frappe.share.add(doc.doctype, doc.name, args['assign_to'])
-			frappe.msgprint(_('Shared with user {0} with read access').format(args['assign_to']), alert=True)
+			d = frappe.get_doc({
+				"doctype": "ToDo",
+				"owner": assign_to,
+				"reference_type": args['doctype'],
+				"reference_name": args['name'],
+				"description": args.get('description'),
+				"priority": args.get("priority", "Medium"),
+				"status": "Open",
+				"date": args.get('date', nowdate()),
+				"assigned_by": args.get('assigned_by', frappe.session.user),
+				'assignment_rule': args.get('assignment_rule')
+			}).insert(ignore_permissions=True)
 
-		# make this document followed by assigned user
-		follow_document(args['doctype'], args['name'], args['assign_to'])
+			# set assigned_to if field exists
+			if frappe.get_meta(args['doctype']).get_field("assigned_to"):
+				frappe.db.set_value(args['doctype'], args['name'], "assigned_to", assign_to)
 
-	# notify
-	notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',\
-			description=args.get("description"))
+			doc = frappe.get_doc(args['doctype'], args['name'])
+
+			# if assignee does not have permissions, share
+			if not frappe.has_permission(doc=doc, user=assign_to):
+				frappe.share.add(doc.doctype, doc.name, assign_to)
+				shared_with_users.append(assign_to)
+
+			# make this document followed by assigned user
+			follow_document(args['doctype'], args['name'], assign_to)
+
+			# notify
+			notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',
+				description=args.get("description"))
+
+	if shared_with_users:
+		user_list = format_message_for_assign_to(shared_with_users)
+		frappe.msgprint(_("Shared with the following Users with Read access:{0}").format(user_list, alert=True))
+
+	if users_with_duplicate_todo:
+		user_list = format_message_for_assign_to(users_with_duplicate_todo)
+		frappe.msgprint(_("Already in the following Users ToDo list:{0}").format(user_list, alert=True))
 
 	return get(args)
 
 @frappe.whitelist()
 def add_multiple(args=None):
-	import json
-
 	if not args:
 		args = frappe.local.form_dict
 
@@ -183,3 +196,5 @@ def notify_assignment(assigned_by, owner, doc_type, doc_name, action='CLOSE',
 
 	enqueue_create_notification(owner, notification_doc)
 
+def format_message_for_assign_to(users):
+	return "<br><br>" + "<br>".join(users)

frappe/desk/form/linked_with.py

@@ -13,7 +13,7 @@ from frappe.modules import load_doctype_module
 
 
 @frappe.whitelist()
-def get_submitted_linked_docs(doctype, name, docs=None):
+def get_submitted_linked_docs(doctype, name, docs=None, linked=None):
 	"""
 	Get all nested submitted linked doctype linkinfo
 
@@ -31,12 +31,26 @@ def get_submitted_linked_docs(doctype, name, docs=None):
 	if not docs:
 		docs = []
 
+	if not linked:
+		linked = {}
+
 	linkinfo = get_linked_doctypes(doctype)
 	linked_docs = get_linked_docs(doctype, name, linkinfo)
 
 	link_count = 0
 	for link_doctype, link_names in linked_docs.items():
+		if link_doctype not in linked:
+			linked[link_doctype] = []
+
 		for link in link_names:
+			if link['name'] == name:
+				continue
+
+			if linked and name in linked[link_doctype]:
+				continue
+
+			linked[link_doctype].append(link['name'])
+
 			docinfo = link.update({"doctype": link_doctype})
 			validated_doc = validate_linked_doc(docinfo)
 
@@ -47,7 +61,7 @@ def get_submitted_linked_docs(doctype, name, docs=None):
 			if link.name in [doc.get("name") for doc in docs]:
 				continue
 
-			links = get_submitted_linked_docs(link_doctype, link.name, docs)
+			links = get_submitted_linked_docs(link_doctype, link.name, docs, linked)
 			docs.append({
 				"doctype": link_doctype,
 				"name": link.name,

frappe/desk/notifications.py

@@ -212,7 +212,10 @@ def get_notification_config():
 def get_filters_for(doctype):
 	'''get open filters for doctype'''
 	config = get_notification_config()
-	return config.get("for_doctype").get(doctype, {})
+	doctype_config = config.get("for_doctype").get(doctype, {})
+	filters = doctype_config if not isinstance(doctype_config, string_types) else None
+
+	return filters
 
 @frappe.whitelist()
 @frappe.read_only()

frappe/desk/page/setup_wizard/install_fixtures.py

@@ -14,6 +14,7 @@ def install():
 	update_global_search_doctypes()
 	setup_email_linking()
 	sync_dashboards()
+	add_unsubscribe()
 
 @frappe.whitelist()
 def update_genders():
@@ -37,3 +38,15 @@ def setup_email_linking():
 		"email_id": "email_linking@example.com",
 	})
 	doc.insert(ignore_permissions=True, ignore_if_duplicate=True)
+
+def add_unsubscribe():
+	email_unsubscribe = [
+		{"email": "admin@example.com", "global_unsubscribe": 1},
+		{"email": "guest@example.com", "global_unsubscribe": 1}
+	]
+
+	for unsubscribe in email_unsubscribe:
+		if not frappe.get_all("Email Unsubscribe", filters=unsubscribe):
+			doc = frappe.new_doc("Email Unsubscribe")
+			doc.update(unsubscribe)
+			doc.insert(ignore_permissions=True)

frappe/desk/page/user_profile/user_profile.js

@@ -108,21 +108,6 @@ class UserProfile {
 		});
 	}
 
-	get_years_since_creation() {
-		//Get years since user account created
-		this.user_creation = frappe.boot.user.creation;
-		let creation_year = this.get_year(this.user_creation);
-		let current_year = this.get_year(frappe.datetime.now_date());
-		let years_list = [];
-		for (var year = current_year; year >= creation_year; year--) {
-			years_list.push(year);
-		}
-		return years_list;
-	}
-
-	get_year(date_str) {
-		return date_str.substring(0, date_str.indexOf('-'));
-	}
 
 	render_line_chart() {
 		this.line_chart_filters = [['Energy Point Log', 'user', '=', this.user_id, false]];
@@ -246,8 +231,8 @@ class UserProfile {
 	create_heatmap_chart_filters() {
 		let filters = [
 			{
-				label: this.get_year(frappe.datetime.now_date()),
-				options: this.get_years_since_creation(),
+				label: frappe.dashboard_utils.get_year(frappe.datetime.now_date()),
+				options: frappe.dashboard_utils.get_years_since_creation(frappe.boot.user.creation),
 				action: (selected_item) => {
 					this.update_heatmap_data(frappe.datetime.obj_to_str(selected_item));
 				}

frappe/desk/query_report.py

@@ -62,8 +62,16 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 		ljust_list(res, 6)
 
 	if report.custom_columns:
+		# Original query columns, needed to reorder data as per custom columns
+		query_columns = columns
+		# Reordered columns
 		columns = json.loads(report.custom_columns)
+
+		if report.report_type == 'Query Report':
+			result = reorder_data_for_custom_columns(columns, query_columns, result)
+
 		result = add_data_to_custom_columns(columns, result)
+
 	if custom_columns:
 		result = add_data_to_custom_columns(custom_columns, result)
 
@@ -208,6 +216,23 @@ def add_data_to_custom_columns(columns, result):
 
 	return data
 
+def reorder_data_for_custom_columns(custom_columns, columns, result):
+	reordered_result = []
+	columns = [col.split(":")[0] for col in columns]
+
+	for res in result:
+		r = []
+		for col in custom_columns:
+			try:
+				idx = columns.index(col.get("label"))
+				r.append(res[idx])
+			except ValueError:
+				pass
+
+		reordered_result.append(r)
+
+	return reordered_result
+
 def get_prepared_report_result(report, filters, dn="", user=None):
 	latest_report_data = {}
 	doc = None

frappe/email/doctype/document_follow/test_document_follow.py

@@ -14,7 +14,7 @@ class TestDocumentFollow(unittest.TestCase):
 		event_doc = get_event()
 
 		event_doc.description = "This is a test description for sending mail"
-		event_doc.save()
+		event_doc.save(ignore_version=False)
 
 		doc = document_follow.follow_document("Event", event_doc.name , user.name, force=True)
 		self.assertEquals(doc.user, user.name)
@@ -45,12 +45,12 @@ def get_event():
 	return doc
 
 def get_user():
-		doc = frappe.new_doc("User")
-		doc.email = "test@docsub.com"
-		doc.first_name = "Test"
-		doc.last_name = "User"
-		doc.send_welcome_email = 0
-		doc.document_follow_notify = 1
-		doc.document_follow_frequency = "Hourly"
-		doc.insert()
-		return doc
+	doc = frappe.new_doc("User")
+	doc.email = "test@docsub.com"
+	doc.first_name = "Test"
+	doc.last_name = "User"
+	doc.send_welcome_email = 0
+	doc.document_follow_notify = 1
+	doc.document_follow_frequency = "Hourly"
+	doc.insert()
+	return doc

frappe/email/doctype/newsletter/newsletter.json

@@ -17,7 +17,7 @@
   "subject",
   "message",
   "send_unsubscribe_link",
-  "send_attachements",
+  "send_attachments",
   "published",
   "route",
   "test_the_newsletter",
@@ -73,12 +73,6 @@
    "fieldtype": "Check",
    "label": "Send Unsubscribe Link"
   },
-  {
-   "default": "0",
-   "fieldname": "send_attachements",
-   "fieldtype": "Check",
-   "label": "Send Attachements"
-  },
   {
    "default": "0",
    "fieldname": "published",
@@ -127,6 +121,12 @@
   {
    "fieldname": "column_break_2",
    "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "send_attachments",
+   "fieldtype": "Check",
+   "label": "Send Attachments"
   }
  ],
  "has_web_view": 1,
@@ -135,7 +135,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 3,
- "modified": "2020-03-02 06:26:51.622521",
+ "modified": "2020-05-12 18:09:40.137138",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",

frappe/email/doctype/newsletter/newsletter.py

@@ -67,7 +67,7 @@ class Newsletter(WebsiteGenerator):
 			frappe.db.auto_commit_on_many_writes = True
 
 		attachments = []
-		if self.send_attachements:
+		if self.send_attachments:
 			files = frappe.get_all("File", fields=["name"], filters={"attached_to_doctype": "Newsletter",
 				"attached_to_name": self.name}, order_by="creation desc")
 

frappe/exceptions.py

@@ -49,6 +49,11 @@ class Redirect(Exception):
 class CSRFTokenError(Exception):
 	http_status_code = 400
 
+
+class TooManyRequestsError(Exception):
+	http_status_code = 429
+
+
 class ImproperDBConfigurationError(Exception):
 	"""
 	Used when frappe detects that database or tables are not properly

frappe/frappeclient.py

@@ -15,6 +15,9 @@ class AuthError(Exception):
 class SiteExpiredError(Exception):
 	pass
 
+class SiteUnreachableError(Exception):
+	pass
+
 class FrappeException(Exception):
 	pass
 
@@ -53,9 +56,16 @@ class FrappeClient(object):
 
 		if r.status_code==200 and r.json().get('message') in ("Logged In", "No App"):
 			return r.json()
+		elif r.status_code == 502:
+			raise SiteUnreachableError
 		else:
-			if json.loads(r.text).get('exc_type') == "SiteExpiredError":
-				raise SiteExpiredError
+			try:
+				error = json.loads(r.text)
+				if error.get('exc_type') == "SiteExpiredError":
+					raise SiteExpiredError
+			except json.decoder.JSONDecodeError:
+				error = r.text
+				print(error)
 			raise AuthError
 
 	def setup_key_authentication_headers(self):

frappe/integrations/doctype/google_contacts/google_contacts.py

@@ -218,7 +218,7 @@ def insert_contacts_to_google_contacts(doc, method=None):
 	emailAddresses = [{"value": email_id.email_id} for email_id in doc.email_ids]
 
 	try:
-		contact = google_contacts.people().createContact(parent='people/me', body={"names": [names],"phoneNumbers": phoneNumbers,
+		contact = google_contacts.people().createContact(body={"names": [names],"phoneNumbers": phoneNumbers,
 			"emailAddresses": emailAddresses}).execute()
 		frappe.db.set_value("Contact", doc.name, "google_contacts_id", contact.get("resourceName"))
 	except HttpError as err:

frappe/integrations/frappe_providers/__init__.py

@@ -0,0 +1,14 @@
+# imports - standard imports
+import sys
+
+# imports - module imports
+from frappe.integrations.frappe_providers.frappecloud import frappecloud_migrator
+
+
+def migrate_to(local_site, frappe_provider):
+	if frappe_provider in ("frappe.cloud", "frappecloud.com"):
+		frappe_provider = "frappecloud.com"
+		return frappecloud_migrator(local_site, frappe_provider)
+	else:
+		print("{} is not supported yet".format(frappe_provider))
+		sys.exit(1)

frappe/integrations/frappe_providers/frappecloud.py

@@ -0,0 +1,268 @@
+# imports - standard imports
+import getpass
+import json
+import re
+import sys
+
+# imports - third party imports
+import click
+from html2text import html2text
+import requests
+
+# imports - module imports
+import frappe
+import frappe.utils.backups
+from frappe.utils import get_installed_apps_info
+from frappe.utils.commands import render_table, add_line_after
+
+
+def get_new_site_options():
+	site_options_sc = session.post(options_url)
+
+	if site_options_sc.ok:
+		site_options = site_options_sc.json()["message"]
+		return site_options
+	else:
+		print("Couldn't retrive New site information: {}".format(site_options_sc.status_code))
+
+
+def is_valid_subdomain(subdomain):
+	if len(subdomain) < 5:
+		print("Subdomain too short. Use 5 or more characters")
+		return False
+	matched = re.match("^[a-z0-9][a-z0-9-]*[a-z0-9]$", subdomain)
+	if matched:
+		return True
+	print("Subdomain contains invalid characters. Use lowercase characters, numbers and hyphens")
+
+
+def is_subdomain_available(subdomain):
+	res = session.post(site_exists_url, {"subdomain": subdomain})
+	if res.ok:
+		available = not res.json()["message"]
+		if not available:
+			print("Subdomain already exists! Try another one")
+
+		return available
+
+
+def render_plan_table(plans_list):
+	plans_table = []
+
+	# title row
+	visible_headers = ["name", "cpu_time_per_day"]
+	plans_table.append(["Plan", "CPU Time"])
+
+	# all rows
+	for plan in plans_list:
+		plan, cpu_time = [plan[header] for header in visible_headers]
+		plans_table.append([plan, "{} hour{}/day".format(cpu_time, "" if cpu_time < 2 else "s")])
+
+	render_table(plans_table)
+
+
+@add_line_after
+def choose_plan(plans_list):
+	print("{} plans available".format(len(plans_list)))
+	available_plans = [plan["name"] for plan in plans_list]
+	render_plan_table(plans_list)
+
+	while True:
+		input_plan = click.prompt("Select Plan").strip()
+		if input_plan in available_plans:
+			print("{} Plan selected ✅".format(input_plan))
+			return input_plan
+		else:
+			print("Invalid Selection ❌")
+
+
+@add_line_after
+def check_app_compat(available_group):
+	is_compat = True
+	incompatible_apps, filtered_apps, branch_msgs = [], [], []
+	existing_group = [(app["app_name"], app["branch"]) for app in get_installed_apps_info()]
+	print("Checking availability of existing app group")
+
+	for (app, branch) in existing_group:
+		info = [ (a["name"], a["branch"]) for a in available_group["apps"] if a["scrubbed"] == app ]
+		if info:
+			app_title, available_branch = info[0]
+
+			if branch != available_branch:
+				print("⚠️  App {}:{} => {}".format(app, branch, available_branch))
+				branch_msgs.append([app, branch, available_branch])
+				filtered_apps.append(app_title)
+				is_compat = False
+
+			else:
+				print("✅ App {}:{}".format(app, branch))
+				filtered_apps.append(app_title)
+
+		else:
+			incompatible_apps.append(app)
+			print("❌ App {}:{}".format(app, branch))
+			is_compat = False
+
+	start_msg = "\nSelecting this group will "
+	incompatible_apps = ("\n\nDrop the following apps:\n" + "\n".join(incompatible_apps)) if incompatible_apps else ""
+	branch_change = ("\n\nUpgrade the following apps:\n" + "\n".join(["{}: {} => {}".format(*x) for x in branch_msgs])) if branch_msgs else ""
+	changes = (incompatible_apps + branch_change) or "be perfect for you :)"
+	warning_message = start_msg + changes
+	print(warning_message)
+
+	return is_compat, filtered_apps
+
+
+def render_group_table(app_groups):
+	# title row
+	app_groups_table = [["#", "App Group", "Apps"]]
+
+	# all rows
+	for idx, app_group in enumerate(app_groups):
+		apps_list = ", ".join(["{}:{}".format(app["scrubbed"], app["branch"]) for app in app_group["apps"]])
+		row = [idx + 1, app_group["name"], apps_list]
+		app_groups_table.append(row)
+
+	render_table(app_groups_table)
+
+
+@add_line_after
+def filter_apps(app_groups):
+	render_group_table(app_groups)
+
+	while True:
+		app_group_index = click.prompt("Select App Group Number", type=int) - 1
+		try:
+			if app_group_index == -1:
+				raise IndexError
+			selected_group = app_groups[app_group_index]
+		except IndexError:
+			print("Invalid Selection ❌")
+			continue
+
+		is_compat, filtered_apps = check_app_compat(selected_group)
+
+		if is_compat or click.confirm("Continue anyway?"):
+			print("App Group {} selected! ✅".format(selected_group["name"]))
+			break
+
+	return selected_group["name"], filtered_apps
+
+@add_line_after
+def create_session():
+	# take user input from STDIN
+	username = click.prompt("Username").strip()
+	password = getpass.unix_getpass()
+
+	auth_credentials = {"usr": username, "pwd": password}
+
+	session = requests.Session()
+	login_sc = session.post(login_url, auth_credentials)
+
+	if login_sc.ok:
+		print("Authorization Successful! ✅")
+		session.headers.update({"X-Press-Team": username})
+		return session
+	else:
+		print("Authorization Failed with Error Code {}".format(login_sc.status_code))
+
+
+@add_line_after
+def get_subdomain(domain):
+	while True:
+		subdomain = click.prompt("Enter subdomain").strip()
+		if is_valid_subdomain(subdomain) and is_subdomain_available(subdomain):
+			print("Site Domain: {}.{}".format(subdomain, domain))
+			return subdomain
+
+
+@add_line_after
+def upload_backup(local_site):
+	# take backup
+	files_session = {}
+	print("Taking backup for site {}".format(local_site))
+	odb = frappe.utils.backups.new_backup(ignore_files=False, force=True)
+
+	# upload files
+	for x, (file_type, file_path) in enumerate([
+				("database", odb.backup_path_db),
+				("public", odb.backup_path_files),
+				("private", odb.backup_path_private_files)
+			]):
+		file_upload_response = session.post(files_url, data={}, files={
+			"file": open(file_path, "rb"),
+			"is_private": 1,
+			"folder": "Home",
+			"method": "press.api.site.upload_backup",
+			"type": file_type
+		})
+		print("Uploading files ({}/3)".format(x+1), end="\r")
+		if file_upload_response.ok:
+			files_session[file_type] = file_upload_response.json()["message"]
+		else:
+			print("Upload failed for: {}".format(file_path))
+
+	files_uploaded = { k: v["file_url"] for k, v in files_session.items() }
+	print("Uploaded backup files! ✅")
+
+	return files_uploaded
+
+
+def frappecloud_migrator(local_site, remote_site):
+	global login_url, upload_url, files_url, options_url, site_exists_url, session
+
+	login_url = "https://{}/api/method/login".format(remote_site)
+	upload_url = "https://{}/api/method/press.api.site.new".format(remote_site)
+	files_url = "https://{}/api/method/upload_file".format(remote_site)
+	options_url = "https://{}/api/method/press.api.site.options_for_new".format(remote_site)
+	site_exists_url = "https://{}/api/method/press.api.site.exists".format(remote_site)
+
+	print("Frappe Cloud credentials @ {}".format(remote_site))
+
+	# get credentials + auth user + start session
+	session = create_session()
+
+	if session:
+		# connect to site db
+		frappe.init(site=local_site)
+		frappe.connect()
+
+		# get new site options
+		site_options = get_new_site_options()
+
+		# set preferences from site options
+		subdomain = get_subdomain(site_options["domain"])
+		plan = choose_plan(site_options["plans"])
+
+		app_groups = site_options["groups"]
+		selected_group, filtered_apps = filter_apps(app_groups)
+		files_uploaded = upload_backup(local_site)
+
+		# push to frappe_cloud
+		payload = json.dumps({
+			"site": {
+				"apps": filtered_apps,
+				"files": files_uploaded,
+				"group": selected_group,
+				"name": subdomain,
+				"plan": plan
+			}
+		})
+
+		session.headers.update({"Content-Type": "application/json; charset=utf-8"})
+		site_creation_request = session.post(upload_url, payload)
+		frappe.destroy()
+
+		if site_creation_request.ok:
+			site_url = site_creation_request.json()["message"]
+			print("Your site {} is being migrated ✨".format(local_site))
+			print("View your site dashboard at {}/dashboard/#/sites/{}".format(remote_site, site_url))
+			print("Your site URL: {}".format(site_url))
+		else:
+			print("Request failed with error code {}".format(site_creation_request.status_code))
+			reason = html2text(site_creation_request.text)
+			print(reason)
+			sys.exit(1)
+
+	else:
+		sys.exit(1)

frappe/migrate.py

@@ -5,11 +5,13 @@ from __future__ import unicode_literals
 
 import json
 import os
+import sys
 import frappe
 import frappe.translate
 import frappe.modules.patch_handler
 import frappe.model.sync
 from frappe.utils.fixtures import sync_fixtures
+from frappe.utils.connections import check_connection
 from frappe.utils.dashboard import sync_dashboards
 from frappe.cache_manager import clear_global_cache
 from frappe.desk.notifications import clear_notifications
@@ -19,6 +21,7 @@ from frappe.modules.utils import sync_customizations
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.utils import global_search
 
+
 def migrate(verbose=True, rebuild_website=False, skip_failing=False):
 	'''Migrate all apps to the latest version, will:
 	- run before migrate hooks
@@ -32,6 +35,19 @@ def migrate(verbose=True, rebuild_website=False, skip_failing=False):
 	- run after migrate hooks
 	'''
 
+	service_status = check_connection(redis_services=["redis_cache"])
+	if False in service_status.values():
+		for service in service_status:
+			if not service_status.get(service, True):
+				print("{} service is not running.".format(service))
+		print("""Cannot run bench migrate without the services running.
+If you are running bench in development mode, make sure that bench is running:
+
+$ bench start
+
+Otherwise, check the server logs and ensure that all the required services are running.""")
+		sys.exit(1)
+
 	touched_tables_file = frappe.get_site_path('touched_tables.json')
 	if os.path.exists(touched_tables_file):
 		os.remove(touched_tables_file)
@@ -67,6 +83,9 @@ def migrate(verbose=True, rebuild_website=False, skip_failing=False):
 		# add static pages to global search
 		global_search.update_global_search_for_all_web_pages()
 
+		# updating installed applications data
+		frappe.get_single('Installed Applications').update_versions()
+
 		#run after_migrate hooks
 		for app in frappe.get_installed_apps():
 			for fn in frappe.get_hooks('after_migrate', app_name=app):

frappe/model/document.py

@@ -297,8 +297,7 @@ class Document(BaseDocument):
 		if ignore_permissions!=None:
 			self.flags.ignore_permissions = ignore_permissions
 
-		if ignore_version!=None:
-			self.flags.ignore_version = ignore_version
+		self.flags.ignore_version = frappe.flags.in_test if ignore_version is None else ignore_version
 
 		if self.get("__islocal") or not self.get("name"):
 			self.insert()
@@ -1339,4 +1338,4 @@ def check_doctype_has_consumers(doctype):
 
 	if len(event_consumers) and event_consumers[0]:
 		return True
-	return False
+	return False

frappe/model/meta.py

@@ -437,40 +437,47 @@ class Meta(Document):
 
 		if not self.custom:
 			for hook in frappe.get_hooks("override_doctype_dashboards", {}).get(self.name, []):
-				data = frappe.get_attr(hook)(data=data)
+				data = frappe._dict(frappe.get_attr(hook)(data=data))
 
 		return data
 
 	def add_doctype_links(self, data):
 		'''add `links` child table in standard link dashboard format'''
+		dashboard_links = []
+
 		if hasattr(self, 'links') and self.links:
-			if not data.transactions:
-				# init groups
-				data.transactions = []
-				data.non_standard_fieldnames = {}
+			dashboard_links.extend(self.links)
 
-			for link in self.links:
-				link.added = False
-				for group in data.transactions:
-					group = frappe._dict(group)
-					# group found
-					if link.group and group.label == link.group:
-						if link.link_doctype not in group.get('items'):
-							group.get('items').append(link.link_doctype)
-						link.added = True
+		if frappe.get_all("Custom Link", {"document_type": self.name}):
+			dashboard_links.extend(frappe.get_doc("Custom Link", self.name).links)
 
-				if not link.added:
-					# group not found, make a new group
-					data.transactions.append(dict(
-						label = link.group,
-						items = [link.link_doctype]
-					))
+		if not data.transactions:
+			# init groups
+			data.transactions = []
+			data.non_standard_fieldnames = {}
 
-				if link.link_fieldname != data.fieldname:
-					if data.fieldname:
-						data.non_standard_fieldnames[link.link_doctype] = link.link_fieldname
-					else:
-						data.fieldname = link.link_fieldname
+		for link in dashboard_links:
+			link.added = False
+			for group in data.transactions:
+				group = frappe._dict(group)
+				# group found
+				if link.group and group.label == link.group:
+					if link.link_doctype not in group.get('items'):
+						group.get('items').append(link.link_doctype)
+					link.added = True
+
+			if not link.added:
+				# group not found, make a new group
+				data.transactions.append(dict(
+					label = link.group,
+					items = [link.link_doctype]
+				))
+
+			if link.link_fieldname != data.fieldname:
+				if data.fieldname:
+					data.non_standard_fieldnames[link.link_doctype] = link.link_fieldname
+				else:
+					data.fieldname = link.link_fieldname
 
 
 	def get_row_template(self):

frappe/model/sync.py

@@ -51,7 +51,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
 			("desk", "onboarding_permission"),
 			("desk", "onboarding_step"),
 			("desk", "onboarding_step_map"),
-			("desk", "onboarding"),
+			("desk", "module_onboarding"),
 			("desk", "desk_card"),
 			("desk", "desk_chart"),
 			("desk", "desk_shortcut"),
@@ -85,7 +85,7 @@ def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=F
 	document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format',
 		'website_theme', 'web_form', 'web_template', 'notification', 'print_style',
 		'data_migration_mapping', 'data_migration_plan', 'desk_page',
-		'onboarding_step', 'onboarding']
+		'onboarding_step', 'module_onboarding']
 
 	for doctype in document_types:
 		doctype_path = os.path.join(start_path, doctype)

frappe/modules/import_file.py

@@ -13,7 +13,7 @@ ignore_values = {
 	"Print Format": ["disabled"],
 	"Notification": ["enabled"],
 	"Print Style": ["disabled"],
-	"Onboarding": ['is_complete'],
+	"Module Onboarding": ['is_complete'],
 	"Onboarding Step": ['is_complete', 'is_skipped']
 }
 

frappe/monitor.py

@@ -81,6 +81,12 @@ class Monitor:
 				self.data.request.status_code = response.status_code
 				self.data.request.response_length = int(response.headers.get("Content-Length", 0))
 
+				if hasattr(frappe.local, "rate_limiter"):
+					limiter = frappe.local.rate_limiter
+					self.data.request.counter = limiter.counter
+					if limiter.rejected:
+						self.data.request.reset = limiter.reset
+
 			self.store()
 		except Exception:
 			traceback.print_exc()

frappe/patches.txt

@@ -278,3 +278,8 @@ frappe.patches.v13_0.set_path_for_homepage_in_web_page_view
 frappe.patches.v13_0.migrate_translation_column_data
 frappe.patches.v13_0.set_read_times
 frappe.patches.v13_0.remove_web_view
+frappe.patches.v13_0.set_unique_for_page_view
+frappe.patches.v13_0.remove_tailwind_from_page_builder
+frappe.patches.v13_0.rename_onboarding
+frappe.patches.v13_0.email_unsubscribe
+execute:frappe.delete_doc("Web Template", "Section with Left Image", force=1)

frappe/patches/v13_0/email_unsubscribe.py

@@ -0,0 +1,13 @@
+import frappe
+
+def execute():
+	email_unsubscribe = [
+		{"email": "admin@example.com", "global_unsubscribe": 1},
+		{"email": "guest@example.com", "global_unsubscribe": 1}
+	]
+
+	for unsubscribe in email_unsubscribe:
+		if not frappe.get_all("Email Unsubscribe", filters=unsubscribe):
+			doc = frappe.new_doc("Email Unsubscribe")
+			doc.update(unsubscribe)
+			doc.insert(ignore_permissions=True)

frappe/patches/v13_0/remove_tailwind_from_page_builder.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+
+def execute():
+	frappe.reload_doc("website", "doctype", "web_page_block")
+	# remove unused templates
+	frappe.delete_doc("Web Template", "Navbar with Links on Right", force=1)
+	frappe.delete_doc("Web Template", "Footer Horizontal", force=1)
+