Merge branch 'frappe:develop' into custom-doctype-webform-perm

.github/helper/roulette.py

@@ -25,7 +25,7 @@ def is_ci(file):
 	return ".github" in file
 
 def is_frontend_code(file):
-	return file.endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts"))
+	return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue"))
 
 def is_docs(file):
 	regex = re.compile(r'\.(md|png|jpg|jpeg|csv)$|^.github|LICENSE')
@@ -38,8 +38,12 @@ if __name__ == "__main__":
 	pr_number = os.environ.get("PR_NUMBER")
 	repo = os.environ.get("REPO_NAME")
 
-	if not files_list and pr_number:
-		files_list = get_files_list(pr_number=pr_number, repo=repo)
+	# this is a push build, run all builds
+	if not pr_number:
+		os.system('echo "::set-output name=build::strawberry"')
+		sys.exit(0)
+
+	files_list = files_list or get_files_list(pr_number=pr_number, repo=repo)
 
 	if not files_list:
 		print("No files' changes detected. Build is shutting")

.github/workflows/patch-mariadb-tests.yml

@@ -2,6 +2,11 @@ name: Patch
 
 on: [pull_request, workflow_dispatch]
 
+
+concurrency:
+  group: patch-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

.github/workflows/server-mariadb-tests.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches: [ develop ]
 
+concurrency:
+  group: server-mariadb-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
+
 jobs:
   test:
     runs-on: ubuntu-18.04
@@ -131,17 +136,29 @@ jobs:
           COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
           COVERALLS_PARALLEL: true
 
+      - run: echo ${{ steps.check-build.outputs.build }} > guess-the-fruit.txt
+      - uses: actions/upload-artifact@v1
+        with:
+          name: fruit
+          path: guess-the-fruit.txt
+
   coveralls:
     name: Coverage Wrap Up
     needs: test
-    if: ${{ needs.test.steps.check-build.build == 'strawberry' }}
     container: python:3-slim
     runs-on: ubuntu-18.04
     steps:
+      - uses: actions/download-artifact@v1
+        with:
+          name: fruit
+      - run: echo "WILDCARD=$(cat fruit/guess-the-fruit.txt)" >> $GITHUB_ENV
+
       - name: Clone
+        if: ${{ env.WILDCARD == 'strawberry' }}
         uses: actions/checkout@v2
 
       - name: Coveralls Finished
+        if: ${{ env.WILDCARD == 'strawberry' }}
         run: |
           cd ${GITHUB_WORKSPACE}
           pip3 install coverage==5.5

.github/workflows/server-postgres-tests.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: server-postgres-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

.github/workflows/ui-tests.yml

@@ -6,6 +6,10 @@ on:
   push:
     branches: [ develop ]
 
+concurrency:
+  group: ui-develop-${{ github.event.number }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-18.04

CODEOWNERS

@@ -7,10 +7,13 @@
 templates/				@surajshetty3416
 www/							@surajshetty3416
 integrations/			@leela
-patches/					@surajshetty3416
+patches/					@surajshetty3416 @gavindsouza
 email/						@leela
 event_streaming/	@ruchamahabal
 data_import*			@netchampfaris
 core/							@surajshetty3416
+database          @gavindsouza
+model             @gavindsouza
 requirements.txt	@gavindsouza
 commands/					@gavindsouza
+workspace         @shariquerik

cypress/integration/datetime_field_form_validation.js

@@ -0,0 +1,19 @@
+context('Datetime Field Validation', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/communication');
+		cy.window().its('frappe').then(frappe => {
+			frappe.call("frappe.tests.ui_test_helpers.create_communication_records");
+		});
+	});
+
+	// validating datetime field value when value is set from backend and get validated on form load.
+	it('datetime field form validation', () => {
+		cy.visit('/app/communication');
+		cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name')
+			.then((name) => {
+				cy.visit(`/app/communication/${name}`);
+				cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
+			});
+	});
+});

cypress/integration/folder_navigation.js

@@ -0,0 +1,79 @@
+context('Folder Navigation', () => {
+	before(() => {
+		cy.visit('/login');
+		cy.login();
+		cy.visit('/app/file');
+	});
+
+	it('Adding Folders', () => {
+		//Adding filter to go into the home folder
+		cy.get('.filter-selector > .btn').findByText('1 filter').click();
+		cy.findByRole('button', {name: 'Clear Filters'}).click();
+		cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click();
+		cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}');
+		cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}');
+		cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click();
+
+		//Adding folder (Test Folder)
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();   
+	});
+
+	it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => {
+		//Navigating inside the Attachments folder
+		cy.get('[title="Attachments"] > span').click();
+
+		//To check if the URL formed after visiting the attachments folder is correct
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments');
+		cy.visit('/app/file/view/home/Attachments');
+
+		//Adding folder inside the attachments folder
+		cy.get('.menu-btn-group > .btn').click();
+		cy.get('.menu-btn-group [data-label="New Folder"]').click();
+		cy.get('form > [data-fieldname="value"]').type('Test Folder');
+		cy.findByRole('button', {name: 'Create'}).click();
+
+		//Navigating inside the added folder in the Attachments folder
+		cy.get('[title="Test Folder"] > span').click();
+
+		//To check if the URL is correct after visiting the Test Folder
+		cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder');
+		cy.visit('/app/file/view/home/Attachments/Test%20Folder');
+
+		//Adding a file inside the Test Folder
+		cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true});
+		cy.get('.file-uploader').findByText('Link').click();
+		cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg');
+		cy.findByRole('button', {name: 'Upload'}).click();
+
+		//To check if the added file is present in the Test Folder
+		cy.get('span.level-item > span').should('contain', 'Test Folder');
+		cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg');
+		cy.get('.list-row-checkbox').eq(0).click();
+
+		//Deleting the added file from the Test folder
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.wait(700);
+		cy.findByRole('button', {name: 'Yes'}).click();
+		cy.wait(700);
+
+		//Deleting the Test Folder
+		cy.visit('/app/file/view/home/Attachments');
+		cy.get('.list-row-checkbox').eq(0).click();
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();  
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+
+	it('Deleting Test Folder from the home', () => {
+	//Deleting the Test Folder added in the home directory
+		cy.visit('/app/file/view/home');
+		cy.get('.level-left > .list-subject > .list-row-checkbox').eq(0).click({force: true, delay: 500});
+		cy.findByRole('button', {name: 'Actions'}).click();
+		cy.get('.actions-btn-group [data-label="Delete"]').click();
+		cy.findByRole('button', {name: 'Yes'}).click();
+	});
+}); 

frappe/core/doctype/file/file.js

@@ -29,15 +29,8 @@ frappe.ui.form.on("File", "refresh", function(frm) {
 	if (is_optimizable) {
 		frm.add_custom_button(__("Optimize"), function() {
 			frappe.show_alert(__("Optimizing image..."));
-			frappe.call({
-				method: "frappe.core.doctype.file.file.optimize_saved_image",
-				args: {
-					doc_name: frm.doc.name,
-				},
-				callback: function() {
-					frappe.show_alert(__("Image optimized"));
-					frappe.set_route("List", "File");
-				}
+			frm.call("optimize_file").then(() => {
+				frappe.show_alert(__("Image optimized"));
 			});
 		});
 	}

frappe/core/doctype/file/file.py

@@ -313,8 +313,16 @@ class File(Document):
 			self.delete_file_data_content(only_thumbnail=True)
 
 	def on_rollback(self):
-		self.flags.on_rollback = True
-		self.on_trash()
+		# if original_content flag is set, this rollback should revert the file to its original state
+		if self.flags.original_content:
+			file_path = self.get_full_path()
+			with open(file_path, "wb+") as f:
+				f.write(self.flags.original_content)
+
+		# following condition is only executed when an insert has been rolledback
+		else:
+			self.flags.on_rollback = True
+			self.on_trash()
 
 	def unzip(self):
 		'''Unzip current file and replace it by its children'''
@@ -531,6 +539,35 @@ class File(Document):
 		if self.file_url:
 			self.is_private = cint(self.file_url.startswith('/private'))
 
+	@frappe.whitelist()
+	def optimize_file(self):
+		if self.is_folder:
+			raise TypeError('Folders cannot be optimized')
+
+		content_type = mimetypes.guess_type(self.file_name)[0]
+		is_local_image = content_type.startswith('image/') and self.file_size > 0
+		is_svg = content_type == 'image/svg+xml'
+
+		if not is_local_image:
+			raise NotImplementedError('Only local image files can be optimized')
+
+		if is_svg:
+			raise TypeError('Optimization of SVG images is not supported')
+
+		content = self.get_content()
+		file_path = self.get_full_path()
+		optimized_content = optimize_image(content, content_type)
+
+		with open(file_path, 'wb+') as f:
+			f.write(optimized_content)
+
+		self.file_size = len(optimized_content)
+		self.content_hash = get_content_hash(optimized_content)
+		# if rolledback, revert back to original
+		self.flags.original_content = content
+		frappe.local.rollback_observers.append(self)
+		self.save()
+
 def on_doctype_update():
 	frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"])
 
@@ -838,22 +875,6 @@ def unzip_file(name):
 	files = file_obj.unzip()
 	return files
 
-@frappe.whitelist()
-def optimize_saved_image(doc_name):
-	file_doc = frappe.get_doc('File', doc_name)
-	content = file_doc.get_content()
-	content_type = mimetypes.guess_type(file_doc.file_name)[0]
-
-	optimized_content = optimize_image(content, content_type)
-
-	file_path = get_files_path(is_private=file_doc.is_private)
-	file_path = os.path.join(file_path.encode('utf-8'), file_doc.file_name.encode('utf-8'))
-	with open(file_path, 'wb+') as f:
-		f.write(optimized_content)
-
-	file_doc.file_size = len(optimized_content)
-	file_doc.content_hash = get_content_hash(optimized_content)
-	file_doc.save()
 
 @frappe.whitelist()
 def get_attached_images(doctype, names):

frappe/core/doctype/file/test_file.py

@@ -440,6 +440,7 @@ class TestFile(unittest.TestCase):
 		}).insert(ignore_permissions=True)
 		self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip)
 
+
 class TestAttachment(unittest.TestCase):
 	test_doctype = 'Test For Attachment'
 
@@ -569,3 +570,68 @@ class TestFileUtils(unittest.TestCase):
 		from frappe.core.doctype.file.file import create_new_folder
 		folder = create_new_folder('test_folder', 'Home')
 		self.assertTrue(folder.is_folder)
+
+
+class TestFileOptimization(unittest.TestCase):
+	def test_optimize_file(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		original_size = test_file.file_size
+		original_content_hash = test_file.content_hash
+
+		test_file.optimize_file()
+		optimized_size = test_file.file_size
+		updated_content_hash = test_file.content_hash
+
+		self.assertLess(optimized_size, original_size)
+		self.assertNotEqual(original_content_hash, updated_content_hash)
+		test_file.delete()
+
+	def test_optimize_svg(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_svg.svg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_svg.svg",
+			"content": file_content
+		}).insert()
+		self.assertRaises(TypeError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_textfile(self):
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_text.txt",
+			"content": "Text files cannot be optimized"
+		}).insert()
+		self.assertRaises(NotImplementedError, test_file.optimize_file)
+		test_file.delete()
+
+	def test_optimize_folder(self):
+		test_folder = frappe.get_doc("File", "Home/Attachments")
+		self.assertRaises(TypeError, test_folder.optimize_file)
+
+	def test_revert_optimized_file_on_rollback(self):
+		file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg")
+		with open(file_path, "rb") as f:
+			file_content = f.read()
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": "sample_image_for_optimization.jpg",
+			"content": file_content
+		}).insert()
+		image_path = test_file.get_full_path()
+		size_before_optimization = os.stat(image_path).st_size
+
+		test_file.optimize_file()
+		frappe.db.rollback()
+		size_after_rollback = os.stat(image_path).st_size
+		self.assertEqual(size_before_optimization, size_after_rollback)
+		test_file.delete()

frappe/custom/doctype/customize_form/customize_form.py

@@ -193,6 +193,16 @@ class CustomizeForm(Document):
 		if prop == "fieldtype":
 			self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop))
 
+		elif prop == "length":
+			old_value_length = cint(meta_df[0].get(prop))
+			new_value_length = cint(df.get(prop))
+
+			if new_value_length and (old_value_length > new_value_length):
+				self.check_length_for_fieldtypes.append({'df': df, 'old_value': meta_df[0].get(prop)})
+				self.validate_fieldtype_length()
+			else:
+				self.flags.update_db = True
+
 		elif prop == "allow_on_submit" and df.get(prop):
 			if not frappe.db.get_value("DocField",
 				{"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit"):

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -188,6 +188,26 @@ class TestCustomizeForm(unittest.TestCase):
 	def test_core_doctype_customization(self):
 		self.assertRaises(frappe.ValidationError, self.get_customize_form, 'User')
 
+	def test_save_customization_length_field_property(self):
+		# Using Notification Log doctype as it doesn't have any other custom fields
+		d = self.get_customize_form("Notification Log")
+
+		document_name = d.get("fields", {"fieldname": "document_name"})[0]
+		document_name.length = 255
+		d.run_method("save_customization")
+
+		self.assertEqual(frappe.db.get_value("Property Setter",
+			{"doc_type": "Notification Log", "property": "length", "field_name": "document_name"}, "value"), '255')
+
+		self.assertTrue(d.flags.update_db)
+
+		length = frappe.db.sql("""SELECT character_maximum_length
+			FROM information_schema.columns
+			WHERE table_name = 'tabNotification Log'
+			AND column_name = 'document_name'""")[0][0]
+
+		self.assertEqual(length, 255)
+
 	def test_custom_link(self):
 		try:
 			# create a dummy doctype linked to Event

frappe/desk/query_report.py

@@ -391,7 +391,7 @@ def handle_duration_fieldtype_values(result, columns):
 	return result
 
 
-def build_xlsx_data(columns, data, visible_idx, include_indentation):
+def build_xlsx_data(columns, data, visible_idx, include_indentation, ignore_visible_idx=False):
 	result = [[]]
 	column_widths = []
 
@@ -407,7 +407,7 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation):
 	# build table from result
 	for row_idx, row in enumerate(data.result):
 		# only pick up rows that are visible in the report
-		if row_idx in visible_idx:
+		if ignore_visible_idx or row_idx in visible_idx:
 			row_data = []
 			if isinstance(row, dict):
 				for col_idx, column in enumerate(data.columns):

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -13,6 +13,7 @@ from frappe.utils import (format_time, get_link_to_form, get_url_to_report,
 from frappe.model.naming import append_number_if_name_exists
 from frappe.utils.csvutils import to_csv
 from frappe.utils.xlsxutils import make_xlsx
+from frappe.desk.query_report import build_xlsx_data
 
 max_reports_per_user = frappe.local.conf.max_reports_per_user or 3
 
@@ -99,13 +100,21 @@ class AutoEmailReport(Document):
 			return self.get_html_table(columns, data)
 
 		elif self.format == 'XLSX':
-			spreadsheet_data = self.get_spreadsheet_data(columns, data)
-			xlsx_file = make_xlsx(spreadsheet_data, "Auto Email Report")
+			report_data = frappe._dict()
+			report_data['columns'] = columns
+			report_data['result'] = data
+
+			xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True)
+			xlsx_file = make_xlsx(xlsx_data, "Auto Email Report", column_widths=column_widths)
 			return xlsx_file.getvalue()
 
 		elif self.format == 'CSV':
-			spreadsheet_data = self.get_spreadsheet_data(columns, data)
-			return to_csv(spreadsheet_data)
+			report_data = frappe._dict()
+			report_data['columns'] = columns
+			report_data['result'] = data
+
+			xlsx_data, column_widths = build_xlsx_data(columns, report_data, [], 1, ignore_visible_idx=True)
+			return to_csv(xlsx_data)
 
 		else:
 			frappe.throw(_('Invalid Output Format'))
@@ -126,18 +135,6 @@ class AutoEmailReport(Document):
 			'edit_report_settings': get_link_to_form('Auto Email Report', self.name)
 		})
 
-	@staticmethod
-	def get_spreadsheet_data(columns, data):
-		out = [[_(df.label) for df in columns], ]
-		for row in data:
-			new_row = []
-			out.append(new_row)
-			for df in columns:
-				if df.fieldname not in row: continue
-				new_row.append(frappe.format(row[df.fieldname], df, row))
-
-		return out
-
 	def get_file_name(self):
 		return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())
 

frappe/model/base_document.py

@@ -874,7 +874,7 @@ class BaseDocument(object):
 		return self._precision[cache_key][fieldname]
 
 
-	def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False):
+	def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False, format=None):
 		from frappe.utils.formatters import format_value
 
 		df = self.meta.get_field(fieldname)
@@ -898,7 +898,7 @@ class BaseDocument(object):
 		if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)):
 			val = abs(self.get(fieldname))
 
-		return format_value(val, df=df, doc=doc, currency=currency)
+		return format_value(val, df=df, doc=doc, currency=currency, format=format)
 
 	def is_print_hide(self, fieldname, df=None, for_print=True):
 		"""Returns true if fieldname is to be hidden for print.

frappe/public/icons/timeless/symbol-defs.svg

@@ -567,7 +567,7 @@
 		<path d="M6.45466 8.81824L4.47873 10.7942C3.85205 11.4211 3.5 12.2713 3.5 13.1577C3.5 14.0442 3.85205 14.8943 4.47873 15.5213V15.5213C5.10568 16.148 5.95584 16.5 6.84229 16.5C7.72874 16.5 8.5789 16.148 9.20584 15.5213L11.1818 13.5453" stroke="var(--icon-stroke)" stroke-linecap="round" stroke-linejoin="round"/>
 	</symbol>
 	<symbol viewBox="0 0 24 24" fill="none" id="icon-scan" xmlns="http://www.w3.org/2000/svg">
-		<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3" 
+		<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"
 			stroke="var(--icon-stroke)" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"/>
 	</symbol>
 	<symbol viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" id="icon-dashboard">

frappe/public/js/frappe/data_import/data_exporter.js

@@ -13,6 +13,13 @@ frappe.data_import.DataExporter = class DataExporter {
 		this.dialog = new frappe.ui.Dialog({
 			title: __('Export Data'),
 			fields: [
+				{
+					fieldtype: 'Select',
+					fieldname: 'file_type',
+					label: __('File Type'),
+					options: ['Excel', 'CSV'],
+					default: 'CSV'
+				},
 				{
 					fieldtype: 'Select',
 					fieldname: 'export_records',
@@ -45,13 +52,6 @@ frappe.data_import.DataExporter = class DataExporter {
 					fieldname: 'filter_area',
 					depends_on: doc => doc.export_records === 'by_filter'
 				},
-				{
-					fieldtype: 'Select',
-					fieldname: 'file_type',
-					label: __('File Type'),
-					options: ['Excel', 'CSV'],
-					default: 'CSV'
-				},
 				{
 					fieldtype: 'Section Break'
 				},
@@ -141,7 +141,7 @@ frappe.data_import.DataExporter = class DataExporter {
 		let for_insert = this.exporting_for === 'Insert New Records';
 		let section_title = for_insert ? __('Select Fields To Insert') : __('Select Fields To Update');
 		let $select_all_buttons = $(`
-			<div>
+			<div class="mb-3">
 				<h6 class="form-section-heading uppercase">${section_title}</h6>
 				<button class="btn btn-default btn-xs" data-action="select_all">
 					${__('Select All')}

frappe/public/js/frappe/file_uploader/index.js

@@ -28,7 +28,7 @@ export default class FileUploader {
 		}
 
 		if (attach_doc_image) {
-			restrictions.allowed_file_types = ['.jpg', '.jpeg', '.png'];
+			restrictions.allowed_file_types = ['image/jpeg', 'image/png'];
 		}
 
 		this.$fileuploader = new Vue({
@@ -70,8 +70,10 @@ export default class FileUploader {
 		this.uploader.$watch('hide_dialog_footer', (hide_dialog_footer) => {
 			if (hide_dialog_footer) {
 				this.dialog && this.dialog.footer.addClass('hide');
+				this.dialog.$wrapper.data('bs.modal')._config.backdrop = 'static';
 			} else {
 				this.dialog && this.dialog.footer.removeClass('hide');
+				this.dialog.$wrapper.data('bs.modal')._config.backdrop = true;
 			}
 		});
 

frappe/public/js/frappe/form/controls/datetime.js

@@ -36,4 +36,9 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co
 			$tp.$secondsText.prev().css('display', 'none');
 		}
 	}
+
+	get_model_value() {
+		let value = super.get_model_value();
+		return frappe.datetime.get_datetime_as_string(value);
+	}
 };

frappe/public/js/frappe/list/bulk_operations.js

@@ -261,4 +261,14 @@ export default class BulkOperations {
 		});
 		dialog.show();
 	}
+
+	export(doctype, docnames) {
+		frappe.require('data_import_tools.bundle.js', () => {
+			const data_exporter = new frappe.data_import.DataExporter(doctype, 'Insert New Records');
+			data_exporter.dialog.set_value('export_records', 'by_filter');
+			data_exporter.filter_group.add_filters_to_filter_group(
+				[[doctype, "name", "in", docnames, false]]
+			);
+		});
+	}
 }

frappe/public/js/frappe/list/list_view.js

@@ -868,8 +868,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 			filters: this.get_filters_for_args()
 		}).then(total_count => {
 			this.total_count = total_count || current_count;
+			this.count_without_children = count_without_children !== current_count ? count_without_children : undefined;
 			let str = __('{0} of {1}', [current_count, this.total_count]);
-			if (count_without_children !== current_count) {
+			if (this.count_without_children) {
 				str = __('{0} of {1} ({2} rows with children)', [count_without_children, this.total_count, current_count]);
 			}
 			return str;
@@ -1731,11 +1732,25 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 			};
 		};
 
+		const bulk_export = () => {
+			return {
+				label: __("Export"),
+				action: () => {
+					const docnames = this.get_checked_items(true);
+
+					bulk_operations.export(doctype, docnames);
+				},
+				standard: true
+			};
+		};
+
 		// bulk edit
 		if (has_editable_fields(doctype)) {
 			actions_menu_items.push(bulk_edit());
 		}
 
+		actions_menu_items.push(bulk_export());
+
 		// bulk assignment
 		actions_menu_items.push(bulk_assignment());
 

frappe/public/js/frappe/views/reports/report_view.js

@@ -1401,7 +1401,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 						}
 					];
 
-					if (this.total_count > args.page_length) {
+					if (this.total_count > this.count_without_children || args.page_length) {
 						fields.push({
 							fieldtype: 'Check',
 							fieldname: 'export_all_rows',

frappe/tests/data/sample_svg.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="16px" height="16px" viewBox="0 0 16 16" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 61.2 (89653) - https://sketch.com -->
+    <title>Artboard</title>
+    <desc>Created with Sketch.</desc>
+    <g id="Artboard" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="frappe" transform="translate(3.000000, 1.000000)" fill="#0089FF" fill-rule="nonzero">
+            <polygon id="Path" points="9.360932 0 0 0 0 2.46232 9.360932 2.46232"></polygon>
+            <polygon id="Path" points="0 6.281996 0 14 2.98788 14 2.98788 8.74846 8.740172 8.74846 8.740172 6.281996"></polygon>
+        </g>
+    </g>
+</svg>

frappe/tests/test_document.py

@@ -227,3 +227,28 @@ class TestDocument(unittest.TestCase):
 		self.assertEqual(frappe.db.get_value("Currency", d.name), d.name)
 
 		frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1)
+
+	def test_get_formatted(self):
+		frappe.get_doc({
+			'doctype': 'DocType',
+			'name': 'Test Formatted',
+			'module': 'Custom',
+			'custom': 1,
+			'fields': [
+				{'label': 'Currency', 'fieldname': 'currency', 'reqd': 1, 'fieldtype': 'Currency'},
+			]
+		}).insert()
+
+		frappe.delete_doc_if_exists("Currency", "INR", 1)
+
+		d = frappe.get_doc({
+			'doctype': 'Currency',
+			'currency_name': 'INR',
+			'symbol': '₹',
+		}).insert()
+
+		d = frappe.get_doc({
+			'doctype': 'Test Formatted',
+			'currency': 100000
+		})
+		self.assertEquals(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00')

frappe/tests/test_fmt_money.py

@@ -92,6 +92,9 @@ class TestFmtMoney(unittest.TestCase):
 		self.assertEqual(fmt_money(1000.456), "1.000,456")
 		frappe.db.set_default("currency_precision", "")
 
+	def test_custom_fmt_money_format(self):
+		self.assertEqual(fmt_money(100000, format="#,###.##"), '100,000.00')
+
 if __name__=="__main__":
 	frappe.connect()
 	unittest.main()

frappe/tests/test_formatter.py

@@ -17,9 +17,9 @@ class TestFormatter(unittest.TestCase):
 		frappe.db.set_default("currency", 'INR')
 
 		# if currency field is not passed then default currency should be used.
-		self.assertEqual(format(100, df, doc), '₹ 100.00')
+		self.assertEqual(format(100000, df, doc, format="#,###.##"), '₹ 100,000.00')
 
 		doc.currency = 'USD'
-		self.assertEqual(format(100, df, doc), "$ 100.00")
+		self.assertEqual(format(100000, df, doc, format="#,###.##"), "$ 100,000.00")
 
 		frappe.db.set_default("currency", None)

frappe/tests/ui_test_helpers.py

@@ -61,6 +61,18 @@ def create_todo_records():
 		"description": "this is fourth todo"
 	}).insert()
 
+@frappe.whitelist()
+def create_communication_records():
+	if frappe.db.get_all('Communication', {'subject': 'Test Form Communication 1'}):
+		return
+
+	frappe.get_doc({
+		"doctype": "Communication",
+		"recipients": "test@gmail.com",
+		"subject": "Test Form Communication 1",
+		"communication_date": frappe.utils.now_datetime(),
+	}).insert()
+
 @frappe.whitelist()
 def setup_workflow():
 	from frappe.workflow.doctype.workflow.test_workflow import create_todo_workflow

frappe/utils/background_jobs.py

@@ -3,12 +3,14 @@ import socket
 import time
 from uuid import uuid4
 from collections import defaultdict
+from typing import List
 
 
 import redis
-from typing import List
+from redis.exceptions import BusyLoadingError, ConnectionError
 from rq import Connection, Queue, Worker
 from rq.logutils import setup_loghandlers
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
 import frappe
 from frappe import _
@@ -233,6 +235,11 @@ def validate_queue(queue, default_queue_list=None):
 	if queue not in default_queue_list:
 		frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list)))
 
+@retry(
+	retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError),
+	stop=stop_after_attempt(10),
+	wait=wait_fixed(1)
+)
 def get_redis_conn(username=None, password=None):
 	if not hasattr(frappe.local, 'conf'):
 		raise Exception('You need to call frappe.init')

frappe/utils/formatters.py

@@ -7,7 +7,7 @@ from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime
 from frappe.model.meta import get_field_currency, get_field_precision
 import re
 
-def format_value(value, df=None, doc=None, currency=None, translated=False):
+def format_value(value, df=None, doc=None, currency=None, translated=False, format=None):
 	'''Format value based on given fieldtype, document reference, currency reference.
 	If docfield info (df) is not given, it will try and guess based on the datatype of the value'''
 	if isinstance(df, str):
@@ -56,7 +56,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False):
 	elif df.get("fieldtype") == "Currency":
 		default_currency = frappe.db.get_default("currency")
 		currency = currency or get_field_currency(df, doc) or default_currency
-		return fmt_money(value, precision=get_field_precision(df, doc), currency=currency)
+		return fmt_money(value, precision=get_field_precision(df, doc), currency=currency, format=format)
 
 	elif df.get("fieldtype") == "Float":
 		precision = get_field_precision(df, doc)

requirements.txt

@@ -77,3 +77,4 @@ Whoosh~=2.7.4
 wrapt~=1.12.1
 xlrd~=2.0.1
 zxcvbn-python~=4.4.24
+tenacity~=8.0.1