feat: Initialize Exporter

- Export into a simple format
- Basic tests for exporter
- DataExporter: user friendly exporter dialog

frappe/core/doctype/data_import/exporter_new.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+from frappe import _
+from frappe.model import display_fieldtypes, no_value_fields, table_fields
+from frappe.utils.csvutils import build_csv_response
+
+class Exporter:
+	def __init__(self, doctype, export_fields=None, export_data=False, export_filters=None, file_type='CSV'):
+		"""
+		Exports records of a DocType for use with Importer
+			:param doctype: Document Type to export
+			:param export_fields=None: One of 'All', 'Mandatory' or {'DocType': ['field1', 'field2'], 'Child DocType': ['childfield1']}
+			:param export_data=False: Whether to export data as well
+			:param export_filters=None: The filters (dict or list) which is used to query the records
+			:param file_type: One of 'Excel' or 'CSV'
+		"""
+		self.doctype = doctype
+		self.meta = frappe.get_meta(doctype)
+		self.export_fields = export_fields
+
+		# this will contain the csv content
+		self.csv_array = []
+
+		# fields that get exported
+		# can be All, Mandatory or User Selected Fields
+		self.fields = self.get_all_exportable_fields()
+		self.add_header()
+
+		if export_data:
+			self.data = self.get_data_to_export(export_filters)
+		else:
+			self.data = []
+		self.add_data()
+
+
+	def get_all_exportable_fields(self):
+		return self.get_exportable_parent_fields() + self.get_exportable_children_fields()
+
+
+	def get_exportable_parent_fields(self):
+		name_field = frappe._dict({
+			'fieldtype': 'Data',
+			'fieldname': 'name',
+			'label': 'ID',
+			'reqd': 1,
+			'parent': self.doctype
+		})
+		return [name_field] + self.get_exportable_fields(self.doctype)
+
+
+	def get_exportable_children_fields(self):
+		children = [df.options for df in self.meta.fields if df.fieldtype in table_fields]
+		children_fields = []
+		for child in children:
+			children_fields += self.get_exportable_fields(child)
+
+		return children_fields
+
+
+	def get_exportable_fields(self, doctype):
+		def is_exportable(df):
+			return (
+				df.fieldtype not in display_fieldtypes
+				and df.fieldtype not in no_value_fields
+				and not df.hidden
+			)
+
+		meta = frappe.get_meta(doctype)
+
+		# filter out layout fields
+		fields = [df for df in meta.fields if is_exportable(df)]
+
+		if self.export_fields == 'All':
+			return fields
+
+		if self.export_fields == 'Mandatory':
+			return [df for df in fields if df.reqd]
+
+		if isinstance(self.export_fields, dict):
+			whitelist = self.export_fields.get(doctype, [])
+			return [df for df in fields if df.fieldname in whitelist]
+
+		return [df for df in fields if df.reqd or df.in_list_view or df.bold]
+
+
+	def get_data_to_export(self, filters=None):
+		frappe.permissions.can_export(self.doctype, raise_exception=True)
+
+		def get_column_name(df):
+			return '`tab{0}`.`{1}`'.format(df.parent, df.fieldname)
+
+		fieldnames = [get_column_name(df) for df in self.fields]
+
+		if self.meta.is_nested_set():
+			order_by = '`tab{0}`.`lft` ASC'.format(self.doctype)
+		else:
+			order_by = '`tab{0}`.`creation` DESC'.format(self.doctype)
+
+		data = frappe.db.get_list(self.doctype,
+			filters=filters,
+			fields=fieldnames,
+			limit_page_length=None,
+			order_by=order_by,
+			as_list=1,
+		)
+
+		return self.remove_duplicate_parent_values(data)
+
+	def remove_duplicate_parent_values(self, data):
+		out = []
+
+		parent_fields = self.get_exportable_parent_fields()
+		parent_fields_count = len(parent_fields)
+
+		current_name = None
+		# first column is always name, can be used to find duplicate rows
+		for row in data:
+			row = list(row)
+			name = row[0]
+
+			if name != current_name:
+				current_name = name
+				out.append(row)
+
+			elif name == current_name:
+				# remove the parent values from the row
+				row = row[parent_fields_count:]
+				# replace them with None values and add back the row contents
+				row = [None] * parent_fields_count + row
+				out.append(row)
+
+		return out
+
+
+	def add_header(self):
+		def get_label(df):
+			if df.parent == self.doctype:
+				return df.label
+			else:
+				return '{0} / {1}'.format(df.parent, df.label)
+
+		header = [get_label(df) for df in self.fields]
+		self.csv_array.append(header)
+
+
+	def add_data(self):
+		self.csv_array += self.data
+
+
+	def get_csv_array(self):
+		return self.csv_array
+
+	def build_csv_response(self):
+		csv_array = self.csv_array
+
+		if not self.data:
+			# add 5 empty rows
+			csv_array += [[]] * 5
+
+		build_csv_response(csv_array, self.doctype)

frappe/core/doctype/data_import/test_exporter_new.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import unittest
+import frappe
+from frappe.core.doctype.data_import.exporter_new import Exporter
+
+
+class TestExporter(unittest.TestCase):
+	def test_exports_mandatory_fields(self):
+		e = Exporter('Web Page', export_fields='Mandatory')
+		csv_array = e.get_csv_array()
+		header_row = csv_array[0]
+		self.assertEqual(header_row, ['ID (name)', 'Title (title)'])
+
+
+	def test_exports_all_fields(self):
+		e = Exporter('Web Page', export_fields='All')
+		csv_array = e.get_csv_array()
+		header = csv_array[0]
+		self.assertEqual(len(header), 23)
+
+
+	def test_exports_selected_fields(self):
+		export_fields = ['title', 'route', 'published']
+		e = Exporter('Web Page', export_fields=export_fields)
+		csv_array = e.get_csv_array()
+		header = csv_array[0]
+		self.assertEqual(header, ['Title (title)', 'Route (route)', 'Published (published)'])
+
+
+	def test_exports_data(self):
+		e = Exporter('ToDo', export_fields='All', export_data=True)
+		todo_records = frappe.db.count('ToDo')
+		csv_array = e.get_csv_array()
+		self.assertEqual(len(csv_array), todo_records + 1)

frappe/public/js/frappe/data_import/data_exporter.js

@@ -0,0 +1,280 @@
+frappe.provide('frappe.data_import');
+
+class ColumnPickerFields extends frappe.views.ReportView {
+	show() {}
+}
+
+frappe.data_import.DataExporter = class DataExporter {
+	constructor(doctype) {
+		this.doctype = doctype;
+		frappe.model.with_doctype(doctype, () => {
+			this.make_dialog();
+		});
+	}
+
+	make_dialog() {
+		let column_map = new ColumnPickerFields({
+			doctype: this.doctype
+		}).get_columns_for_picker();
+
+		let doctypes = [this.doctype].concat(
+			...frappe.meta
+				.get_table_fields(this.doctype)
+				.filter(df => !df.hidden)
+				.map(df => df.options)
+		);
+
+		this.dialog = new frappe.ui.Dialog({
+			title: __('Export Data'),
+			fields: [
+				{
+					fieldtype: 'Select',
+					fieldname: 'export_records',
+					label: __('Export Records'),
+					options: [
+						{
+							label: __('Export All Records'),
+							value: 'all'
+						},
+						// {
+						// 	label: __('Export 10 Records'),
+						// 	value: 'last_10_records'
+						// },
+						{
+							label: __('Export Filtered Records'),
+							value: 'by_filter'
+						}
+					],
+					default: 'all',
+					change: () => {
+						this.update_record_count_message();
+					}
+				},
+				{
+					fieldtype: 'HTML',
+					fieldname: 'filter_area',
+					depends_on: doc => doc.export_records === 'by_filter'
+				},
+				{
+					fieldtype: 'Select',
+					fieldname: 'file_type',
+					label: __('File Type'),
+					options: ['Excel', 'CSV'],
+					default: 'CSV'
+				},
+				{
+					fieldtype: 'Section Break'
+				},
+				{
+					fieldtype: 'HTML',
+					fieldname: 'select_all_buttons'
+				},
+				...doctypes.map(doctype => {
+					return {
+						label: __(doctype),
+						fieldname: doctype,
+						fieldtype: 'MultiCheck',
+						columns: 2,
+						on_change: () => {
+							this.update_primary_action();
+						},
+						options: column_map[doctype].map(df => ({
+							label: __(df.label),
+							value: df.fieldname,
+							danger: df.reqd,
+							checked: df.reqd,
+							description: `${df.fieldname} ${
+								df.reqd ? __('(Mandatory)') : ''
+							}`
+						}))
+					};
+				})
+			],
+			primary_action_label: __('Export'),
+			primary_action: values => {
+				this.export_records(values);
+			}
+		});
+
+		this.make_filter_area();
+		this.make_select_all_buttons();
+		this.update_record_count_message();
+
+		this.dialog.show();
+	}
+
+	export_records() {
+		let method =
+			'/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_template';
+
+		let multicheck_fields = this.dialog.fields
+			.filter(df => df.fieldtype === 'MultiCheck')
+			.map(df => df.fieldname);
+
+		let values = this.dialog.get_values();
+
+		let doctype_field_map = Object.assign({}, values);
+		for (let key in doctype_field_map) {
+			if (!multicheck_fields.includes(key)) {
+				delete doctype_field_map[key];
+			}
+		}
+
+		let filters = null;
+		if (values.export_records === 'by_filter') {
+			filters = this.get_filters();
+		}
+
+		open_url_post(method, {
+			doctype: this.doctype,
+			file_type: values.file_type,
+			export_records: values.export_records,
+			export_fields: doctype_field_map,
+			export_filters: filters
+		});
+	}
+
+	make_filter_area() {
+		this.filter_group = new frappe.ui.FilterGroup({
+			parent: this.dialog.get_field('filter_area').$wrapper,
+			doctype: this.doctype,
+			on_change: e => {
+				this.update_record_count_message();
+			}
+		});
+	}
+
+	make_select_all_buttons() {
+		let $select_all_buttons = $(`
+			<div>
+				<h6 class="form-section-heading uppercase">${__('Select fields to export')}</h6>
+				<button class="btn btn-default btn-xs" data-action="select_all">
+					${__('Select All')}
+				</button>
+				<button class="btn btn-default btn-xs" data-action="select_mandatory">
+					${__('Select Mandatory')}
+				</button>
+				<button class="btn btn-default btn-xs" data-action="select_mandatory_without_children">
+					${__('Select Mandatory without children')}
+				</button>
+				<button class="btn btn-default btn-xs" data-action="unselect_all">
+					${__('Unselect All')}
+				</button>
+			</div>
+		`).on('click', '[data-action]', e => {
+			let $target = $(e.currentTarget);
+			let action = $target.data('action');
+			let method = this[action];
+			method ? this[action]() : null;
+		});
+		this.dialog
+			.get_field('select_all_buttons')
+			.$wrapper.html($select_all_buttons);
+	}
+
+	select_all() {
+		this.dialog.$wrapper
+			.find(':checkbox')
+			.prop('checked', true)
+			.trigger('change');
+	}
+
+	select_mandatory() {
+		let multicheck_fields = this.dialog.fields
+			.filter(df => df.fieldtype === 'MultiCheck')
+			.map(df => df.fieldname);
+
+		let checkboxes = [].concat(
+			...multicheck_fields.map(fieldname => {
+				let field = this.dialog.get_field(fieldname);
+				return field.options
+					.filter(option => option.danger)
+					.map(option => option.$checkbox.find('input').get(0));
+			})
+		);
+
+		this.unselect_all();
+		$(checkboxes)
+			.prop('checked', true)
+			.trigger('change');
+	}
+
+	select_mandatory_without_children() {
+		let field = this.dialog.get_field(this.doctype);
+		let checkboxes = field.options
+			.filter(option => option.danger)
+			.map(option => option.$checkbox.find('input').get(0));
+
+		this.unselect_all();
+		$(checkboxes)
+			.prop('checked', true)
+			.trigger('change');
+	}
+
+	unselect_all() {
+		this.dialog.$wrapper
+			.find(':checkbox')
+			.prop('checked', false)
+			.trigger('change');
+	}
+
+	update_record_count_message() {
+		let export_records = this.dialog.get_value('export_records');
+		let count_method = {
+			last_10_records: () => Promise.resolve('10'),
+			all: () => frappe.db.count(this.doctype),
+			by_filter: () =>
+				frappe.db.count(this.doctype, {
+					filters: this.get_filters()
+				})
+		};
+
+		count_method[export_records]().then(value => {
+			let message = '';
+			value = parseInt(value);
+			if (value === 0) {
+				message = __('No records will be exported');
+			} else if (value === 1) {
+				message = __('1 record will be exported');
+			} else {
+				message = __('{0} records will be exported', [value]);
+			}
+			this.dialog.set_df_property(
+				'export_records',
+				'description',
+				message
+			);
+
+			this.update_primary_action(value);
+		});
+	}
+
+	update_primary_action(no_of_records) {
+		let $primary_action = this.dialog.get_primary_btn();
+
+		if (no_of_records != null) {
+			$primary_action.prop('disabled', no_of_records === 0);
+
+			let label = '';
+			if (no_of_records === 0) {
+				label = __('Export');
+			} else if (no_of_records === 1) {
+				label = __('Export 1 record');
+			} else {
+				label = __('Export {0} records', [no_of_records]);
+			}
+			$primary_action.html(label);
+		} else {
+			let parent_fields = this.dialog.get_value(this.doctype);
+			$primary_action.prop('disabled', parent_fields.length === 0);
+		}
+	}
+
+	get_filters() {
+		return this.filter_group.get_filters().reduce((acc, filter) => {
+			return Object.assign(acc, {
+				[filter[1]]: [filter[2], filter[3]]
+			});
+		}, {});
+	}
+};

frappe/public/js/frappe/form/controls/multicheck.js

@@ -70,6 +70,7 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
 			if (option.danger) {
 				checkbox.find('.label-area').addClass('text-danger');
 			}
+			option.$checkbox = checkbox;
 		});
 		if(this.df.select_all) {
 			this.setup_select_all();
@@ -138,7 +139,7 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
 		const column_size = this.get_column_size();
 		return $(`
 			<div class="checkbox unit-checkbox col-sm-${column_size}">
-				<label>
+				<label title="${option.description || ''}">
 				<input type="checkbox" data-unit="${option.value}">
 				</input>
 				<span class="label-area small" data-unit="${option.value}">${__(option.label)}</span>

frappe/public/js/frappe/form/sidebar/form_sidebar.js

@@ -216,7 +216,7 @@ frappe.ui.form.Sidebar = Class.extend({
 			callback: (r) => {
 				// docinfo will be synced
 				if(callback) callback(r.docinfo);
-				this.frm.timeline.refresh();
+				this.frm.timeline && this.frm.timeline.refresh();
 				this.frm.assign_to.refresh();
 				this.frm.attachments.refresh();
 			}

frappe/public/js/frappe/views/reports/report_view.js

@@ -780,7 +780,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 		doctype_fields = [{
 			label: __('ID'),
 			fieldname: 'name',
-			fieldtype: 'Data'
+			fieldtype: 'Data',
+			reqd: 1
 		}].concat(doctype_fields, frappe.model.std_fields);
 
 		out[this.doctype] = doctype_fields;