feat: Remap columns to fields

- Green colored are mapped
- Red colored are skipped
- Skip columns from import

frappe/core/doctype/data_import/exporter_new.py

@@ -154,7 +154,7 @@ class Exporter:
 			if df.parent == self.doctype:
 				return df.label
 			else:
-				return '{0} / {1}'.format(df.parent, df.label)
+				return '{0} ({1})'.format(df.label, df.parent)
 
 		header = [get_label(df) for df in self.fields]
 		self.csv_array.append(header)

frappe/core/doctype/data_import/importer_new.py

@@ -95,35 +95,50 @@ class Importer:
 		self.header_row = header_row
 
 
-	def get_data_for_import_preview(self):
-		fields, fields_warnings = self.parse_fields_from_header_row()
+	def get_data_for_import_preview(self, import_options=None):
+		import_options = import_options or frappe._dict()
+		remap_columns = import_options.remap_column
+		skip_import = import_options.skip_import
+
+		fields, fields_warnings = self.parse_fields_from_header_row(remap_columns, skip_import)
 		formats, formats_warnings = self.parse_formats_from_first_10_rows()
 		fields, data = self.add_serial_no_column(fields, self.data)
 
 		warnings = fields_warnings + formats_warnings
 
 		return dict(
+			header_row=self.header_row,
 			fields=fields,
 			data=data,
 			warnings=warnings
 		)
 
 
-	def parse_fields_from_header_row(self):
+	def parse_fields_from_header_row(self, remap_columns, skip_import):
+		remap_columns = remap_columns or frappe._dict()
+		skip_import = skip_import or []
 		fields = []
 		warnings = []
 
 		df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching()
 
 		for i, value in enumerate(self.header_row):
+			if remap_columns.get(value):
+				column_name = value
+				value = remap_columns.get(value)
+				warnings.append(_('Column {0}: Mapping column {1} to field {2}').format(
+					i, frappe.bold(column_name), frappe.bold(value)))
+
 			field = df_by_labels_and_fieldnames.get(value)
-			if not field:
+			if not field or value in skip_import:
 				field = {
 					'label': value,
 					'skip_import': True
 				}
-				if value:
+				if value and value not in skip_import:
 					warnings.append(_('Column {0}: Cannot match column {1} with any field').format(i, frappe.bold(value)))
+				elif value in skip_import:
+					warnings.append(_('Column {0}: Skipping column {1}').format(i, frappe.bold(value)))
 				else:
 					warnings.append(_('Column {0}: Skipping untitled column').format(i))
 			fields.append(field)
@@ -140,7 +155,8 @@ class Importer:
 		 'customer': df1,
 		 'Due Date': df2,
 		 'due_date': df2,
-		 'Sales Invoice Item / Item Code': df3
+		 'Item Code (Sales Invoice Item)': df3,
+		 'Sales Invoice Item:item_code': df3,
 		}
 		"""
 		out = {
@@ -159,11 +175,14 @@ class Importer:
 			for df in meta.fields:
 				if df.fieldtype not in no_value_fields:
 					# label as key
-					label = df.label if self.doctype == doctype else '{0} / {1}'.format(df.parent, df.label)
+					label = df.label if self.doctype == doctype else '{0} ({1})'.format(df.label, df.parent)
 					out[label] = df
 					# fieldname as key
 					if self.doctype == doctype:
 						out[df.fieldname] = df
+					else:
+						key = '{0}:{1}'.format(doctype, df.fieldname)
+						out[key] = df
 
 		# if autoname is based on field
 		# add an entry for "ID (Autoname Field)"
@@ -197,7 +216,7 @@ class Importer:
 	def add_serial_no_column(self, fields, data):
 		fields_with_serial_no = [
 			{
-				'label': _('Sr. No'),
+				'label': 'Sr. No',
 				'skip_import': True
 			}
 		] + fields

frappe/core/doctype/data_import_beta/data_import_beta.js

@@ -23,20 +23,65 @@ frappe.ui.form.on('Data Import Beta', {
 				.html(__('Loading import file...'))
 				.appendTo(frm.get_field('import_preview').$wrapper);
 
-			frm.call('get_preview_from_template').then(r => {
-				let preview_data = r.message;
-
-				frappe.require('/assets/js/data_import_tools.min.js', () => {
-					new frappe.data_import.ImportPreview(
-						frm.get_field('import_preview').$wrapper,
-						frm.doc.reference_doctype,
-						preview_data
-					);
+			frm
+				.call({
+					doc: frm.doc,
+					method: 'get_preview_from_template',
+					freeze: true,
+					freeze_message: __('Preparing Preview...')
+				})
+				.then(r => {
+					let preview_data = r.message;
+					frm.events.show_import_preview(frm, preview_data);
 				});
-			});
 		} else {
 			frm.get_field('import_preview').$wrapper.empty();
 		}
 		frm.toggle_display('section_import_preview', frm.doc.import_file);
+	},
+
+	show_import_preview(frm, preview_data) {
+		frappe.require('/assets/js/data_import_tools.min.js', () => {
+			new frappe.data_import.ImportPreview({
+				wrapper: frm.get_field('import_preview').$wrapper,
+				doctype: frm.doc.reference_doctype,
+				preview_data,
+				events: {
+					remap_column(column_name, fieldname) {
+						let import_json = JSON.parse(frm.doc.import_json || '{}');
+						import_json.remap_column = import_json.remap_column || {};
+						import_json.remap_column[column_name] = fieldname;
+						// if the column is remapped, remove it from skip_import
+						if (
+							import_json.skip_import &&
+							import_json.skip_import.includes(column_name)
+						) {
+							import_json.skip_import = import_json.skip_import.filter(
+								d => d !== column_name
+							);
+						}
+						frm.set_value('import_json', JSON.stringify(import_json));
+						frm.trigger('import_file');
+					},
+
+					skip_import(column_name) {
+						let import_json = JSON.parse(frm.doc.import_json || '{}');
+						import_json.skip_import = import_json.skip_import || [];
+						if (!import_json.skip_import.includes(column_name)) {
+							import_json.skip_import.push(column_name);
+						}
+						// if column is being skipped, remove it from remap_column
+						if (
+							import_json.remap_column &&
+							import_json.remap_column[column_name]
+						) {
+							delete import_json.remap_column[column_name];
+						}
+						frm.set_value('import_json', JSON.stringify(import_json));
+						frm.trigger('import_file');
+					}
+				}
+			});
+		});
 	}
 });

frappe/core/doctype/data_import_beta/data_import_beta.json

@@ -6,11 +6,12 @@
  "engine": "InnoDB",
  "field_order": [
   "reference_doctype",
-  "download_sample_file",
   "import_type",
+  "download_sample_file",
   "import_file",
   "section_import_preview",
-  "import_preview"
+  "import_preview",
+  "import_json"
  ],
  "fields": [
   {
@@ -51,10 +52,18 @@
    "fieldname": "download_sample_file",
    "fieldtype": "Button",
    "label": "Download Sample File"
+  },
+  {
+   "fieldname": "import_json",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Import JSON",
+   "options": "JSON",
+   "read_only": 1
   }
  ],
  "hide_toolbar": 1,
- "modified": "2019-08-10 02:08:33.133016",
+ "modified": "2019-08-13 19:10:02.943686",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import Beta",

frappe/core/doctype/data_import_beta/data_import_beta.py

@@ -10,6 +10,10 @@ from frappe.core.doctype.data_import.exporter_new import Exporter
 
 class DataImportBeta(Document):
 
+	def validate(self):
+		if not self.import_file:
+			self.import_json = ''
+
 	def get_preview_from_template(self):
 		if not self.import_file:
 			return
@@ -18,7 +22,8 @@ class DataImportBeta(Document):
 		file_content = f.get_content()
 
 		i = Importer(self.reference_doctype, content=file_content)
-		return i.get_data_for_import_preview()
+		import_options = frappe.parse_json(self.import_json or '{}')
+		return i.get_data_for_import_preview(import_options)
 
 
 @frappe.whitelist()

frappe/public/js/frappe/data_import/column_picker_fields.js

@@ -0,0 +1,31 @@
+export default class ColumnPickerFields extends frappe.views.ReportView {
+	show() {}
+
+	get_fields_as_options() {
+		let column_map = this.get_columns_for_picker();
+		let doctypes = [this.doctype].concat(
+			...frappe.meta
+				.get_table_fields(this.doctype)
+				.filter(df => !df.hidden)
+				.map(df => df.options)
+		);
+		// flatten array
+		return [].concat(
+			...doctypes.map(doctype => {
+				return column_map[doctype].map(df => {
+					let label = df.label;
+					let value = df.fieldname;
+					if (this.doctype !== doctype) {
+						label = `${df.label} (${doctype})`;
+						value = `${doctype}:${df.fieldname}`;
+					}
+					return {
+						label,
+						value,
+						description: value
+					};
+				});
+			})
+		);
+	}
+}

frappe/public/js/frappe/data_import/data_exporter.js

@@ -1,9 +1,6 @@
+import ColumnPickerFields from './column_picker_fields';
 frappe.provide('frappe.data_import');
 
-class ColumnPickerFields extends frappe.views.ReportView {
-	show() {}
-}
-
 frappe.data_import.DataExporter = class DataExporter {
 	constructor(doctype) {
 		this.doctype = doctype;

frappe/public/js/frappe/data_import/import_preview.js

@@ -1,21 +1,27 @@
 import DataTable from 'frappe-datatable';
+import ColumnManager from 'frappe-datatable/src/columnmanager';
+import ColumnPickerFields from './column_picker_fields';
 
 frappe.provide('frappe.data_import');
 
 frappe.data_import.ImportPreview = class ImportPreview {
-	constructor(wrapper, doctype, preview_data) {
+	constructor({ wrapper, doctype, preview_data, events = {} }) {
 		frappe.import_preview = this;
 		this.wrapper = wrapper;
 		this.doctype = doctype;
+		this.header_row = preview_data.header_row;
 		this.preview_fields = preview_data.fields;
 		this.preview_data = preview_data.data;
 		this.preview_warnings = preview_data.warnings;
+		this.events = events;
 
-		this.make_wrapper();
-		this.prepare_columns();
-		this.prepare_data();
-		this.render_warnings();
-		this.render_datatable();
+		frappe.model.with_doctype(doctype, () => {
+			this.make_wrapper();
+			this.prepare_columns();
+			this.prepare_data();
+			this.render_warnings();
+			this.render_datatable();
+		});
 	}
 
 	make_wrapper() {
@@ -55,11 +61,16 @@ frappe.data_import.ImportPreview = class ImportPreview {
 			if (this.doctype !== df.parent) {
 				column_title = `${df.label} (${df.parent})`;
 			}
+			let meta = frappe.get_meta(this.doctype);
+			if (meta.autoname === `field:${df.fieldname}`) {
+				column_title = `ID (${df.label})`;
+			}
 			return {
 				id: df.fieldname,
 				name: column_title,
 				df: df,
-				editable: true
+				editable: true,
+				align: 'left'
 			};
 		});
 	}
@@ -87,6 +98,8 @@ frappe.data_import.ImportPreview = class ImportPreview {
 	}
 
 	render_datatable() {
+		let self = this;
+
 		this.datatable = new DataTable(this.$table_preview.get(0), {
 			data: this.preview_data,
 			columns: this.columns,
@@ -97,23 +110,121 @@ frappe.data_import.ImportPreview = class ImportPreview {
 			pasteFromClipboard: true,
 			headerDropdown: [
 				{
-					label: __('Change column mapping'),
-					action: console.log
+					label: __('Remap Column'),
+					action: col => this.remap_column(col)
 				},
 				{
-					label: __("Don't Import"),
-					action: console.log
+					label: __('Skip Import'),
+					action: col => this.skip_import(col)
 				}
-			]
+			],
+			overrideComponents: {
+				ColumnManager: class CustomColumnManager extends ColumnManager {
+					getHeaderHTML(columns) {
+						let html = super.getHeaderHTML(columns);
+
+						let header_row_columns = [
+							{
+								id: '_checkbox',
+								colIndex: 0,
+								format: () => ''
+							},
+							{
+								id: 'Sr. No',
+								colIndex: 1,
+								format: () => ''
+							}
+						].concat(
+							...self.header_row.map((col, i) => {
+								return {
+									id: col,
+									name: col,
+									align: 'left',
+									dropdown: false,
+									content: col,
+									colIndex: i + 2
+								};
+							})
+						);
+
+						let header_row_html = this.rowmanager.getRowHTML(
+							header_row_columns,
+							{
+								rowIndex: 'header-row'
+							}
+						);
+						return header_row_html + html;
+					}
+				}
+			}
 		});
 
 		this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', {
 			display: 'none'
 		});
+
+		let columns = this.datatable.getColumns();
+		columns.forEach(col => {
+			if (!col.skip_import && col.df) {
+				this.datatable.style.setStyle(
+					`.dt-header .dt-cell--col-${col.colIndex}`,
+					{
+						backgroundColor: frappe.ui.color.get_color_shade(
+							'green',
+							'extra-light'
+						),
+						color: frappe.ui.color.get_color_shade('green', 'dark')
+					}
+				);
+			}
+			if (col.skip_import && col.name !== 'Sr. No') {
+				this.datatable.style.setStyle(
+					`.dt-header .dt-cell--col-${col.colIndex}`,
+					{
+						backgroundColor: frappe.ui.color.get_color_shade(
+							'orange',
+							'extra-light'
+						),
+						color: frappe.ui.color.get_color_shade('orange', 'dark')
+					}
+				);
+				this.datatable.style.setStyle(`.dt-cell--col-${col.colIndex}`, {
+					backgroundColor: frappe.ui.color.get_color_shade('white', 'light')
+				});
+			}
+		});
 	}
 
 	add_row() {
 		this.preview_data.push([]);
 		this.datatable.refresh(this.preview_data);
 	}
+
+	remap_column(col) {
+		let column_picker_fields = new ColumnPickerFields({
+			doctype: this.doctype
+		});
+		let dialog = new frappe.ui.Dialog({
+			title: __('Remap Column: {0}', [col.name]),
+			fields: [
+				{
+					fieldtype: 'Autocomplete',
+					fieldname: 'fieldname',
+					label: __('Select field'),
+					max_items: Infinity,
+					options: column_picker_fields.get_fields_as_options()
+				}
+			],
+			primary_action: ({ fieldname }) => {
+				if (!fieldname) return;
+				this.events.remap_column(col.name, fieldname);
+				dialog.hide();
+			}
+		});
+		dialog.show();
+	}
+
+	skip_import(col) {
+		this.events.skip_import(col.name);
+	}
 };