From bae336154f0c2d5eaaf8baabc843b8f488c8bd6d Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 13 Aug 2019 00:57:52 +0530 Subject: [PATCH] fix: Parse template and prepare preview data - Match column header with label or fieldname - Remove empty rows and columns - Merge ID and autoname field --- .../core/doctype/data_import/exporter_new.py | 16 +- .../core/doctype/data_import/importer_new.py | 157 +++++++++++++++++- .../data_import_beta/data_import_beta.js | 5 +- .../data_import_beta/data_import_beta.py | 6 +- .../js/frappe/data_import/data_exporter.js | 64 ++++--- .../js/frappe/data_import/import_preview.js | 103 +++++++++--- frappe/public/js/frappe/utils/utils.js | 10 ++ 7 files changed, 309 insertions(+), 52 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index f541dd74be..b1ae89c0a4 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -48,7 +48,21 @@ class Exporter: 'reqd': 1, 'parent': self.doctype }) - return [name_field] + self.get_exportable_fields(self.doctype) + + parent_fields = self.get_exportable_fields(self.doctype) + + # if autoname is based on field + # then merge ID and the field column title as "ID (Autoname Field)" + autoname = self.meta.autoname + if autoname and autoname.startswith('field:'): + fieldname = autoname[len('field:'):] + autoname_field = self.meta.get_field(fieldname) + if autoname_field: + name_field.label = 'ID ({})'.format(autoname_field.label) + # remove the autoname field as it is a duplicate of ID field + parent_fields = [df for df in parent_fields if df.fieldname != autoname_field.fieldname] + + return [name_field] + parent_fields def get_exportable_children_fields(self): diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 885b50e749..46b84d85bf 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -10,6 +10,7 @@ from frappe import _ from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT from frappe.utils.csvutils import read_csv_content from frappe.exceptions import ValidationError, MandatoryError +from frappe.model import display_fieldtypes, no_value_fields, table_fields # set user lang # set flags: frappe.flags.in_import = True @@ -18,6 +19,8 @@ from frappe.exceptions import ValidationError, MandatoryError # check empty row # validate naming +INVALID_VALUES = ['', None] + class Importer: def __init__(self, doctype, file_path=None, content=None, options=None): @@ -33,6 +36,7 @@ class Importer: elif content: self.read_content(content) + self.remove_empty_rows_and_columns() def read_file(self, file_path): extn = file_path.split('.')[1] @@ -53,8 +57,159 @@ class Importer: self.data = data[1:] + def remove_empty_rows_and_columns(self): + self.row_index_map = [] + removed_rows = [] + removed_columns = [] + + # remove empty rows + data = [] + for i, row in enumerate(self.data): + if all(v in INVALID_VALUES for v in row): + # empty row + removed_rows.append(i) + else: + data.append(row) + self.row_index_map.append(i) + + # remove empty columns + # a column with a header and no data is a valid column + # a column with no header and no data will be removed + header_row = [] + for i, column in enumerate(self.header_row): + column_values = [row[i] for row in data] + values = [column] + column_values + if all(v in INVALID_VALUES for v in values): + # empty column + removed_columns.append(i) + else: + header_row.append(column) + + data_without_empty_columns = [] + # remove empty columns from data + for i, row in enumerate(data): + new_row = [v for j, v in enumerate(row) if j not in removed_columns] + data_without_empty_columns.append(new_row) + + self.data = data_without_empty_columns + self.header_row = header_row + + + def get_data_for_import_preview(self): + fields, fields_warnings = self.parse_fields_from_header_row() + formats, formats_warnings = self.parse_formats_from_first_10_rows() + fields, data = self.add_serial_no_column(fields, self.data) + + warnings = fields_warnings + formats_warnings + + return dict( + fields=fields, + data=data, + warnings=warnings + ) + + + def parse_fields_from_header_row(self): + fields = [] + warnings = [] + + df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() + + for i, value in enumerate(self.header_row): + field = df_by_labels_and_fieldnames.get(value) + if not field: + field = { + 'label': value, + 'skip_import': True + } + if value: + warnings.append(_('Column {0}: Cannot match column {1} with any field').format(i, frappe.bold(value))) + else: + warnings.append(_('Column {0}: Skipping untitled column').format(i)) + fields.append(field) + + return fields, warnings + + + def build_fields_dict_for_column_matching(self): + """ + Build a dict with various keys to match with column headers and value as docfield + The keys can be label or fieldname + { + 'Customer': df1, + 'customer': df1, + 'Due Date': df2, + 'due_date': df2, + 'Sales Invoice Item / Item Code': df3 + } + """ + out = { + 'ID': frappe._dict({ + 'fieldtype': 'Data', + 'fieldname': 'name', + 'label': 'ID', + 'reqd': 1, + 'parent': self.doctype + }) + } + + doctypes = [self.doctype] + [df.options for df in self.meta.get_table_fields()] + for doctype in doctypes: + meta = frappe.get_meta(doctype) + for df in meta.fields: + if df.fieldtype not in no_value_fields: + # label as key + label = df.label if self.doctype == doctype else '{0} / {1}'.format(df.parent, df.label) + out[label] = df + # fieldname as key + if self.doctype == doctype: + out[df.fieldname] = df + + # if autoname is based on field + # add an entry for "ID (Autoname Field)" + autoname = self.meta.autoname + if autoname and autoname.startswith('field:'): + fieldname = autoname[len('field:'):] + autoname_field = self.meta.get_field(fieldname) + if autoname_field: + out['ID ({})'.format(autoname_field.label)] = autoname_field + # ID field should also map to the autoname field + out['ID'] = autoname_field + + return out + + + def parse_formats_from_first_10_rows(self): + """ + Returns a list of column descriptors for columns that might need parsing. + For e.g if it is a Date column return the Date format + [ + [['Data']], + [['Date', '%m/%d/%y']], + [['Currency', '#,###.##']], + ... + ] + """ + formats = [] + return formats, [] + + + def add_serial_no_column(self, fields, data): + fields_with_serial_no = [ + { + 'label': _('Sr. No'), + 'skip_import': True + } + ] + fields + + data_with_serial_no = [] + for i, row in enumerate(data): + data_with_serial_no.append([self.row_index_map[i] + 1] + row) + + return fields_with_serial_no, data_with_serial_no + + def parse_data_for_import(self, row, index): - INVALID_VALUES = ['', None] if all(v in INVALID_VALUES for v in row): # empty row diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index be9d315dd3..1277b88c55 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -24,12 +24,13 @@ frappe.ui.form.on('Data Import Beta', { .appendTo(frm.get_field('import_preview').$wrapper); frm.call('get_preview_from_template').then(r => { - let csv_array = r.message; + let preview_data = r.message; frappe.require('/assets/js/data_import_tools.min.js', () => { new frappe.data_import.ImportPreview( frm.get_field('import_preview').$wrapper, - csv_array + frm.doc.reference_doctype, + preview_data ); }); }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index b4514c2dc9..d4a72c8ecb 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document -from frappe.utils.csvutils import read_csv_content +from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter class DataImportBeta(Document): @@ -16,9 +16,9 @@ class DataImportBeta(Document): f = frappe.get_doc('File', { 'file_url': self.import_file }) file_content = f.get_content() - csv_content = read_csv_content(file_content) - return csv_content[:100] + i = Importer(self.reference_doctype, content=file_content) + return i.get_data_for_import_preview() @frappe.whitelist() diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 9a366911c1..2b3d893667 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -13,10 +13,6 @@ frappe.data_import.DataExporter = class DataExporter { } make_dialog() { - let column_map = new ColumnPickerFields({ - doctype: this.doctype - }).get_columns_for_picker(); - let doctypes = [this.doctype].concat( ...frappe.meta .get_table_fields(this.doctype) @@ -78,15 +74,7 @@ frappe.data_import.DataExporter = class DataExporter { on_change: () => { this.update_primary_action(); }, - options: column_map[doctype].map(df => ({ - label: __(df.label), - value: df.fieldname, - danger: df.reqd, - checked: df.reqd, - description: `${df.fieldname} ${ - df.reqd ? __('(Mandatory)') : '' - }` - })) + options: this.get_multicheck_options(doctype) }; }) ], @@ -161,12 +149,8 @@ frappe.data_import.DataExporter = class DataExporter { ${__('Unselect All')} - `).on('click', '[data-action]', e => { - let $target = $(e.currentTarget); - let action = $target.data('action'); - let method = this[action]; - method ? this[action]() : null; - }); + `); + frappe.utils.bind_actions_with_class($select_all_buttons, this); this.dialog .get_field('select_all_buttons') .$wrapper.html($select_all_buttons); @@ -239,11 +223,7 @@ frappe.data_import.DataExporter = class DataExporter { } else { message = __('{0} records will be exported', [value]); } - this.dialog.set_df_property( - 'export_records', - 'description', - message - ); + this.dialog.set_df_property('export_records', 'description', message); this.update_primary_action(value); }); @@ -277,4 +257,40 @@ frappe.data_import.DataExporter = class DataExporter { }); }, {}); } + + get_multicheck_options(doctype) { + if (!this.column_map) { + this.column_map = new ColumnPickerFields({ + doctype: this.doctype + }).get_columns_for_picker(); + } + + let autoname_field = null; + let meta = frappe.get_meta(doctype); + if (meta.autoname && meta.autoname.startsWith('field:')) { + let fieldname = meta.autoname.slice('field:'.length); + autoname_field = frappe.meta.get_field(doctype, fieldname); + } + + return this.column_map[doctype] + .filter(df => { + if (autoname_field && df.fieldname === autoname_field.fieldname) { + return false; + } + return true; + }) + .map(df => { + let label = __(df.label); + if (autoname_field && df.fieldname === 'name') { + label = label + ` (${__(autoname_field.label)})`; + } + return { + label, + value: df.fieldname, + danger: df.reqd, + checked: df.reqd, + description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}` + }; + }); + } }; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index eeca9deb3a..e963a20477 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -3,17 +3,69 @@ import DataTable from 'frappe-datatable'; frappe.provide('frappe.data_import'); frappe.data_import.ImportPreview = class ImportPreview { - constructor(wrapper, csv_array) { + constructor(wrapper, doctype, preview_data) { frappe.import_preview = this; this.wrapper = wrapper; - this.csv_array = csv_array; + this.doctype = doctype; + this.preview_fields = preview_data.fields; + this.preview_data = preview_data.data; + this.preview_warnings = preview_data.warnings; - this.prepare_csv_array(); + this.make_wrapper(); + this.prepare_columns(); + this.prepare_data(); + this.render_warnings(); this.render_datatable(); } - prepare_csv_array() { - this.csv_array = this.csv_array.map(row => { + make_wrapper() { + this.wrapper.html(` +
+
+
+
+ +
+
+ `); + frappe.utils.bind_actions_with_class(this.wrapper, this); + + this.$warnings = this.wrapper.find('.warnings'); + this.$table_preview = this.wrapper.find('.table-preview'); + } + + prepare_columns() { + this.columns = this.preview_fields.map(df => { + if (df.skip_import) { + return { + id: frappe.utils.get_random(6), + name: df.label, + skip_import: true, + editable: false, + focusable: false, + format: (value, row, column, data) => { + return `
${value}
`; + } + }; + } + + let column_title = df.label; + if (this.doctype !== df.parent) { + column_title = `${df.label} (${df.parent})`; + } + return { + id: df.fieldname, + name: column_title, + df: df, + editable: true + }; + }); + } + + prepare_data() { + this.preview_data = this.preview_data.map(row => { return row.map(cell => { if (cell == null) { return ''; @@ -23,19 +75,26 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } + render_warnings() { + let warning_html = this.preview_warnings + .map(warning => { + return `
${warning}
`; + }) + .join(''); + + let html = `
${warning_html}
`; + this.$warnings.html(html); + } + render_datatable() { - let columns = this.csv_array[0].map(col => { - return { - id: col, - name: col - }; - }); - let data = this.csv_array.slice(1); - this.datatable = new DataTable(this.wrapper.get(0), { - data, - columns, + this.datatable = new DataTable(this.$table_preview.get(0), { + data: this.preview_data, + columns: this.columns, layout: 'fixed', cellHeight: 35, + serialNoColumn: false, + checkboxColumn: true, + pasteFromClipboard: true, headerDropdown: [ { label: __('Change column mapping'), @@ -48,11 +107,13 @@ frappe.data_import.ImportPreview = class ImportPreview { ] }); - this.datatable.style.setStyle( - '.dt-dropdown__list-item:nth-child(-n+4)', - { - display: 'none' - } - ); + this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { + display: 'none' + }); + } + + add_row() { + this.preview_data.push([]); + this.datatable.refresh(this.preview_data); } }; diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 1486e819fc..a285264272 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -725,6 +725,16 @@ Object.assign(frappe.utils, { }, is_rtl() { return ["ar", "he", "fa"].includes(frappe.boot.lang); + }, + bind_actions_with_class($el, class_instance) { + $($el).on('click', '[data-action]', e => { + let $target = $(e.currentTarget); + let action = $target.data('action'); + let method = class_instance[action]; + method ? class_instance[action]() : null; + }); + + return $el; } });