diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index e6305e2437..170627a949 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -40,6 +40,7 @@ class Importer: self.header_row = None self.data = None + # used to store date formats guessed from data rows per column self._guessed_date_formats = {} self.meta = frappe.get_meta(doctype) self.prepare_content(file_path, content) @@ -191,7 +192,9 @@ class Importer: for df in meta.fields: if df.fieldtype not in no_value_fields: # label as key - label = df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) + label = ( + df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) + ) out[label] = df # fieldname as key if self.doctype == doctype: @@ -294,25 +297,35 @@ class Importer: if missing_link_values: return {"missing_link_values": missing_link_values} - if warnings: - return warnings + # parse import data + payloads = self.get_payloads_for_import(fields, data) - for doc in docs: - doc = self.process_doc(doc) - import_log.append({"inserted": True, "name": doc.name}) + # collect warnings + warnings = [] + for payload in payloads: + warnings += payload.warnings + if warnings: + return {"warnings": warnings} + + # start import + print("Importing {0} rows...".format(len(data))) + for payload in payloads: + doc = payload.doc + row_indexes = [row[0] for row in payload.rows] + try: + doc = self.process_doc(doc) + import_log.append({"success": True, "docname": doc.name, "row_indexes": row_indexes}) + except Exception as e: + import_log.append({"success": False, "exception": frappe.get_traceback(), "row_indexes": row_indexes}) self.data_import.db_set("import_log", json.dumps(import_log)) - def get_docs_for_import(self, fields, data): - docs = [] - parse_warnings = [] + def get_payloads_for_import(self, fields, data): + payloads = [] while data: - doc, data, warnings = self.parse_next_row_for_import(fields, data) - if not warnings: - docs.append(doc) - else: - parse_warnings += warnings - return docs, parse_warnings + doc, rows, data, warnings = self.parse_next_row_for_import(fields, data) + payloads.append(frappe._dict(doc=doc, rows=rows, warnings=warnings)) + return payloads def parse_next_row_for_import(self, fields, data): doc = {} @@ -383,7 +396,7 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs - return doc, data[len(rows) :], warnings + return doc, rows, data[len(rows) :], warnings def get_first_parent_column_index(self, fields): """ @@ -406,7 +419,8 @@ class Importer: pass def insert_record(self, doc): - doc.update({"doctype": self.doctype}) + # name shouldn't be set when inserting a new record + doc.update({"doctype": self.doctype, "name": None}) new_doc = frappe.get_doc(doc) return new_doc.insert() @@ -415,6 +429,8 @@ class Importer: def has_one_mandatory_field(doctype): meta = frappe.get_meta(doctype) + # get mandatory fields with default not set + # mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] mandatory_fields = [df for df in meta.fields if df.reqd] mandatory_fields_count = len(mandatory_fields) if meta.autoname.lower() == "prompt": diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 56b90deeeb..a9439494d6 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -6,7 +6,7 @@ frappe.ui.form.on('Data Import Beta', { frm.page.hide_icon_group(); frm.trigger('import_file'); frm.trigger('reference_doctype'); - frm.trigger('show_import_log'); + // frm.trigger('show_import_log'); if (!frm.is_new()) { frm.page.set_primary_action(__('Start Import'), () => frm.events.start_import(frm) @@ -43,34 +43,46 @@ frappe.ui.form.on('Data Import Beta', { }, import_file(frm) { - if (frm.doc.import_file) { - $('') - .html(__('Loading import file...')) - .appendTo(frm.get_field('import_preview').$wrapper); - - frm - .call({ - doc: frm.doc, - method: 'get_preview_from_template', - freeze: true, - freeze_message: __('Preparing Preview...') - }) - .then(r => { - let preview_data = r.message; - frm.events.show_import_preview(frm, preview_data); - }); - } else { - frm.get_field('import_preview').$wrapper.empty(); - } frm.toggle_display('section_import_preview', frm.doc.import_file); + if (!frm.doc.import_file) { + frm.get_field('import_preview').$wrapper.empty(); + return; + } + + // load import preview + $('') + .html(__('Loading import file...')) + .appendTo(frm.get_field('import_preview').$wrapper); + + frm + .call({ + doc: frm.doc, + method: 'get_preview_from_template', + freeze: true, + freeze_message: __('Preparing Preview...') + }) + .then(r => { + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); + }); }, show_import_preview(frm, preview_data) { + let import_log = JSON.parse(frm.doc.import_log || '[]'); + + if (frm.import_preview) { + frm.import_preview.preview_data = preview_data; + frm.import_preview.import_log = import_log; + frm.import_preview.refresh(); + return; + } + frappe.require('/assets/js/data_import_tools.min.js', () => { frm.import_preview = new frappe.data_import.ImportPreview({ wrapper: frm.get_field('import_preview').$wrapper, doctype: frm.doc.reference_doctype, preview_data, + import_log, events: { remap_column(header_row_index, fieldname) { let template_options = JSON.parse(frm.doc.template_options || '{}'); @@ -123,9 +135,15 @@ frappe.ui.form.on('Data Import Beta', { let import_log = JSON.parse(frm.doc.import_log); let rows = import_log .map(log => { + if (log.inserted) { + return ` + ${log.name} + ${log.inserted ? 'Inserted' : ''} + `; + } return ` - ${log.name} - ${log.inserted ? 'Inserted' : ''} + Failed + 
${log.exception}
`; }) .join(''); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 0b3b946a58..5170e2340f 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -8,10 +8,14 @@ from frappe.model.document import Document from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter + class DataImportBeta(Document): def validate(self): - if not self.import_file: - self.import_json = '' + doc_before_save = self.get_doc_before_save() + if not self.import_file or ( + doc_before_save and doc_before_save.import_file != self.import_file + ): + self.template_options = "" def get_preview_from_template(self): if not self.import_file: @@ -44,7 +48,7 @@ class DataImportBeta(Document): return docs @frappe.whitelist() -def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type='CSV'): +def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type="CSV"): """ Download template from Exporter :param doctype: Document Type @@ -57,10 +61,11 @@ def download_template(doctype, export_fields=None, export_records=None, export_f export_fields = frappe.parse_json(export_fields) export_filters = frappe.parse_json(export_filters) - e = Exporter(doctype, + e = Exporter( + doctype, export_fields=export_fields, export_data=True, export_filters=export_filters, - file_type=file_type + file_type=file_type, ) e.build_csv_response() diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index f580293af9..0d4a09de2c 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -4,26 +4,41 @@ import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); +const SVG_ICONS = { + 'checkbox-circle-line': ` + + + + + ` +}; + frappe.data_import.ImportPreview = class ImportPreview { - constructor({ wrapper, doctype, preview_data, events = {} }) { + constructor({ wrapper, doctype, preview_data, import_log, events = {} }) { frappe.import_preview = this; this.wrapper = wrapper; this.doctype = doctype; - this.header_row = preview_data.header_row; - this.preview_fields = preview_data.fields; - this.preview_data = preview_data.data; - this.preview_warnings = preview_data.warnings; + this.preview_data = preview_data; this.events = events; + this.import_log = import_log; frappe.model.with_doctype(doctype, () => { this.make_wrapper(); - this.prepare_columns(); - this.prepare_data(); - this.render_warnings(this.preview_warnings); - this.render_datatable(); + this.refresh(); }); } + refresh() { + this.header_row = this.preview_data.header_row; + this.fields = this.preview_data.fields; + this.data = this.preview_data.data; + this.warnings = this.preview_data.warnings; + this.prepare_columns(); + this.prepare_data(); + this.render_warnings(this.warnings); + this.render_datatable(); + } + make_wrapper() { this.wrapper.html(`
@@ -43,7 +58,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { - this.columns = this.preview_fields.map((df, i) => { + this.columns = this.fields.map((df, i) => { let header_row_index = i - 1; if (df.skip_import) { return { @@ -54,7 +69,13 @@ frappe.data_import.ImportPreview = class ImportPreview { focusable: false, header_row_index, format: (value, row, column, data) => { - return `
${value}
`; + let html = `
${value}
`; + if (df.label === 'Sr. No' && this.is_row_imported(row)) { + html = ` +
${SVG_ICONS['checkbox-circle-line'] + html}
+ `; + } + return html; } }; } @@ -79,7 +100,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_data() { - this.preview_data = this.preview_data.map(row => { + this.data = this.data.map(row => { return row.map(cell => { if (cell == null) { return ''; @@ -104,7 +125,7 @@ frappe.data_import.ImportPreview = class ImportPreview { let self = this; this.datatable = new DataTable(this.$table_preview.get(0), { - data: this.preview_data, + data: this.data, columns: this.columns, layout: 'fixed', cellHeight: 35, @@ -144,7 +165,9 @@ frappe.data_import.ImportPreview = class ImportPreview { columns.forEach(col => { if (!col.skip_import && col.df) { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ + col.colIndex + } .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'green', @@ -156,7 +179,9 @@ frappe.data_import.ImportPreview = class ImportPreview { } if (col.skip_import && col.name !== 'Sr. No') { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ + col.colIndex + } .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'orange', @@ -170,11 +195,15 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } }); + this.datatable.style.setStyle(`svg.import-success`, { + width: '16px', + fill: frappe.ui.color.get_color_shade('green', 'dark') + }); } add_row() { - this.preview_data.push([]); - this.datatable.refresh(this.preview_data); + this.data.push([]); + this.datatable.refresh(this.data); } remap_column(col) { @@ -204,4 +233,11 @@ frappe.data_import.ImportPreview = class ImportPreview { skip_import(col) { this.events.skip_import(col.header_row_index); } + + is_row_imported(row) { + let serial_no = row[0].content; + return this.import_log.find(log => { + return log.success && log.row_indexes.includes(serial_no); + }); + } };