From 62be3abc0cd793047dc97e3762147e51b7f3eb48 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 4 Aug 2019 14:06:35 +0530 Subject: [PATCH 01/83] feat: Initialize new Importer - Read csv file or content - Parse data for import - Guess date format - Basic tests for parsing csv content --- .../core/doctype/data_import/importer_new.py | 205 ++++++++++++++++++ .../doctype/data_import/test_importer_new.py | 53 +++++ 2 files changed, 258 insertions(+) create mode 100644 frappe/core/doctype/data_import/importer_new.py create mode 100644 frappe/core/doctype/data_import/test_importer_new.py diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py new file mode 100644 index 0000000000..885b50e749 --- /dev/null +++ b/frappe/core/doctype/data_import/importer_new.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import io +import csv +import frappe +from datetime import datetime +from frappe import _ +from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT +from frappe.utils.csvutils import read_csv_content +from frappe.exceptions import ValidationError, MandatoryError + +# set user lang +# set flags: frappe.flags.in_import = True + +# during import + # check empty row + # validate naming + +class Importer: + + def __init__(self, doctype, file_path=None, content=None, options=None): + self.doctype = doctype + self.header_row = None + self.data = None + self.skipped_rows = [] + self._guessed_date_formats = {} + self.meta = frappe.get_meta(doctype) + + if file_path: + self.read_file(file_path) + elif content: + self.read_content(content) + + + def read_file(self, file_path): + extn = file_path.split('.')[1] + + file_content = None + with io.open(file_path, mode='rb') as f: + file_content = f.read() + + if extn == 'csv': + data = read_csv_content(file_content) + self.header_row = data[0] + self.data = data[1:] + + + def read_content(self, content): + data = read_csv_content(content) + self.header_row = data[0] + self.data = data[1:] + + + def parse_data_for_import(self, row, index): + INVALID_VALUES = ['', None] + + if all(v in INVALID_VALUES for v in row): + # empty row + self.skipped_rows.append([index, 'Empty Row']) + return + + doc = {} + + for i, field in enumerate(self.header_row): + if not self.meta.has_field(field): + continue + + df = self.meta.get_field(field) + value = row[i] + + if value in INVALID_VALUES: + if df.reqd: + raise MandatoryError(_('Row {0}: {1} is a mandatory field').format(i, frappe.bold(df.label))) + else: + value = None + + # convert boolean values to 0 or 1 + if df.fieldtype == 'Check' and value.lower().strip() in ['t', 'f', 'true', 'false']: + value = value.lower().strip() + value = 1 if value in ['t', 'true'] else 0 + + if df.fieldtype in ['Int', 'Check']: + value = cint(value) + elif df.fieldtype in ['Float', 'Percent', 'Currency']: + value = flt(value) + elif df.fieldtype in ['Date', 'Datetime']: + value = self.parse_date_format(value, df) + + doc[df.fieldname] = value + + return frappe._dict(doc) + + + def parse_date_format(self, value, df): + date_format = self.guess_date_format_for_column(df.fieldname) + return datetime.strptime(value, date_format) + + + def guess_date_format_for_column(self, fieldname): + ''' Guesses date format for a column by parsing the first 10 values in the column, + getting the date format and then returning the one which has the maximum frequency + ''' + PARSE_ROW_COUNT = 10 + + if not self._guessed_date_formats.get(fieldname): + column_index = -1 + + for i, field in enumerate(self.header_row): + if self.meta.has_field(field) and field == fieldname: + column_index = i + break + + if column_index == -1: + self._guessed_date_formats[fieldname] = None + + column_values = map(lambda x: x[column_index], self.data[:PARSE_ROW_COUNT]) + column_values = filter(lambda x: bool(x), column_values) + date_formats = list(map(lambda x: guess_date_format(x), column_values)) + max_occurred_date_format = max(set(date_formats), key=date_formats.count) + + self._guessed_date_formats[fieldname] = max_occurred_date_format + + return self._guessed_date_formats[fieldname] + + + def import_data(self): + print('Importing {0} rows...'.format(len(self.data))) + + for i, row in enumerate(self.data): + doc = self.parse_data_for_import(row, i) + + if doc: + break + + + +DATE_FORMATS = [ + r'%Y-%m-%d', + r'%d-%m-%Y', + r'%m-%d-%Y', + + r'%Y/%m/%d', + r'%d/%m/%Y', + r'%m/%d/%Y', + + r'%m/%d/%y', + r'%d/%m/%y', + + r'%Y.%m.%d', + r'%d.%m.%Y', + r'%m.%d.%Y', +] + +TIME_FORMATS = [ + r'%H:%M:%S.%f', + r'%H:%M:%S', + r'%H:%M', + + r'%I:%M:%S.%f %p', + r'%I:%M:%S %p', + r'%I:%M %p', +] + +def guess_date_format(date_string): + date_string = date_string.strip() + + _date = None + _time = None + + if ' ' in date_string: + _date, _time = date_string.split(' ', 1) + else: + _date = date_string + + date_format = None + time_format = None + + for f in DATE_FORMATS: + try: + parsed_date = datetime.strptime(_date, f) + date_format = f + break + except ValueError: + pass + + if _time: + for f in TIME_FORMATS: + try: + parsed_time = datetime.strptime(_time, f) + time_format = f + break + except ValueError: + pass + + full_format = date_format + if time_format: + full_format += ' ' + time_format + return full_format + + +def import_data(doctype, file_path): + i = Importer(doctype, file_path) + i.import_data() diff --git a/frappe/core/doctype/data_import/test_importer_new.py b/frappe/core/doctype/data_import/test_importer_new.py new file mode 100644 index 0000000000..396cac577a --- /dev/null +++ b/frappe/core/doctype/data_import/test_importer_new.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +import datetime +import unittest +import frappe +from frappe.core.doctype.data_import.importer_new import Importer + +content_empty_rows = '''title,start_date,idx,show_title +,,, +est phasellus sit amet,5/20/2019,52,1 +nibh in,7/29/2019,77,1 +''' + +content_mandatory_missing = '''title,start_date,idx,show_title +,5/20/2019,52,1 +''' + +content_convert_value = '''title,start_date,idx,show_title +est phasellus sit amet,5/20/2019,52,True +''' + +content_invalid_column = '''title,start_date,idx,show_title,invalid_column +est phasellus sit amet,5/20/2019,52,True,invalid value +''' + + +class TestImporter(unittest.TestCase): + def test_should_skip_empty_rows(self): + i = Importer('Web Page', content=content_empty_rows) + i.import_data() + self.assertEqual(len(i.skipped_rows), 1) + + def test_should_throw_if_mandatory_is_missing(self): + i = Importer('Web Page', content=content_mandatory_missing) + self.assertRaises(frappe.MandatoryError, i.import_data) + + def test_should_convert_value_based_on_fieldtype(self): + i = Importer('Web Page', content=content_convert_value) + doc = i.parse_data_for_import(i.data[0], 0) + + self.assertEqual(type(doc.show_title), int) + self.assertEqual(type(doc.idx), int) + self.assertEqual(type(doc.start_date), datetime.datetime) + + def test_should_ignore_invalid_columns(self): + i = Importer('Web Page', content=content_invalid_column) + doc = i.parse_data_for_import(i.data[0], 0) + + self.assertTrue('invalid_column' not in doc) + self.assertTrue('title' in doc) From bdc5ec32df64de20ddd002bad52158e2b1c992e1 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 10 Aug 2019 21:33:53 +0530 Subject: [PATCH 02/83] feat: New Doctype - Data Import Beta - Import Preview using DataTable --- .../data_import_beta/data_import_beta.js | 41 ++++++++++ .../data_import_beta/data_import_beta.json | 79 +++++++++++++++++++ .../data_import_beta/data_import_beta.py | 44 +++++++++++ frappe/public/build.json | 3 + .../js/frappe/data_import/import_preview.js | 58 ++++++++++++++ frappe/public/js/frappe/data_import/index.js | 2 + 6 files changed, 227 insertions(+) create mode 100644 frappe/core/doctype/data_import_beta/data_import_beta.js create mode 100644 frappe/core/doctype/data_import_beta/data_import_beta.json create mode 100644 frappe/core/doctype/data_import_beta/data_import_beta.py create mode 100644 frappe/public/js/frappe/data_import/import_preview.js create mode 100644 frappe/public/js/frappe/data_import/index.js diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js new file mode 100644 index 0000000000..be9d315dd3 --- /dev/null +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -0,0 +1,41 @@ +// Copyright (c) 2019, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import Beta', { + refresh(frm) { + frm.trigger('import_file'); + frm.trigger('reference_doctype'); + }, + + reference_doctype(frm) { + // + }, + + download_sample_file(frm) { + frappe.require('/assets/js/data_import_tools.min.js', () => { + new frappe.data_import.DataExporter(frm.doc.reference_doctype); + }); + }, + + import_file(frm) { + if (frm.doc.import_file) { + $('') + .html(__('Loading import file...')) + .appendTo(frm.get_field('import_preview').$wrapper); + + frm.call('get_preview_from_template').then(r => { + let csv_array = r.message; + + frappe.require('/assets/js/data_import_tools.min.js', () => { + new frappe.data_import.ImportPreview( + frm.get_field('import_preview').$wrapper, + csv_array + ); + }); + }); + } else { + frm.get_field('import_preview').$wrapper.empty(); + } + frm.toggle_display('section_import_preview', frm.doc.import_file); + } +}); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json new file mode 100644 index 0000000000..8ae8315b33 --- /dev/null +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -0,0 +1,79 @@ +{ + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "download_sample_file", + "import_type", + "import_file", + "section_import_preview", + "import_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "Insert new records\nUpdate existing records", + "reqd": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Import Preview" + }, + { + "depends_on": "reference_doctype", + "fieldname": "download_sample_file", + "fieldtype": "Button", + "label": "Download Sample File" + } + ], + "hide_toolbar": 1, + "modified": "2019-08-10 02:08:33.133016", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import Beta", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py new file mode 100644 index 0000000000..b4514c2dc9 --- /dev/null +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.document import Document +from frappe.utils.csvutils import read_csv_content +from frappe.core.doctype.data_import.exporter_new import Exporter + +class DataImportBeta(Document): + + def get_preview_from_template(self): + if not self.import_file: + return + + f = frappe.get_doc('File', { 'file_url': self.import_file }) + file_content = f.get_content() + csv_content = read_csv_content(file_content) + + return csv_content[:100] + + +@frappe.whitelist() +def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type='CSV'): + """ + Download template from Exporter + :param doctype: Document Type + :param export_fields=None: Fields to export as dict {'Sales Invoice': ['name', 'customer'], 'Sales Invoice Item': ['item_code']} + :param export_records=None: One of 'all', 'last_10_records', 'by_filter' + :param export_filters: Filter dict + :param file_type: File type to export into + """ + + export_fields = frappe.parse_json(export_fields) + export_filters = frappe.parse_json(export_filters) + + e = Exporter(doctype, + export_fields=export_fields, + export_data=True, + export_filters=export_filters, + file_type=file_type + ) + e.build_csv_response() diff --git a/frappe/public/build.json b/frappe/public/build.json index 2c502c5f72..a88e0f5d54 100755 --- a/frappe/public/build.json +++ b/frappe/public/build.json @@ -325,5 +325,8 @@ ], "js/barcode_scanner.min.js": [ "public/js/frappe/barcode_scanner/quagga.js" + ], + "js/data_import_tools.min.js": [ + "public/js/frappe/data_import/index.js" ] } diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js new file mode 100644 index 0000000000..eeca9deb3a --- /dev/null +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -0,0 +1,58 @@ +import DataTable from 'frappe-datatable'; + +frappe.provide('frappe.data_import'); + +frappe.data_import.ImportPreview = class ImportPreview { + constructor(wrapper, csv_array) { + frappe.import_preview = this; + this.wrapper = wrapper; + this.csv_array = csv_array; + + this.prepare_csv_array(); + this.render_datatable(); + } + + prepare_csv_array() { + this.csv_array = this.csv_array.map(row => { + return row.map(cell => { + if (cell == null) { + return ''; + } + return cell; + }); + }); + } + + render_datatable() { + let columns = this.csv_array[0].map(col => { + return { + id: col, + name: col + }; + }); + let data = this.csv_array.slice(1); + this.datatable = new DataTable(this.wrapper.get(0), { + data, + columns, + layout: 'fixed', + cellHeight: 35, + headerDropdown: [ + { + label: __('Change column mapping'), + action: console.log + }, + { + label: __("Don't Import"), + action: console.log + } + ] + }); + + this.datatable.style.setStyle( + '.dt-dropdown__list-item:nth-child(-n+4)', + { + display: 'none' + } + ); + } +}; diff --git a/frappe/public/js/frappe/data_import/index.js b/frappe/public/js/frappe/data_import/index.js new file mode 100644 index 0000000000..626d17c85d --- /dev/null +++ b/frappe/public/js/frappe/data_import/index.js @@ -0,0 +1,2 @@ +import './import_preview'; +import './data_exporter'; From cf796cf29bdc71845842b876ef754d4d6815a7fc Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 10 Aug 2019 21:34:01 +0530 Subject: [PATCH 03/83] feat: Initialize Exporter - Export into a simple format - Basic tests for exporter - DataExporter: user friendly exporter dialog --- .../core/doctype/data_import/exporter_new.py | 163 ++++++++++ .../doctype/data_import/test_exporter_new.py | 38 +++ .../core/doctype/data_import_beta/__init__.py | 0 .../js/frappe/data_import/data_exporter.js | 280 ++++++++++++++++++ .../js/frappe/form/controls/multicheck.js | 3 +- .../js/frappe/form/sidebar/form_sidebar.js | 2 +- .../js/frappe/views/reports/report_view.js | 3 +- 7 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 frappe/core/doctype/data_import/exporter_new.py create mode 100644 frappe/core/doctype/data_import/test_exporter_new.py create mode 100644 frappe/core/doctype/data_import_beta/__init__.py create mode 100644 frappe/public/js/frappe/data_import/data_exporter.js diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py new file mode 100644 index 0000000000..f541dd74be --- /dev/null +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe +from frappe import _ +from frappe.model import display_fieldtypes, no_value_fields, table_fields +from frappe.utils.csvutils import build_csv_response + +class Exporter: + def __init__(self, doctype, export_fields=None, export_data=False, export_filters=None, file_type='CSV'): + """ + Exports records of a DocType for use with Importer + :param doctype: Document Type to export + :param export_fields=None: One of 'All', 'Mandatory' or {'DocType': ['field1', 'field2'], 'Child DocType': ['childfield1']} + :param export_data=False: Whether to export data as well + :param export_filters=None: The filters (dict or list) which is used to query the records + :param file_type: One of 'Excel' or 'CSV' + """ + self.doctype = doctype + self.meta = frappe.get_meta(doctype) + self.export_fields = export_fields + + # this will contain the csv content + self.csv_array = [] + + # fields that get exported + # can be All, Mandatory or User Selected Fields + self.fields = self.get_all_exportable_fields() + self.add_header() + + if export_data: + self.data = self.get_data_to_export(export_filters) + else: + self.data = [] + self.add_data() + + + def get_all_exportable_fields(self): + return self.get_exportable_parent_fields() + self.get_exportable_children_fields() + + + def get_exportable_parent_fields(self): + name_field = frappe._dict({ + 'fieldtype': 'Data', + 'fieldname': 'name', + 'label': 'ID', + 'reqd': 1, + 'parent': self.doctype + }) + return [name_field] + self.get_exportable_fields(self.doctype) + + + def get_exportable_children_fields(self): + children = [df.options for df in self.meta.fields if df.fieldtype in table_fields] + children_fields = [] + for child in children: + children_fields += self.get_exportable_fields(child) + + return children_fields + + + def get_exportable_fields(self, doctype): + def is_exportable(df): + return ( + df.fieldtype not in display_fieldtypes + and df.fieldtype not in no_value_fields + and not df.hidden + ) + + meta = frappe.get_meta(doctype) + + # filter out layout fields + fields = [df for df in meta.fields if is_exportable(df)] + + if self.export_fields == 'All': + return fields + + if self.export_fields == 'Mandatory': + return [df for df in fields if df.reqd] + + if isinstance(self.export_fields, dict): + whitelist = self.export_fields.get(doctype, []) + return [df for df in fields if df.fieldname in whitelist] + + return [df for df in fields if df.reqd or df.in_list_view or df.bold] + + + def get_data_to_export(self, filters=None): + frappe.permissions.can_export(self.doctype, raise_exception=True) + + def get_column_name(df): + return '`tab{0}`.`{1}`'.format(df.parent, df.fieldname) + + fieldnames = [get_column_name(df) for df in self.fields] + + if self.meta.is_nested_set(): + order_by = '`tab{0}`.`lft` ASC'.format(self.doctype) + else: + order_by = '`tab{0}`.`creation` DESC'.format(self.doctype) + + data = frappe.db.get_list(self.doctype, + filters=filters, + fields=fieldnames, + limit_page_length=None, + order_by=order_by, + as_list=1, + ) + + return self.remove_duplicate_parent_values(data) + + def remove_duplicate_parent_values(self, data): + out = [] + + parent_fields = self.get_exportable_parent_fields() + parent_fields_count = len(parent_fields) + + current_name = None + # first column is always name, can be used to find duplicate rows + for row in data: + row = list(row) + name = row[0] + + if name != current_name: + current_name = name + out.append(row) + + elif name == current_name: + # remove the parent values from the row + row = row[parent_fields_count:] + # replace them with None values and add back the row contents + row = [None] * parent_fields_count + row + out.append(row) + + return out + + + def add_header(self): + def get_label(df): + if df.parent == self.doctype: + return df.label + else: + return '{0} / {1}'.format(df.parent, df.label) + + header = [get_label(df) for df in self.fields] + self.csv_array.append(header) + + + def add_data(self): + self.csv_array += self.data + + + def get_csv_array(self): + return self.csv_array + + def build_csv_response(self): + csv_array = self.csv_array + + if not self.data: + # add 5 empty rows + csv_array += [[]] * 5 + + build_csv_response(csv_array, self.doctype) diff --git a/frappe/core/doctype/data_import/test_exporter_new.py b/frappe/core/doctype/data_import/test_exporter_new.py new file mode 100644 index 0000000000..32cd7a6a46 --- /dev/null +++ b/frappe/core/doctype/data_import/test_exporter_new.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +import unittest +import frappe +from frappe.core.doctype.data_import.exporter_new import Exporter + + +class TestExporter(unittest.TestCase): + def test_exports_mandatory_fields(self): + e = Exporter('Web Page', export_fields='Mandatory') + csv_array = e.get_csv_array() + header_row = csv_array[0] + self.assertEqual(header_row, ['ID (name)', 'Title (title)']) + + + def test_exports_all_fields(self): + e = Exporter('Web Page', export_fields='All') + csv_array = e.get_csv_array() + header = csv_array[0] + self.assertEqual(len(header), 23) + + + def test_exports_selected_fields(self): + export_fields = ['title', 'route', 'published'] + e = Exporter('Web Page', export_fields=export_fields) + csv_array = e.get_csv_array() + header = csv_array[0] + self.assertEqual(header, ['Title (title)', 'Route (route)', 'Published (published)']) + + + def test_exports_data(self): + e = Exporter('ToDo', export_fields='All', export_data=True) + todo_records = frappe.db.count('ToDo') + csv_array = e.get_csv_array() + self.assertEqual(len(csv_array), todo_records + 1) diff --git a/frappe/core/doctype/data_import_beta/__init__.py b/frappe/core/doctype/data_import_beta/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js new file mode 100644 index 0000000000..9a366911c1 --- /dev/null +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -0,0 +1,280 @@ +frappe.provide('frappe.data_import'); + +class ColumnPickerFields extends frappe.views.ReportView { + show() {} +} + +frappe.data_import.DataExporter = class DataExporter { + constructor(doctype) { + this.doctype = doctype; + frappe.model.with_doctype(doctype, () => { + this.make_dialog(); + }); + } + + make_dialog() { + let column_map = new ColumnPickerFields({ + doctype: this.doctype + }).get_columns_for_picker(); + + let doctypes = [this.doctype].concat( + ...frappe.meta + .get_table_fields(this.doctype) + .filter(df => !df.hidden) + .map(df => df.options) + ); + + this.dialog = new frappe.ui.Dialog({ + title: __('Export Data'), + fields: [ + { + fieldtype: 'Select', + fieldname: 'export_records', + label: __('Export Records'), + options: [ + { + label: __('Export All Records'), + value: 'all' + }, + // { + // label: __('Export 10 Records'), + // value: 'last_10_records' + // }, + { + label: __('Export Filtered Records'), + value: 'by_filter' + } + ], + default: 'all', + change: () => { + this.update_record_count_message(); + } + }, + { + fieldtype: 'HTML', + fieldname: 'filter_area', + depends_on: doc => doc.export_records === 'by_filter' + }, + { + fieldtype: 'Select', + fieldname: 'file_type', + label: __('File Type'), + options: ['Excel', 'CSV'], + default: 'CSV' + }, + { + fieldtype: 'Section Break' + }, + { + fieldtype: 'HTML', + fieldname: 'select_all_buttons' + }, + ...doctypes.map(doctype => { + return { + label: __(doctype), + fieldname: doctype, + fieldtype: 'MultiCheck', + columns: 2, + on_change: () => { + this.update_primary_action(); + }, + options: column_map[doctype].map(df => ({ + label: __(df.label), + value: df.fieldname, + danger: df.reqd, + checked: df.reqd, + description: `${df.fieldname} ${ + df.reqd ? __('(Mandatory)') : '' + }` + })) + }; + }) + ], + primary_action_label: __('Export'), + primary_action: values => { + this.export_records(values); + } + }); + + this.make_filter_area(); + this.make_select_all_buttons(); + this.update_record_count_message(); + + this.dialog.show(); + } + + export_records() { + let method = + '/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_template'; + + let multicheck_fields = this.dialog.fields + .filter(df => df.fieldtype === 'MultiCheck') + .map(df => df.fieldname); + + let values = this.dialog.get_values(); + + let doctype_field_map = Object.assign({}, values); + for (let key in doctype_field_map) { + if (!multicheck_fields.includes(key)) { + delete doctype_field_map[key]; + } + } + + let filters = null; + if (values.export_records === 'by_filter') { + filters = this.get_filters(); + } + + open_url_post(method, { + doctype: this.doctype, + file_type: values.file_type, + export_records: values.export_records, + export_fields: doctype_field_map, + export_filters: filters + }); + } + + make_filter_area() { + this.filter_group = new frappe.ui.FilterGroup({ + parent: this.dialog.get_field('filter_area').$wrapper, + doctype: this.doctype, + on_change: e => { + this.update_record_count_message(); + } + }); + } + + make_select_all_buttons() { + let $select_all_buttons = $(` +
+
${__('Select fields to export')}
+ + + + +
+ `).on('click', '[data-action]', e => { + let $target = $(e.currentTarget); + let action = $target.data('action'); + let method = this[action]; + method ? this[action]() : null; + }); + this.dialog + .get_field('select_all_buttons') + .$wrapper.html($select_all_buttons); + } + + select_all() { + this.dialog.$wrapper + .find(':checkbox') + .prop('checked', true) + .trigger('change'); + } + + select_mandatory() { + let multicheck_fields = this.dialog.fields + .filter(df => df.fieldtype === 'MultiCheck') + .map(df => df.fieldname); + + let checkboxes = [].concat( + ...multicheck_fields.map(fieldname => { + let field = this.dialog.get_field(fieldname); + return field.options + .filter(option => option.danger) + .map(option => option.$checkbox.find('input').get(0)); + }) + ); + + this.unselect_all(); + $(checkboxes) + .prop('checked', true) + .trigger('change'); + } + + select_mandatory_without_children() { + let field = this.dialog.get_field(this.doctype); + let checkboxes = field.options + .filter(option => option.danger) + .map(option => option.$checkbox.find('input').get(0)); + + this.unselect_all(); + $(checkboxes) + .prop('checked', true) + .trigger('change'); + } + + unselect_all() { + this.dialog.$wrapper + .find(':checkbox') + .prop('checked', false) + .trigger('change'); + } + + update_record_count_message() { + let export_records = this.dialog.get_value('export_records'); + let count_method = { + last_10_records: () => Promise.resolve('10'), + all: () => frappe.db.count(this.doctype), + by_filter: () => + frappe.db.count(this.doctype, { + filters: this.get_filters() + }) + }; + + count_method[export_records]().then(value => { + let message = ''; + value = parseInt(value); + if (value === 0) { + message = __('No records will be exported'); + } else if (value === 1) { + message = __('1 record will be exported'); + } else { + message = __('{0} records will be exported', [value]); + } + this.dialog.set_df_property( + 'export_records', + 'description', + message + ); + + this.update_primary_action(value); + }); + } + + update_primary_action(no_of_records) { + let $primary_action = this.dialog.get_primary_btn(); + + if (no_of_records != null) { + $primary_action.prop('disabled', no_of_records === 0); + + let label = ''; + if (no_of_records === 0) { + label = __('Export'); + } else if (no_of_records === 1) { + label = __('Export 1 record'); + } else { + label = __('Export {0} records', [no_of_records]); + } + $primary_action.html(label); + } else { + let parent_fields = this.dialog.get_value(this.doctype); + $primary_action.prop('disabled', parent_fields.length === 0); + } + } + + get_filters() { + return this.filter_group.get_filters().reduce((acc, filter) => { + return Object.assign(acc, { + [filter[1]]: [filter[2], filter[3]] + }); + }, {}); + } +}; diff --git a/frappe/public/js/frappe/form/controls/multicheck.js b/frappe/public/js/frappe/form/controls/multicheck.js index 5e8c437445..98b38a5434 100644 --- a/frappe/public/js/frappe/form/controls/multicheck.js +++ b/frappe/public/js/frappe/form/controls/multicheck.js @@ -70,6 +70,7 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({ if (option.danger) { checkbox.find('.label-area').addClass('text-danger'); } + option.$checkbox = checkbox; }); if(this.df.select_all) { this.setup_select_all(); @@ -138,7 +139,7 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({ const column_size = this.get_column_size(); return $(`
-
- `).on('click', '[data-action]', e => { - let $target = $(e.currentTarget); - let action = $target.data('action'); - let method = this[action]; - method ? this[action]() : null; - }); + `); + frappe.utils.bind_actions_with_class($select_all_buttons, this); this.dialog .get_field('select_all_buttons') .$wrapper.html($select_all_buttons); @@ -239,11 +223,7 @@ frappe.data_import.DataExporter = class DataExporter { } else { message = __('{0} records will be exported', [value]); } - this.dialog.set_df_property( - 'export_records', - 'description', - message - ); + this.dialog.set_df_property('export_records', 'description', message); this.update_primary_action(value); }); @@ -277,4 +257,40 @@ frappe.data_import.DataExporter = class DataExporter { }); }, {}); } + + get_multicheck_options(doctype) { + if (!this.column_map) { + this.column_map = new ColumnPickerFields({ + doctype: this.doctype + }).get_columns_for_picker(); + } + + let autoname_field = null; + let meta = frappe.get_meta(doctype); + if (meta.autoname && meta.autoname.startsWith('field:')) { + let fieldname = meta.autoname.slice('field:'.length); + autoname_field = frappe.meta.get_field(doctype, fieldname); + } + + return this.column_map[doctype] + .filter(df => { + if (autoname_field && df.fieldname === autoname_field.fieldname) { + return false; + } + return true; + }) + .map(df => { + let label = __(df.label); + if (autoname_field && df.fieldname === 'name') { + label = label + ` (${__(autoname_field.label)})`; + } + return { + label, + value: df.fieldname, + danger: df.reqd, + checked: df.reqd, + description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}` + }; + }); + } }; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index eeca9deb3a..e963a20477 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -3,17 +3,69 @@ import DataTable from 'frappe-datatable'; frappe.provide('frappe.data_import'); frappe.data_import.ImportPreview = class ImportPreview { - constructor(wrapper, csv_array) { + constructor(wrapper, doctype, preview_data) { frappe.import_preview = this; this.wrapper = wrapper; - this.csv_array = csv_array; + this.doctype = doctype; + this.preview_fields = preview_data.fields; + this.preview_data = preview_data.data; + this.preview_warnings = preview_data.warnings; - this.prepare_csv_array(); + this.make_wrapper(); + this.prepare_columns(); + this.prepare_data(); + this.render_warnings(); this.render_datatable(); } - prepare_csv_array() { - this.csv_array = this.csv_array.map(row => { + make_wrapper() { + this.wrapper.html(` +
+
+
+
+ +
+
+ `); + frappe.utils.bind_actions_with_class(this.wrapper, this); + + this.$warnings = this.wrapper.find('.warnings'); + this.$table_preview = this.wrapper.find('.table-preview'); + } + + prepare_columns() { + this.columns = this.preview_fields.map(df => { + if (df.skip_import) { + return { + id: frappe.utils.get_random(6), + name: df.label, + skip_import: true, + editable: false, + focusable: false, + format: (value, row, column, data) => { + return `
${value}
`; + } + }; + } + + let column_title = df.label; + if (this.doctype !== df.parent) { + column_title = `${df.label} (${df.parent})`; + } + return { + id: df.fieldname, + name: column_title, + df: df, + editable: true + }; + }); + } + + prepare_data() { + this.preview_data = this.preview_data.map(row => { return row.map(cell => { if (cell == null) { return ''; @@ -23,19 +75,26 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } + render_warnings() { + let warning_html = this.preview_warnings + .map(warning => { + return `
${warning}
`; + }) + .join(''); + + let html = `
${warning_html}
`; + this.$warnings.html(html); + } + render_datatable() { - let columns = this.csv_array[0].map(col => { - return { - id: col, - name: col - }; - }); - let data = this.csv_array.slice(1); - this.datatable = new DataTable(this.wrapper.get(0), { - data, - columns, + this.datatable = new DataTable(this.$table_preview.get(0), { + data: this.preview_data, + columns: this.columns, layout: 'fixed', cellHeight: 35, + serialNoColumn: false, + checkboxColumn: true, + pasteFromClipboard: true, headerDropdown: [ { label: __('Change column mapping'), @@ -48,11 +107,13 @@ frappe.data_import.ImportPreview = class ImportPreview { ] }); - this.datatable.style.setStyle( - '.dt-dropdown__list-item:nth-child(-n+4)', - { - display: 'none' - } - ); + this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { + display: 'none' + }); + } + + add_row() { + this.preview_data.push([]); + this.datatable.refresh(this.preview_data); } }; diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 1486e819fc..a285264272 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -725,6 +725,16 @@ Object.assign(frappe.utils, { }, is_rtl() { return ["ar", "he", "fa"].includes(frappe.boot.lang); + }, + bind_actions_with_class($el, class_instance) { + $($el).on('click', '[data-action]', e => { + let $target = $(e.currentTarget); + let action = $target.data('action'); + let method = class_instance[action]; + method ? class_instance[action]() : null; + }); + + return $el; } }); From fcb4e13415f5fd129ff55c59703af55adb461be5 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 13 Aug 2019 19:42:36 +0530 Subject: [PATCH 05/83] fix: Autocomplete - Show value as description - Validate value from options - Override max_items --- .../js/frappe/form/controls/autocomplete.js | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/autocomplete.js b/frappe/public/js/frappe/form/controls/autocomplete.js index 6cbfa27fff..d4a21fc32c 100644 --- a/frappe/public/js/frappe/form/controls/autocomplete.js +++ b/frappe/public/js/frappe/form/controls/autocomplete.js @@ -14,7 +14,7 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ options = options.split('\n'); } if (typeof options[0] === 'string') { - options = options.map(o => ({label: o, value: o})); + options = options.map(o => ({ label: o, value: o })); } this._data = options; } @@ -24,12 +24,12 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ var me = this; return { minChars: 0, - maxItems: 99, + maxItems: this.df.max_items || 99, autoFirst: true, list: this.get_data(), data: function(item) { - if(!(item instanceof Object)) { - var d = {"value": item}; + if (!(item instanceof Object)) { + var d = { value: item }; item = d; } @@ -38,9 +38,13 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ value: item.value }; }, + filter: function(item, input) { + let hay = item.label + item.value; + return Awesomplete.FILTER_CONTAINS(hay, input); + }, item: function(item) { var d = this.get_item(item.value); - if(!d) { + if (!d) { d = item; } @@ -48,9 +52,9 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ d.label = d.value; } - var _label = (me.translate_values) ? __(d.label) : d.label; - var html = "" + _label + ""; - if(d.description && d.value!==d.description) { + var _label = me.translate_values ? __(d.label) : d.label; + var html = '' + _label + ''; + if (d.description) { html += '
' + __(d.description) + ''; } @@ -67,13 +71,21 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ }, setup_awesomplete() { - this.awesomplete = new Awesomplete(this.input, this.get_awesomplete_settings()); + this.awesomplete = new Awesomplete( + this.input, + this.get_awesomplete_settings() + ); - $(this.input_area).find('.awesomplete ul').css('min-width', '100%'); + $(this.input_area) + .find('.awesomplete ul') + .css('min-width', '100%'); - this.$input.on('input', frappe.utils.debounce(() => { - this.awesomplete.list = this.get_data(); - }, 500)); + this.$input.on( + 'input', + frappe.utils.debounce(() => { + this.awesomplete.list = this.get_data(); + }, 500) + ); this.$input.on('focus', () => { if (!this.$input.val()) { @@ -87,6 +99,15 @@ frappe.ui.form.ControlAutocomplete = frappe.ui.form.ControlData.extend({ }); }, + validate(value) { + let valid_values = this.awesomplete._list.map(d => d.value); + if (valid_values.includes(value)) { + return value; + } else { + return ''; + } + }, + get_data() { return this._data || []; }, From a16efb37b8644f7d6fbd70cfbc66661ed2a35f44 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 13 Aug 2019 20:09:52 +0530 Subject: [PATCH 06/83] feat: Remap columns to fields - Green colored are mapped - Red colored are skipped - Skip columns from import --- .../core/doctype/data_import/exporter_new.py | 2 +- .../core/doctype/data_import/importer_new.py | 35 +++-- .../data_import_beta/data_import_beta.js | 65 +++++++-- .../data_import_beta/data_import_beta.json | 15 +- .../data_import_beta/data_import_beta.py | 7 +- .../data_import/column_picker_fields.js | 31 ++++ .../js/frappe/data_import/data_exporter.js | 5 +- .../js/frappe/data_import/import_preview.js | 135 ++++++++++++++++-- 8 files changed, 256 insertions(+), 39 deletions(-) create mode 100644 frappe/public/js/frappe/data_import/column_picker_fields.js diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index b1ae89c0a4..0f12cef52d 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -154,7 +154,7 @@ class Exporter: if df.parent == self.doctype: return df.label else: - return '{0} / {1}'.format(df.parent, df.label) + return '{0} ({1})'.format(df.label, df.parent) header = [get_label(df) for df in self.fields] self.csv_array.append(header) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 46b84d85bf..22e09d5f6f 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -95,35 +95,50 @@ class Importer: self.header_row = header_row - def get_data_for_import_preview(self): - fields, fields_warnings = self.parse_fields_from_header_row() + def get_data_for_import_preview(self, import_options=None): + import_options = import_options or frappe._dict() + remap_columns = import_options.remap_column + skip_import = import_options.skip_import + + fields, fields_warnings = self.parse_fields_from_header_row(remap_columns, skip_import) formats, formats_warnings = self.parse_formats_from_first_10_rows() fields, data = self.add_serial_no_column(fields, self.data) warnings = fields_warnings + formats_warnings return dict( + header_row=self.header_row, fields=fields, data=data, warnings=warnings ) - def parse_fields_from_header_row(self): + def parse_fields_from_header_row(self, remap_columns, skip_import): + remap_columns = remap_columns or frappe._dict() + skip_import = skip_import or [] fields = [] warnings = [] df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() for i, value in enumerate(self.header_row): + if remap_columns.get(value): + column_name = value + value = remap_columns.get(value) + warnings.append(_('Column {0}: Mapping column {1} to field {2}').format( + i, frappe.bold(column_name), frappe.bold(value))) + field = df_by_labels_and_fieldnames.get(value) - if not field: + if not field or value in skip_import: field = { 'label': value, 'skip_import': True } - if value: + if value and value not in skip_import: warnings.append(_('Column {0}: Cannot match column {1} with any field').format(i, frappe.bold(value))) + elif value in skip_import: + warnings.append(_('Column {0}: Skipping column {1}').format(i, frappe.bold(value))) else: warnings.append(_('Column {0}: Skipping untitled column').format(i)) fields.append(field) @@ -140,7 +155,8 @@ class Importer: 'customer': df1, 'Due Date': df2, 'due_date': df2, - 'Sales Invoice Item / Item Code': df3 + 'Item Code (Sales Invoice Item)': df3, + 'Sales Invoice Item:item_code': df3, } """ out = { @@ -159,11 +175,14 @@ class Importer: for df in meta.fields: if df.fieldtype not in no_value_fields: # label as key - label = df.label if self.doctype == doctype else '{0} / {1}'.format(df.parent, df.label) + label = df.label if self.doctype == doctype else '{0} ({1})'.format(df.label, df.parent) out[label] = df # fieldname as key if self.doctype == doctype: out[df.fieldname] = df + else: + key = '{0}:{1}'.format(doctype, df.fieldname) + out[key] = df # if autoname is based on field # add an entry for "ID (Autoname Field)" @@ -197,7 +216,7 @@ class Importer: def add_serial_no_column(self, fields, data): fields_with_serial_no = [ { - 'label': _('Sr. No'), + 'label': 'Sr. No', 'skip_import': True } ] + fields diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 1277b88c55..4d50e7cc64 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -23,20 +23,65 @@ frappe.ui.form.on('Data Import Beta', { .html(__('Loading import file...')) .appendTo(frm.get_field('import_preview').$wrapper); - frm.call('get_preview_from_template').then(r => { - let preview_data = r.message; - - frappe.require('/assets/js/data_import_tools.min.js', () => { - new frappe.data_import.ImportPreview( - frm.get_field('import_preview').$wrapper, - frm.doc.reference_doctype, - preview_data - ); + frm + .call({ + doc: frm.doc, + method: 'get_preview_from_template', + freeze: true, + freeze_message: __('Preparing Preview...') + }) + .then(r => { + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); }); - }); } else { frm.get_field('import_preview').$wrapper.empty(); } frm.toggle_display('section_import_preview', frm.doc.import_file); + }, + + show_import_preview(frm, preview_data) { + frappe.require('/assets/js/data_import_tools.min.js', () => { + new frappe.data_import.ImportPreview({ + wrapper: frm.get_field('import_preview').$wrapper, + doctype: frm.doc.reference_doctype, + preview_data, + events: { + remap_column(column_name, fieldname) { + let import_json = JSON.parse(frm.doc.import_json || '{}'); + import_json.remap_column = import_json.remap_column || {}; + import_json.remap_column[column_name] = fieldname; + // if the column is remapped, remove it from skip_import + if ( + import_json.skip_import && + import_json.skip_import.includes(column_name) + ) { + import_json.skip_import = import_json.skip_import.filter( + d => d !== column_name + ); + } + frm.set_value('import_json', JSON.stringify(import_json)); + frm.trigger('import_file'); + }, + + skip_import(column_name) { + let import_json = JSON.parse(frm.doc.import_json || '{}'); + import_json.skip_import = import_json.skip_import || []; + if (!import_json.skip_import.includes(column_name)) { + import_json.skip_import.push(column_name); + } + // if column is being skipped, remove it from remap_column + if ( + import_json.remap_column && + import_json.remap_column[column_name] + ) { + delete import_json.remap_column[column_name]; + } + frm.set_value('import_json', JSON.stringify(import_json)); + frm.trigger('import_file'); + } + } + }); + }); } }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 8ae8315b33..542f0dbbbb 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -6,11 +6,12 @@ "engine": "InnoDB", "field_order": [ "reference_doctype", - "download_sample_file", "import_type", + "download_sample_file", "import_file", "section_import_preview", - "import_preview" + "import_preview", + "import_json" ], "fields": [ { @@ -51,10 +52,18 @@ "fieldname": "download_sample_file", "fieldtype": "Button", "label": "Download Sample File" + }, + { + "fieldname": "import_json", + "fieldtype": "Code", + "hidden": 1, + "label": "Import JSON", + "options": "JSON", + "read_only": 1 } ], "hide_toolbar": 1, - "modified": "2019-08-10 02:08:33.133016", + "modified": "2019-08-13 19:10:02.943686", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index d4a72c8ecb..8221dc3567 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -10,6 +10,10 @@ from frappe.core.doctype.data_import.exporter_new import Exporter class DataImportBeta(Document): + def validate(self): + if not self.import_file: + self.import_json = '' + def get_preview_from_template(self): if not self.import_file: return @@ -18,7 +22,8 @@ class DataImportBeta(Document): file_content = f.get_content() i = Importer(self.reference_doctype, content=file_content) - return i.get_data_for_import_preview() + import_options = frappe.parse_json(self.import_json or '{}') + return i.get_data_for_import_preview(import_options) @frappe.whitelist() diff --git a/frappe/public/js/frappe/data_import/column_picker_fields.js b/frappe/public/js/frappe/data_import/column_picker_fields.js new file mode 100644 index 0000000000..47aab004cd --- /dev/null +++ b/frappe/public/js/frappe/data_import/column_picker_fields.js @@ -0,0 +1,31 @@ +export default class ColumnPickerFields extends frappe.views.ReportView { + show() {} + + get_fields_as_options() { + let column_map = this.get_columns_for_picker(); + let doctypes = [this.doctype].concat( + ...frappe.meta + .get_table_fields(this.doctype) + .filter(df => !df.hidden) + .map(df => df.options) + ); + // flatten array + return [].concat( + ...doctypes.map(doctype => { + return column_map[doctype].map(df => { + let label = df.label; + let value = df.fieldname; + if (this.doctype !== doctype) { + label = `${df.label} (${doctype})`; + value = `${doctype}:${df.fieldname}`; + } + return { + label, + value, + description: value + }; + }); + }) + ); + } +} diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 2b3d893667..83e276bd1a 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -1,9 +1,6 @@ +import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); -class ColumnPickerFields extends frappe.views.ReportView { - show() {} -} - frappe.data_import.DataExporter = class DataExporter { constructor(doctype) { this.doctype = doctype; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index e963a20477..006d8d64cc 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -1,21 +1,27 @@ import DataTable from 'frappe-datatable'; +import ColumnManager from 'frappe-datatable/src/columnmanager'; +import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); frappe.data_import.ImportPreview = class ImportPreview { - constructor(wrapper, doctype, preview_data) { + constructor({ wrapper, doctype, preview_data, events = {} }) { frappe.import_preview = this; this.wrapper = wrapper; this.doctype = doctype; + this.header_row = preview_data.header_row; this.preview_fields = preview_data.fields; this.preview_data = preview_data.data; this.preview_warnings = preview_data.warnings; + this.events = events; - this.make_wrapper(); - this.prepare_columns(); - this.prepare_data(); - this.render_warnings(); - this.render_datatable(); + frappe.model.with_doctype(doctype, () => { + this.make_wrapper(); + this.prepare_columns(); + this.prepare_data(); + this.render_warnings(); + this.render_datatable(); + }); } make_wrapper() { @@ -55,11 +61,16 @@ frappe.data_import.ImportPreview = class ImportPreview { if (this.doctype !== df.parent) { column_title = `${df.label} (${df.parent})`; } + let meta = frappe.get_meta(this.doctype); + if (meta.autoname === `field:${df.fieldname}`) { + column_title = `ID (${df.label})`; + } return { id: df.fieldname, name: column_title, df: df, - editable: true + editable: true, + align: 'left' }; }); } @@ -87,6 +98,8 @@ frappe.data_import.ImportPreview = class ImportPreview { } render_datatable() { + let self = this; + this.datatable = new DataTable(this.$table_preview.get(0), { data: this.preview_data, columns: this.columns, @@ -97,23 +110,121 @@ frappe.data_import.ImportPreview = class ImportPreview { pasteFromClipboard: true, headerDropdown: [ { - label: __('Change column mapping'), - action: console.log + label: __('Remap Column'), + action: col => this.remap_column(col) }, { - label: __("Don't Import"), - action: console.log + label: __('Skip Import'), + action: col => this.skip_import(col) } - ] + ], + overrideComponents: { + ColumnManager: class CustomColumnManager extends ColumnManager { + getHeaderHTML(columns) { + let html = super.getHeaderHTML(columns); + + let header_row_columns = [ + { + id: '_checkbox', + colIndex: 0, + format: () => '' + }, + { + id: 'Sr. No', + colIndex: 1, + format: () => '' + } + ].concat( + ...self.header_row.map((col, i) => { + return { + id: col, + name: col, + align: 'left', + dropdown: false, + content: col, + colIndex: i + 2 + }; + }) + ); + + let header_row_html = this.rowmanager.getRowHTML( + header_row_columns, + { + rowIndex: 'header-row' + } + ); + return header_row_html + html; + } + } + } }); this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { display: 'none' }); + + let columns = this.datatable.getColumns(); + columns.forEach(col => { + if (!col.skip_import && col.df) { + this.datatable.style.setStyle( + `.dt-header .dt-cell--col-${col.colIndex}`, + { + backgroundColor: frappe.ui.color.get_color_shade( + 'green', + 'extra-light' + ), + color: frappe.ui.color.get_color_shade('green', 'dark') + } + ); + } + if (col.skip_import && col.name !== 'Sr. No') { + this.datatable.style.setStyle( + `.dt-header .dt-cell--col-${col.colIndex}`, + { + backgroundColor: frappe.ui.color.get_color_shade( + 'orange', + 'extra-light' + ), + color: frappe.ui.color.get_color_shade('orange', 'dark') + } + ); + this.datatable.style.setStyle(`.dt-cell--col-${col.colIndex}`, { + backgroundColor: frappe.ui.color.get_color_shade('white', 'light') + }); + } + }); } add_row() { this.preview_data.push([]); this.datatable.refresh(this.preview_data); } + + remap_column(col) { + let column_picker_fields = new ColumnPickerFields({ + doctype: this.doctype + }); + let dialog = new frappe.ui.Dialog({ + title: __('Remap Column: {0}', [col.name]), + fields: [ + { + fieldtype: 'Autocomplete', + fieldname: 'fieldname', + label: __('Select field'), + max_items: Infinity, + options: column_picker_fields.get_fields_as_options() + } + ], + primary_action: ({ fieldname }) => { + if (!fieldname) return; + this.events.remap_column(col.name, fieldname); + dialog.hide(); + } + }); + dialog.show(); + } + + skip_import(col) { + this.events.skip_import(col.name); + } }; From a205917dacd61d8a6e4e7c014d155fd7b7e48728 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Aug 2019 18:39:29 +0530 Subject: [PATCH 07/83] fix: Data Exporter - De duplicate child table rows - Add name column for child tables too - Show hidden tables in export --- .../core/doctype/data_import/exporter_new.py | 174 ++++++++++++------ .../data_import/column_picker_fields.js | 5 +- .../js/frappe/data_import/data_exporter.js | 6 +- .../js/frappe/views/reports/report_view.js | 6 +- 4 files changed, 125 insertions(+), 66 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index 0f12cef52d..df91543049 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -6,9 +6,18 @@ import frappe from frappe import _ from frappe.model import display_fieldtypes, no_value_fields, table_fields from frappe.utils.csvutils import build_csv_response +from .importer_new import INVALID_VALUES + class Exporter: - def __init__(self, doctype, export_fields=None, export_data=False, export_filters=None, file_type='CSV'): + def __init__( + self, + doctype, + export_fields=None, + export_data=False, + export_filters=None, + file_type="CSV", + ): """ Exports records of a DocType for use with Importer :param doctype: Document Type to export @@ -20,6 +29,7 @@ class Exporter: self.doctype = doctype self.meta = frappe.get_meta(doctype) self.export_fields = export_fields + self.export_filters = export_filters # this will contain the csv content self.csv_array = [] @@ -30,40 +40,32 @@ class Exporter: self.add_header() if export_data: - self.data = self.get_data_to_export(export_filters) + self.data = self.get_data_to_export() else: self.data = [] self.add_data() - def get_all_exportable_fields(self): return self.get_exportable_parent_fields() + self.get_exportable_children_fields() - def get_exportable_parent_fields(self): - name_field = frappe._dict({ - 'fieldtype': 'Data', - 'fieldname': 'name', - 'label': 'ID', - 'reqd': 1, - 'parent': self.doctype - }) - parent_fields = self.get_exportable_fields(self.doctype) # if autoname is based on field # then merge ID and the field column title as "ID (Autoname Field)" autoname = self.meta.autoname - if autoname and autoname.startswith('field:'): - fieldname = autoname[len('field:'):] + if autoname and autoname.startswith("field:"): + fieldname = autoname[len("field:") :] autoname_field = self.meta.get_field(fieldname) if autoname_field: - name_field.label = 'ID ({})'.format(autoname_field.label) + name_field = parent_fields[0] + name_field.label = "ID ({})".format(autoname_field.label) # remove the autoname field as it is a duplicate of ID field - parent_fields = [df for df in parent_fields if df.fieldname != autoname_field.fieldname] - - return [name_field] + parent_fields + parent_fields = [ + df for df in parent_fields if df.fieldname != autoname_field.fieldname + ] + return parent_fields def get_exportable_children_fields(self): children = [df.options for df in self.meta.fields if df.fieldtype in table_fields] @@ -73,8 +75,9 @@ class Exporter: return children_fields - def get_exportable_fields(self, doctype): + fields = [] + def is_exportable(df): return ( df.fieldtype not in display_fieldtypes @@ -87,83 +90,146 @@ class Exporter: # filter out layout fields fields = [df for df in meta.fields if is_exportable(df)] - if self.export_fields == 'All': - return fields + if self.export_fields == "Mandatory": + fields = [df for df in fields if df.reqd] - if self.export_fields == 'Mandatory': - return [df for df in fields if df.reqd] - - if isinstance(self.export_fields, dict): + elif isinstance(self.export_fields, dict): whitelist = self.export_fields.get(doctype, []) - return [df for df in fields if df.fieldname in whitelist] + fields = [df for df in fields if df.fieldname in whitelist] + else: + fields = [df for df in fields if df.reqd or df.in_list_view or df.bold] - return [df for df in fields if df.reqd or df.in_list_view or df.bold] + name_field = frappe._dict( + { + "fieldtype": "Data", + "fieldname": "name", + "label": "ID", + "reqd": 1, + "parent": doctype, + } + ) + if fields: + return [name_field] + fields + else: + return [] - def get_data_to_export(self, filters=None): + def get_data_to_export(self): frappe.permissions.can_export(self.doctype, raise_exception=True) def get_column_name(df): - return '`tab{0}`.`{1}`'.format(df.parent, df.fieldname) + return "`tab{0}`.`{1}`".format(df.parent, df.fieldname) - fieldnames = [get_column_name(df) for df in self.fields] + fields = [get_column_name(df) for df in self.fields] + filters = self.export_filters if self.meta.is_nested_set(): - order_by = '`tab{0}`.`lft` ASC'.format(self.doctype) + order_by = "`tab{0}`.`lft` ASC".format(self.doctype) else: - order_by = '`tab{0}`.`creation` DESC'.format(self.doctype) + order_by = "`tab{0}`.`creation` DESC".format(self.doctype) - data = frappe.db.get_list(self.doctype, + data = frappe.db.get_list( + self.doctype, filters=filters, - fields=fieldnames, + fields=fields, limit_page_length=None, order_by=order_by, as_list=1, ) - return self.remove_duplicate_parent_values(data) + data = self.remove_duplicate_values(data) + data = self.remove_row_gaps(data) + data = self.remove_empty_rows(data) - def remove_duplicate_parent_values(self, data): + return data + + def remove_duplicate_values(self, data): out = [] - parent_fields = self.get_exportable_parent_fields() - parent_fields_count = len(parent_fields) + doctypes = set([df.parent for df in self.fields]) + + def name_exists_in_column_before_row(name, column_index, row_index): + column_values = [row[column_index] for i, row in enumerate(data) if i < row_index] + return name in column_values + + for i, row in enumerate(data): + # first row is fine + if i == 0: + out.append(row) + continue - current_name = None - # first column is always name, can be used to find duplicate rows - for row in data: row = list(row) - name = row[0] + for doctype in doctypes: + name_index = self.get_name_column_index(doctype) + name = row[name_index] + column_indexes = self.get_column_indexes(doctype) - if name != current_name: - current_name = name - out.append(row) + if name_exists_in_column_before_row(name, name_index, i): + # remove the values from the row + row = [None if i in column_indexes else d for i, d in enumerate(row)] - elif name == current_name: - # remove the parent values from the row - row = row[parent_fields_count:] - # replace them with None values and add back the row contents - row = [None] * parent_fields_count + row - out.append(row) + out.append(row) return out + def remove_row_gaps(self, data): + doctypes = set([df.parent for df in self.fields if df.parent != self.doctype]) + + def get_nearest_empty_row_index(col_index, row_index): + col_values = [row[col_index] for row in data] + i = row_index - 1 + while not col_values[i]: + i = i - 1 + out = i + 1 + if row_index != out: + return out + + for i, row in enumerate(data): + # if this is the row that contains parent values then skip + if row[0]: + continue + + for doctype in doctypes: + name_index = self.get_name_column_index(doctype) + name = row[name_index] + column_indexes = self.get_column_indexes(doctype) + + if not name: + continue + + row_index = get_nearest_empty_row_index(name_index, i) + if row_index: + for col_index in column_indexes: + data[row_index][col_index] = row[col_index] + row[col_index] = None + + return data + + def remove_empty_rows(self, data): + return [row for row in data if any(v not in INVALID_VALUES for v in row)] + + def get_name_column_index(self, doctype): + for i, df in enumerate(self.fields): + if df.parent == doctype and df.fieldname == 'name': + return i + return -1 + + def get_column_indexes(self, doctype): + return [i for i, df in enumerate(self.fields) if df.parent == doctype] def add_header(self): def get_label(df): if df.parent == self.doctype: return df.label else: - return '{0} ({1})'.format(df.label, df.parent) + return "{0} ({1})".format(df.label, df.parent) header = [get_label(df) for df in self.fields] self.csv_array.append(header) - def add_data(self): self.csv_array += self.data - def get_csv_array(self): return self.csv_array diff --git a/frappe/public/js/frappe/data_import/column_picker_fields.js b/frappe/public/js/frappe/data_import/column_picker_fields.js index 47aab004cd..36cbf3c413 100644 --- a/frappe/public/js/frappe/data_import/column_picker_fields.js +++ b/frappe/public/js/frappe/data_import/column_picker_fields.js @@ -4,10 +4,7 @@ export default class ColumnPickerFields extends frappe.views.ReportView { get_fields_as_options() { let column_map = this.get_columns_for_picker(); let doctypes = [this.doctype].concat( - ...frappe.meta - .get_table_fields(this.doctype) - .filter(df => !df.hidden) - .map(df => df.options) + ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) ); // flatten array return [].concat( diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 83e276bd1a..e5c234abe4 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -3,6 +3,7 @@ frappe.provide('frappe.data_import'); frappe.data_import.DataExporter = class DataExporter { constructor(doctype) { + frappe.data_exporter = this; this.doctype = doctype; frappe.model.with_doctype(doctype, () => { this.make_dialog(); @@ -11,10 +12,7 @@ frappe.data_import.DataExporter = class DataExporter { make_dialog() { let doctypes = [this.doctype].concat( - ...frappe.meta - .get_table_fields(this.doctype) - .filter(df => !df.hidden) - .map(df => df.options) + ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) ); this.dialog = new frappe.ui.Dialog({ diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 0a2a1d6072..6503f1c7ac 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -772,8 +772,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { get_columns_for_picker() { let out = {}; - const standard_fields_filter = df => - !in_list(frappe.model.no_value_type, df.fieldtype) && !df.report_hide; + const standard_fields_filter = df => !in_list(frappe.model.no_value_type, df.fieldtype); let doctype_fields = frappe.meta.get_docfields(this.doctype).filter(standard_fields_filter); @@ -786,8 +785,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { out[this.doctype] = doctype_fields; - const table_fields = frappe.meta.get_table_fields(this.doctype) - .filter(df => !df.hidden); + const table_fields = frappe.meta.get_table_fields(this.doctype); table_fields.forEach(df => { const cdt = df.options; From e60e1189625b7bd5d30cce22b0d800a203790274 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Aug 2019 18:48:39 +0530 Subject: [PATCH 08/83] fix: Data Importer - Parse child table values and prepare a doc - Show Template header in datatable - Template Options: for column mapping, skipped columns and edited rows - Import Log: Keep track of inserted/updated documents --- .../core/doctype/data_import/importer_new.py | 366 +++++++++++------- .../data_import_beta/data_import_beta.js | 101 +++-- .../data_import_beta/data_import_beta.json | 56 ++- .../data_import_beta/data_import_beta.py | 14 +- .../data_import/custom_column_manager.js | 38 ++ .../js/frappe/data_import/import_preview.js | 73 ++-- 6 files changed, 431 insertions(+), 217 deletions(-) create mode 100644 frappe/public/js/frappe/data_import/custom_column_manager.js diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 22e09d5f6f..03fe46d357 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -4,6 +4,7 @@ import io import csv +import json import frappe from datetime import datetime from frappe import _ @@ -16,47 +17,61 @@ from frappe.model import display_fieldtypes, no_value_fields, table_fields # set flags: frappe.flags.in_import = True # during import - # check empty row - # validate naming +# check empty row +# validate naming + +INVALID_VALUES = ["", None] -INVALID_VALUES = ['', None] class Importer: - - def __init__(self, doctype, file_path=None, content=None, options=None): + def __init__(self, doctype, data_import=None, file_path=None, content=None): self.doctype = doctype + self.template_options = frappe._dict( + {"remap_column": {}, "skip_import": [], "edited_rows": []} + ) + + if data_import: + self.data_import = data_import + if self.data_import.template_options: + template_options = frappe.parse_json(self.data_import.template_options) + self.template_options.update(template_options) + else: + self.data_import = None + self.header_row = None self.data = None - self.skipped_rows = [] self._guessed_date_formats = {} self.meta = frappe.get_meta(doctype) + self.prepare_content(file_path, content) + + def prepare_content(self, file_path, content): + if self.data_import: + file_doc = frappe.get_doc("File", {"file_url": self.data_import.import_file}) + content = file_doc.get_content() if file_path: self.read_file(file_path) elif content: self.read_content(content) - self.remove_empty_rows_and_columns() def read_file(self, file_path): - extn = file_path.split('.')[1] + extn = file_path.split(".")[1] file_content = None - with io.open(file_path, mode='rb') as f: + with io.open(file_path, mode="rb") as f: file_content = f.read() - if extn == 'csv': + if extn == "csv": data = read_csv_content(file_content) self.header_row = data[0] self.data = data[1:] - def read_content(self, content): data = read_csv_content(content) self.header_row = data[0] self.data = data[1:] - def remove_empty_rows_and_columns(self): self.row_index_map = [] removed_rows = [] @@ -94,58 +109,54 @@ class Importer: self.data = data_without_empty_columns self.header_row = header_row - - def get_data_for_import_preview(self, import_options=None): - import_options = import_options or frappe._dict() - remap_columns = import_options.remap_column - skip_import = import_options.skip_import - - fields, fields_warnings = self.parse_fields_from_header_row(remap_columns, skip_import) + def get_data_for_import_preview(self): + fields, fields_warnings = self.parse_fields_from_header_row() formats, formats_warnings = self.parse_formats_from_first_10_rows() fields, data = self.add_serial_no_column(fields, self.data) + if self.template_options.edited_rows: + data = self.template_options.edited_rows + warnings = fields_warnings + formats_warnings - return dict( - header_row=self.header_row, - fields=fields, - data=data, - warnings=warnings - ) + return dict(header_row=self.header_row, fields=fields, data=data, warnings=warnings) - - def parse_fields_from_header_row(self, remap_columns, skip_import): - remap_columns = remap_columns or frappe._dict() - skip_import = skip_import or [] + def parse_fields_from_header_row(self): + remap_column = self.template_options.remap_column + skip_import = self.template_options.skip_import fields = [] warnings = [] df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() for i, value in enumerate(self.header_row): - if remap_columns.get(value): + header_row_index = str(i) + if remap_column.get(header_row_index): column_name = value - value = remap_columns.get(value) - warnings.append(_('Column {0}: Mapping column {1} to field {2}').format( - i, frappe.bold(column_name), frappe.bold(value))) + value = remap_column.get(header_row_index) + warnings.append( + _("Column {0}: Mapping column {1} to field {2}").format( + i, frappe.bold(column_name), frappe.bold(value) + ) + ) field = df_by_labels_and_fieldnames.get(value) - if not field or value in skip_import: - field = { - 'label': value, - 'skip_import': True - } - if value and value not in skip_import: - warnings.append(_('Column {0}: Cannot match column {1} with any field').format(i, frappe.bold(value))) - elif value in skip_import: - warnings.append(_('Column {0}: Skipping column {1}').format(i, frappe.bold(value))) + if not field or i in skip_import: + field = frappe._dict({"label": value, "skip_import": True}) + if value and i not in skip_import: + warnings.append( + _("Column {0}: Cannot match column {1} with any field").format( + i, frappe.bold(value) + ) + ) + elif i in skip_import: + warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(value))) else: - warnings.append(_('Column {0}: Skipping untitled column').format(i)) + warnings.append(_("Column {0}: Skipping untitled column").format(i)) fields.append(field) return fields, warnings - def build_fields_dict_for_column_matching(self): """ Build a dict with various keys to match with column headers and value as docfield @@ -159,45 +170,49 @@ class Importer: 'Sales Invoice Item:item_code': df3, } """ - out = { - 'ID': frappe._dict({ - 'fieldtype': 'Data', - 'fieldname': 'name', - 'label': 'ID', - 'reqd': 1, - 'parent': self.doctype - }) - } + out = {} - doctypes = [self.doctype] + [df.options for df in self.meta.get_table_fields()] + table_doctypes = [df.options for df in self.meta.get_table_fields()] + doctypes = [self.doctype] + table_doctypes for doctype in doctypes: + # name field + name_key = "ID" if self.doctype == doctype else "ID ({})".format(doctype) + out[name_key] = frappe._dict( + { + "fieldtype": "Data", + "fieldname": "name", + "label": "ID", + "reqd": self.data_import.import_type == "Update Existing Records", + "parent": doctype, + } + ) + # other fields meta = frappe.get_meta(doctype) for df in meta.fields: if df.fieldtype not in no_value_fields: # label as key - label = df.label if self.doctype == doctype else '{0} ({1})'.format(df.label, df.parent) + label = df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) out[label] = df # fieldname as key if self.doctype == doctype: out[df.fieldname] = df else: - key = '{0}:{1}'.format(doctype, df.fieldname) + key = "{0}:{1}".format(doctype, df.fieldname) out[key] = df # if autoname is based on field # add an entry for "ID (Autoname Field)" autoname = self.meta.autoname - if autoname and autoname.startswith('field:'): - fieldname = autoname[len('field:'):] + if autoname and autoname.startswith("field:"): + fieldname = autoname[len("field:") :] autoname_field = self.meta.get_field(fieldname) if autoname_field: - out['ID ({})'.format(autoname_field.label)] = autoname_field + out["ID ({})".format(autoname_field.label)] = autoname_field # ID field should also map to the autoname field - out['ID'] = autoname_field + out["ID"] = autoname_field return out - def parse_formats_from_first_10_rows(self): """ Returns a list of column descriptors for columns that might need parsing. @@ -212,13 +227,9 @@ class Importer: formats = [] return formats, [] - def add_serial_no_column(self, fields, data): fields_with_serial_no = [ - { - 'label': 'Sr. No', - 'skip_import': True - } + frappe._dict({"label": "Sr. No", "skip_import": True, "parent": None}) ] + fields data_with_serial_no = [] @@ -227,55 +238,29 @@ class Importer: return fields_with_serial_no, data_with_serial_no + def parse_value(self, value, df): + # convert boolean values to 0 or 1 + if df.fieldtype == "Check" and value.lower().strip() in ["t", "f", "true", "false"]: + value = value.lower().strip() + value = 1 if value in ["t", "true"] else 0 - def parse_data_for_import(self, row, index): - - if all(v in INVALID_VALUES for v in row): - # empty row - self.skipped_rows.append([index, 'Empty Row']) - return - - doc = {} - - for i, field in enumerate(self.header_row): - if not self.meta.has_field(field): - continue - - df = self.meta.get_field(field) - value = row[i] - - if value in INVALID_VALUES: - if df.reqd: - raise MandatoryError(_('Row {0}: {1} is a mandatory field').format(i, frappe.bold(df.label))) - else: - value = None - - # convert boolean values to 0 or 1 - if df.fieldtype == 'Check' and value.lower().strip() in ['t', 'f', 'true', 'false']: - value = value.lower().strip() - value = 1 if value in ['t', 'true'] else 0 - - if df.fieldtype in ['Int', 'Check']: - value = cint(value) - elif df.fieldtype in ['Float', 'Percent', 'Currency']: - value = flt(value) - elif df.fieldtype in ['Date', 'Datetime']: - value = self.parse_date_format(value, df) - - doc[df.fieldname] = value - - return frappe._dict(doc) + if df.fieldtype in ["Int", "Check"]: + value = cint(value) + elif df.fieldtype in ["Float", "Percent", "Currency"]: + value = flt(value) + elif df.fieldtype in ["Date", "Datetime"]: + value = self.parse_date_format(value, df) + return value def parse_date_format(self, value, df): date_format = self.guess_date_format_for_column(df.fieldname) return datetime.strptime(value, date_format) - def guess_date_format_for_column(self, fieldname): - ''' Guesses date format for a column by parsing the first 10 values in the column, + """ Guesses date format for a column by parsing the first 10 values in the column, getting the date format and then returning the one which has the maximum frequency - ''' + """ PARSE_ROW_COUNT = 10 if not self._guessed_date_formats.get(fieldname): @@ -298,53 +283,164 @@ class Importer: return self._guessed_date_formats[fieldname] - def import_data(self): - print('Importing {0} rows...'.format(len(self.data))) + out = self.get_data_for_import_preview() + fields = out["fields"] + data = out["data"] + import_log = [] - for i, row in enumerate(self.data): - doc = self.parse_data_for_import(row, i) + print("Importing {0} rows...".format(len(data))) + docs, warnings = self.get_docs_for_import(fields, data) - if doc: + if warnings: + return warnings + + for doc in docs: + doc = self.process_doc(doc) + import_log.append({"inserted": True, "name": doc.name}) + + self.data_import.db_set("import_log", json.dumps(import_log)) + + def get_docs_for_import(self, fields, data): + docs = [] + parse_warnings = [] + while data: + doc, data, warnings = self.parse_next_row_for_import(fields, data) + if not warnings: + docs.append(doc) + else: + parse_warnings += warnings + return docs, parse_warnings + + def parse_next_row_for_import(self, fields, data): + doc = {} + warnings = [] + doctypes = set([df.parent for df in fields if df.parent]) + + # get all rows to make this doc + # first row is included by default + rows = [data[0]] + + # if there are child doctypes, find the subsequent rows + if len(doctypes) > 1: + # subsequent rows dont have any parent value set + # so we use it to check if any value is set + # if not we include that row + parent_column_index = self.get_first_parent_column_index(fields) + for d in data[1:]: + value = d[parent_column_index] + # if cell value exists then it is the next doc + if value: + break + rows.append(d) + + def get_column_indexes(doctype): + return [i for i, df in enumerate(fields) if df.parent == doctype] + + def parse_doc(doctype, docfields, values, row_index): + doc = {} + for index, (df, value) in enumerate(zip(docfields, values)): + if df.get("skip_import", False): + continue + + if value in INVALID_VALUES: + if df.reqd: + warnings.append( + _("Row {0}: {1} is a mandatory field").format(row_index, frappe.bold(df.label)) + ) + continue + else: + value = None + + doc[df.fieldname] = value = self.parse_value(value, df) + return doc + + parsed_docs = {} + for row_index, row in enumerate(rows): + for doctype in doctypes: + column_indexes = get_column_indexes(doctype) + values = [row[i] for i in column_indexes] + + if all(v in INVALID_VALUES for v in values): + # skip values if all of them are empty + continue + + docfields = [fields[i] for i in column_indexes] + doc = parse_doc(doctype, docfields, values, row_index) + parsed_docs[doctype] = parsed_docs.get(doctype, []) + parsed_docs[doctype].append(doc) + + for doctype, docs in parsed_docs.items(): + if doctype == self.doctype: + doc = docs[0] + else: + table_dfs = self.meta.get( + "fields", {"options": doctype, "fieldtype": ["in", table_fields]} + ) + if table_dfs: + table_field = table_dfs[0] + doc[table_field.fieldname] = docs + + return doc, data[len(rows) :], warnings + + def get_first_parent_column_index(self, fields): + """ + Returns the first column's index which must be one of the parent columns + """ + # find a parent column + parent_column_index = -1 + for i, df in enumerate(fields): + if not df.get("skip_import", False) and df.parent == self.doctype: + parent_column_index = i break + return parent_column_index + def process_doc(self, doc): + import_type = self.data_import.import_type + + if import_type == "Insert New Records": + return self.insert_record(doc) + elif import_type == "Update Existing Records": + pass + + def insert_record(self, doc): + doc.update({"doctype": self.doctype}) + new_doc = frappe.get_doc(doc) + return new_doc.insert() DATE_FORMATS = [ - r'%Y-%m-%d', - r'%d-%m-%Y', - r'%m-%d-%Y', - - r'%Y/%m/%d', - r'%d/%m/%Y', - r'%m/%d/%Y', - - r'%m/%d/%y', - r'%d/%m/%y', - - r'%Y.%m.%d', - r'%d.%m.%Y', - r'%m.%d.%Y', + r"%Y-%m-%d", + r"%d-%m-%Y", + r"%m-%d-%Y", + r"%Y/%m/%d", + r"%d/%m/%Y", + r"%m/%d/%Y", + r"%m/%d/%y", + r"%d/%m/%y", + r"%Y.%m.%d", + r"%d.%m.%Y", + r"%m.%d.%Y", ] TIME_FORMATS = [ - r'%H:%M:%S.%f', - r'%H:%M:%S', - r'%H:%M', - - r'%I:%M:%S.%f %p', - r'%I:%M:%S %p', - r'%I:%M %p', + r"%H:%M:%S.%f", + r"%H:%M:%S", + r"%H:%M", + r"%I:%M:%S.%f %p", + r"%I:%M:%S %p", + r"%I:%M %p", ] + def guess_date_format(date_string): date_string = date_string.strip() _date = None _time = None - if ' ' in date_string: - _date, _time = date_string.split(' ', 1) + if " " in date_string: + _date, _time = date_string.split(" ", 1) else: _date = date_string @@ -370,7 +466,7 @@ def guess_date_format(date_string): full_format = date_format if time_format: - full_format += ' ' + time_format + full_format += " " + time_format return full_format diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 4d50e7cc64..e552315374 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -3,12 +3,37 @@ frappe.ui.form.on('Data Import Beta', { refresh(frm) { + frm.page.hide_icon_group(); frm.trigger('import_file'); frm.trigger('reference_doctype'); + frm.trigger('show_import_log'); + if (!frm.is_new()) { + frm.page.set_primary_action(__('Start Import'), () => + frm.events.start_import(frm) + ); + } else { + frm.page.set_primary_action(__('Save'), () => frm.save()); + } }, - reference_doctype(frm) { - // + start_import(frm) { + let csv_array = frm.import_preview.get_rows_as_csv_array(); + let template_options = JSON.parse(frm.doc.template_options || '{}'); + template_options.edited_rows = csv_array; + frm.set_value('template_options', JSON.stringify(template_options)); + + frm.save().then(() => { + frm.trigger('import_file').then(() => + frm.call('start_import').then(r => { + let warnings = r.message || []; + if (warnings.length) { + frm.import_preview.render_warnings(warnings); + } else { + // + } + }) + ); + }); }, download_sample_file(frm) { @@ -42,46 +67,76 @@ frappe.ui.form.on('Data Import Beta', { show_import_preview(frm, preview_data) { frappe.require('/assets/js/data_import_tools.min.js', () => { - new frappe.data_import.ImportPreview({ + frm.import_preview = new frappe.data_import.ImportPreview({ wrapper: frm.get_field('import_preview').$wrapper, doctype: frm.doc.reference_doctype, preview_data, events: { - remap_column(column_name, fieldname) { - let import_json = JSON.parse(frm.doc.import_json || '{}'); - import_json.remap_column = import_json.remap_column || {}; - import_json.remap_column[column_name] = fieldname; + remap_column(header_row_index, fieldname) { + let template_options = JSON.parse(frm.doc.template_options || '{}'); + template_options.remap_column = template_options.remap_column || {}; + template_options.remap_column[header_row_index] = fieldname; // if the column is remapped, remove it from skip_import if ( - import_json.skip_import && - import_json.skip_import.includes(column_name) + template_options.skip_import && + template_options.skip_import.includes(header_row_index) ) { - import_json.skip_import = import_json.skip_import.filter( - d => d !== column_name + template_options.skip_import = template_options.skip_import.filter( + d => d !== header_row_index ); } - frm.set_value('import_json', JSON.stringify(import_json)); - frm.trigger('import_file'); + frm.set_value('template_options', JSON.stringify(template_options)); + frm.save().then(() => { + frm.trigger('import_file'); + }); }, - skip_import(column_name) { - let import_json = JSON.parse(frm.doc.import_json || '{}'); - import_json.skip_import = import_json.skip_import || []; - if (!import_json.skip_import.includes(column_name)) { - import_json.skip_import.push(column_name); + skip_import(header_row_index) { + let template_options = JSON.parse(frm.doc.template_options || '{}'); + template_options.skip_import = template_options.skip_import || []; + if (!template_options.skip_import.includes(header_row_index)) { + template_options.skip_import.push(header_row_index); } // if column is being skipped, remove it from remap_column if ( - import_json.remap_column && - import_json.remap_column[column_name] + template_options.remap_column && + template_options.remap_column[header_row_index] ) { - delete import_json.remap_column[column_name]; + delete template_options.remap_column[header_row_index]; } - frm.set_value('import_json', JSON.stringify(import_json)); - frm.trigger('import_file'); + frm.set_value('template_options', JSON.stringify(template_options)); + frm.save().then(() => { + frm.trigger('import_file'); + }); } } }); }); + }, + + show_import_log(frm) { + frm.toggle_display('import_log', false); + if (!frm.doc.import_log) { + frm.get_field('import_log_preview').$wrapper.empty(); + return; + } + let import_log = JSON.parse(frm.doc.import_log); + let rows = import_log + .map(log => { + return ` + ${log.name} + ${log.inserted ? 'Inserted' : ''} + `; + }) + .join(''); + frm.get_field('import_log_preview').$wrapper.html(` + + + + + + ${rows} +
${__('Document Name')}${__('Status')}
+ `); } }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 542f0dbbbb..e413a9c455 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -1,4 +1,5 @@ { + "autoname": "format:{reference_doctype} Import on {creation}", "beta": 1, "creation": "2019-08-04 14:16:08.318714", "doctype": "DocType", @@ -9,9 +10,16 @@ "import_type", "download_sample_file", "import_file", + "column_break_5", + "skip_rows_with_errors", + "section_break_7", + "date_format", + "template_options", "section_import_preview", "import_preview", - "import_json" + "import_log_section", + "import_log", + "import_log_preview" ], "fields": [ { @@ -27,7 +35,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Import Type", - "options": "Insert new records\nUpdate existing records", + "options": "Insert New Records\nUpdate Existing Records", "reqd": 1 }, { @@ -54,16 +62,54 @@ "label": "Download Sample File" }, { - "fieldname": "import_json", + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "skip_rows_with_errors", + "fieldtype": "Check", + "label": "Skip rows with errors" + }, + { + "collapsible": 1, + "fieldname": "section_break_7", + "fieldtype": "Section Break", + "label": "Import Options" + }, + { + "fieldname": "date_format", + "fieldtype": "Select", + "label": "Date Format", + "options": "\nYYYY-MM-DD\nDD-MM-YYYY\nMM-DD-YYYY\nYYYY/MM/DD\nDD/MM/YYYY\nMM/DD/YYYY\nMM/DD/YY\nDD/MM/YY\nYYYY.MM.DD\nDD.MM.YYYY\nMM.DD.YYYY" + }, + { + "fieldname": "template_options", "fieldtype": "Code", "hidden": 1, - "label": "Import JSON", + "label": "Template Options", "options": "JSON", "read_only": 1 + }, + { + "fieldname": "import_log", + "fieldtype": "Code", + "label": "Import Log", + "options": "JSON" + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" } ], "hide_toolbar": 1, - "modified": "2019-08-13 19:10:02.943686", + "modified": "2019-08-22 18:26:28.939065", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 8221dc3567..213b13ba11 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -9,7 +9,6 @@ from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter class DataImportBeta(Document): - def validate(self): if not self.import_file: self.import_json = '' @@ -18,12 +17,15 @@ class DataImportBeta(Document): if not self.import_file: return - f = frappe.get_doc('File', { 'file_url': self.import_file }) - file_content = f.get_content() + i = self.get_importer() + return i.get_data_for_import_preview() - i = Importer(self.reference_doctype, content=file_content) - import_options = frappe.parse_json(self.import_json or '{}') - return i.get_data_for_import_preview(import_options) + def start_import(self): + i = self.get_importer() + return i.import_data() + + def get_importer(self): + return Importer(self.reference_doctype, data_import=self) @frappe.whitelist() diff --git a/frappe/public/js/frappe/data_import/custom_column_manager.js b/frappe/public/js/frappe/data_import/custom_column_manager.js new file mode 100644 index 0000000000..2e829d349c --- /dev/null +++ b/frappe/public/js/frappe/data_import/custom_column_manager.js @@ -0,0 +1,38 @@ +import ColumnManager from 'frappe-datatable/src/columnmanager'; + +export default function(header_row) { + return class CustomColumnManager extends ColumnManager { + getHeaderHTML(columns) { + let html = super.getHeaderHTML(columns); + + let header_row_columns = [ + { + id: '_checkbox', + colIndex: 0, + format: () => '' + } + // { + // id: 'Sr. No', + // colIndex: 1, + // format: () => '' + // } + ].concat( + ...header_row.map((col, i) => { + return { + id: col, + name: col, + align: 'left', + dropdown: false, + content: col, + colIndex: i + 1 + }; + }) + ); + + let header_row_html = this.rowmanager.getRowHTML(header_row_columns, { + rowIndex: 'header-row' + }); + return header_row_html + html; + } + }; +} diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 006d8d64cc..f580293af9 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -1,5 +1,5 @@ import DataTable from 'frappe-datatable'; -import ColumnManager from 'frappe-datatable/src/columnmanager'; +import get_custom_column_manager from './custom_column_manager'; import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); @@ -19,7 +19,7 @@ frappe.data_import.ImportPreview = class ImportPreview { this.make_wrapper(); this.prepare_columns(); this.prepare_data(); - this.render_warnings(); + this.render_warnings(this.preview_warnings); this.render_datatable(); }); } @@ -43,7 +43,8 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { - this.columns = this.preview_fields.map(df => { + this.columns = this.preview_fields.map((df, i) => { + let header_row_index = i - 1; if (df.skip_import) { return { id: frappe.utils.get_random(6), @@ -51,6 +52,7 @@ frappe.data_import.ImportPreview = class ImportPreview { skip_import: true, editable: false, focusable: false, + header_row_index, format: (value, row, column, data) => { return `
${value}
`; } @@ -70,7 +72,8 @@ frappe.data_import.ImportPreview = class ImportPreview { name: column_title, df: df, editable: true, - align: 'left' + align: 'left', + header_row_index }; }); } @@ -86,8 +89,8 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - render_warnings() { - let warning_html = this.preview_warnings + render_warnings(warnings) { + let warning_html = warnings .map(warning => { return `
${warning}
`; }) @@ -106,7 +109,7 @@ frappe.data_import.ImportPreview = class ImportPreview { layout: 'fixed', cellHeight: 35, serialNoColumn: false, - checkboxColumn: true, + checkboxColumn: false, pasteFromClipboard: true, headerDropdown: [ { @@ -119,43 +122,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } ], overrideComponents: { - ColumnManager: class CustomColumnManager extends ColumnManager { - getHeaderHTML(columns) { - let html = super.getHeaderHTML(columns); - - let header_row_columns = [ - { - id: '_checkbox', - colIndex: 0, - format: () => '' - }, - { - id: 'Sr. No', - colIndex: 1, - format: () => '' - } - ].concat( - ...self.header_row.map((col, i) => { - return { - id: col, - name: col, - align: 'left', - dropdown: false, - content: col, - colIndex: i + 2 - }; - }) - ); - - let header_row_html = this.rowmanager.getRowHTML( - header_row_columns, - { - rowIndex: 'header-row' - } - ); - return header_row_html + html; - } - } + ColumnManager: get_custom_column_manager(this.header_row) } }); @@ -163,11 +130,21 @@ frappe.data_import.ImportPreview = class ImportPreview { display: 'none' }); + this.add_color_to_column_header(); + } + + get_rows_as_csv_array() { + return this.datatable.getRows().map(row => { + return row.map(cell => cell.content); + }); + } + + add_color_to_column_header() { let columns = this.datatable.getColumns(); columns.forEach(col => { if (!col.skip_import && col.df) { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'green', @@ -179,7 +156,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } if (col.skip_import && col.name !== 'Sr. No') { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'orange', @@ -217,7 +194,7 @@ frappe.data_import.ImportPreview = class ImportPreview { ], primary_action: ({ fieldname }) => { if (!fieldname) return; - this.events.remap_column(col.name, fieldname); + this.events.remap_column(col.header_row_index, fieldname); dialog.hide(); } }); @@ -225,6 +202,6 @@ frappe.data_import.ImportPreview = class ImportPreview { } skip_import(col) { - this.events.skip_import(col.name); + this.events.skip_import(col.header_row_index); } }; From 5a8bfc9d104fd95d990fd142ae6f697045fdfe8f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 25 Aug 2019 22:11:18 +0530 Subject: [PATCH 09/83] fix: Method to remove name column values --- frappe/core/doctype/data_import/exporter_new.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index df91543049..aad4ecc3f2 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -140,6 +140,7 @@ class Exporter: data = self.remove_duplicate_values(data) data = self.remove_row_gaps(data) data = self.remove_empty_rows(data) + # data = self.remove_values_from_name_column(data) return data @@ -208,9 +209,16 @@ class Exporter: def remove_empty_rows(self, data): return [row for row in data if any(v not in INVALID_VALUES for v in row)] + def remove_values_from_name_column(self, data): + out = [] + name_columns = [i for i, df in enumerate(self.fields) if df.fieldname == "name"] + for row in data: + out.append(["" if i in name_columns else value for i, value in enumerate(row)]) + return out + def get_name_column_index(self, doctype): for i, df in enumerate(self.fields): - if df.parent == doctype and df.fieldname == 'name': + if df.parent == doctype and df.fieldname == "name": return i return -1 From 9efbeb924ef209bb443f7cbb2db1d59eb3b6e078 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 25 Aug 2019 22:11:33 +0530 Subject: [PATCH 10/83] fix: Allow to export 0 records --- frappe/public/js/frappe/data_import/data_exporter.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index e5c234abe4..0532d5b002 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -27,10 +27,6 @@ frappe.data_import.DataExporter = class DataExporter { label: __('Export All Records'), value: 'all' }, - // { - // label: __('Export 10 Records'), - // value: 'last_10_records' - // }, { label: __('Export Filtered Records'), value: 'by_filter' @@ -200,7 +196,6 @@ frappe.data_import.DataExporter = class DataExporter { update_record_count_message() { let export_records = this.dialog.get_value('export_records'); let count_method = { - last_10_records: () => Promise.resolve('10'), all: () => frappe.db.count(this.doctype), by_filter: () => frappe.db.count(this.doctype, { @@ -228,8 +223,6 @@ frappe.data_import.DataExporter = class DataExporter { let $primary_action = this.dialog.get_primary_btn(); if (no_of_records != null) { - $primary_action.prop('disabled', no_of_records === 0); - let label = ''; if (no_of_records === 0) { label = __('Export'); From 0d6f256e03dfbd0e79cf9bae76988d4d2a03544a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 25 Aug 2019 22:15:26 +0530 Subject: [PATCH 11/83] feat: Find missing link field values - Prompt user to create them --- .../core/doctype/data_import/importer_new.py | 30 ++++++++++++++-- .../data_import_beta/data_import_beta.js | 34 ++++++++++++++++--- .../data_import_beta/data_import_beta.py | 15 ++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 03fe46d357..e6305e2437 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -289,8 +289,10 @@ class Importer: data = out["data"] import_log = [] - print("Importing {0} rows...".format(len(data))) - docs, warnings = self.get_docs_for_import(fields, data) + # validate link field values + missing_link_values = self.get_missing_link_field_values(fields, data) + if missing_link_values: + return {"missing_link_values": missing_link_values} if warnings: return warnings @@ -408,6 +410,30 @@ class Importer: new_doc = frappe.get_doc(doc) return new_doc.insert() + def get_missing_link_field_values(self, fields, data): + link_column_indexes = [i for i, df in enumerate(fields) if df.fieldtype == "Link"] + + def has_one_mandatory_field(doctype): + meta = frappe.get_meta(doctype) + mandatory_fields = [df for df in meta.fields if df.reqd] + mandatory_fields_count = len(mandatory_fields) + if meta.autoname.lower() == "prompt": + mandatory_fields_count += 1 + return mandatory_fields_count == 1 + + missing_values_map = {} + for index in link_column_indexes: + df = fields[index] + column_values = [row[index] for row in data] + values = set([v for v in column_values if v not in INVALID_VALUES]) + doctype = df.options + if has_one_mandatory_field(doctype): + missing_values = [value for value in values if not frappe.db.exists(doctype, value)] + if missing_values: + missing_values_map[doctype] = missing_values + + return missing_values_map + DATE_FORMATS = [ r"%Y-%m-%d", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index e552315374..56b90deeeb 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -25,11 +25,11 @@ frappe.ui.form.on('Data Import Beta', { frm.save().then(() => { frm.trigger('import_file').then(() => frm.call('start_import').then(r => { - let warnings = r.message || []; - if (warnings.length) { + let { warnings, missing_link_values } = r.message || {}; + if (warnings) { frm.import_preview.render_warnings(warnings); - } else { - // + } else if (missing_link_values) { + frm.events.show_missing_link_values(frm, missing_link_values); } }) ); @@ -138,5 +138,31 @@ frappe.ui.form.on('Data Import Beta', { ${rows} `); + }, + + show_missing_link_values(frm, missing_link_values) { + let html = Object.keys(missing_link_values) + .map(doctype => { + let values = missing_link_values[doctype]; + return ` +
${doctype}
+
    ${values.map(v => `
  • ${v}
    • `).join('')}
+ `; + }) + .join(''); + + let message = __('There are some linked records which needs to be created before we can import your file. Do you want to create the following missing link records?'); + frappe.confirm(message + html, () => { + frm + .call('create_missing_link_values', { + missing_link_values + }) + .then(r => { + let records = r.message; + frappe.msgprint( + __('Created {0} records successfully.', [records.length]) + ); + }); + }); } }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 213b13ba11..0b3b946a58 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -27,6 +27,21 @@ class DataImportBeta(Document): def get_importer(self): return Importer(self.reference_doctype, data_import=self) + def create_missing_link_values(self, missing_link_values): + docs = [] + for doctype, values in missing_link_values.items(): + meta = frappe.get_meta(doctype) + # find the autoname field + if meta.autoname and meta.autoname.startswith('field:'): + autoname_field = meta.autoname[len('field:') :] + else: + autoname_field = 'name' + + for value in values: + new_doc = frappe.new_doc(doctype) + new_doc.set(autoname_field, value) + docs.append(new_doc.insert()) + return docs @frappe.whitelist() def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type='CSV'): From 11d07b57b9de5a8d6ea32e79b73dace878befb26 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 26 Aug 2019 12:58:27 +0530 Subject: [PATCH 12/83] wip --- .../core/doctype/data_import/importer_new.py | 50 ++++++++----- .../data_import_beta/data_import_beta.js | 62 ++++++++++------ .../data_import_beta/data_import_beta.py | 15 ++-- .../js/frappe/data_import/import_preview.js | 70 ++++++++++++++----- 4 files changed, 136 insertions(+), 61 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index e6305e2437..170627a949 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -40,6 +40,7 @@ class Importer: self.header_row = None self.data = None + # used to store date formats guessed from data rows per column self._guessed_date_formats = {} self.meta = frappe.get_meta(doctype) self.prepare_content(file_path, content) @@ -191,7 +192,9 @@ class Importer: for df in meta.fields: if df.fieldtype not in no_value_fields: # label as key - label = df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) + label = ( + df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) + ) out[label] = df # fieldname as key if self.doctype == doctype: @@ -294,25 +297,35 @@ class Importer: if missing_link_values: return {"missing_link_values": missing_link_values} - if warnings: - return warnings + # parse import data + payloads = self.get_payloads_for_import(fields, data) - for doc in docs: - doc = self.process_doc(doc) - import_log.append({"inserted": True, "name": doc.name}) + # collect warnings + warnings = [] + for payload in payloads: + warnings += payload.warnings + if warnings: + return {"warnings": warnings} + + # start import + print("Importing {0} rows...".format(len(data))) + for payload in payloads: + doc = payload.doc + row_indexes = [row[0] for row in payload.rows] + try: + doc = self.process_doc(doc) + import_log.append({"success": True, "docname": doc.name, "row_indexes": row_indexes}) + except Exception as e: + import_log.append({"success": False, "exception": frappe.get_traceback(), "row_indexes": row_indexes}) self.data_import.db_set("import_log", json.dumps(import_log)) - def get_docs_for_import(self, fields, data): - docs = [] - parse_warnings = [] + def get_payloads_for_import(self, fields, data): + payloads = [] while data: - doc, data, warnings = self.parse_next_row_for_import(fields, data) - if not warnings: - docs.append(doc) - else: - parse_warnings += warnings - return docs, parse_warnings + doc, rows, data, warnings = self.parse_next_row_for_import(fields, data) + payloads.append(frappe._dict(doc=doc, rows=rows, warnings=warnings)) + return payloads def parse_next_row_for_import(self, fields, data): doc = {} @@ -383,7 +396,7 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs - return doc, data[len(rows) :], warnings + return doc, rows, data[len(rows) :], warnings def get_first_parent_column_index(self, fields): """ @@ -406,7 +419,8 @@ class Importer: pass def insert_record(self, doc): - doc.update({"doctype": self.doctype}) + # name shouldn't be set when inserting a new record + doc.update({"doctype": self.doctype, "name": None}) new_doc = frappe.get_doc(doc) return new_doc.insert() @@ -415,6 +429,8 @@ class Importer: def has_one_mandatory_field(doctype): meta = frappe.get_meta(doctype) + # get mandatory fields with default not set + # mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] mandatory_fields = [df for df in meta.fields if df.reqd] mandatory_fields_count = len(mandatory_fields) if meta.autoname.lower() == "prompt": diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 56b90deeeb..a9439494d6 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -6,7 +6,7 @@ frappe.ui.form.on('Data Import Beta', { frm.page.hide_icon_group(); frm.trigger('import_file'); frm.trigger('reference_doctype'); - frm.trigger('show_import_log'); + // frm.trigger('show_import_log'); if (!frm.is_new()) { frm.page.set_primary_action(__('Start Import'), () => frm.events.start_import(frm) @@ -43,34 +43,46 @@ frappe.ui.form.on('Data Import Beta', { }, import_file(frm) { - if (frm.doc.import_file) { - $('') - .html(__('Loading import file...')) - .appendTo(frm.get_field('import_preview').$wrapper); - - frm - .call({ - doc: frm.doc, - method: 'get_preview_from_template', - freeze: true, - freeze_message: __('Preparing Preview...') - }) - .then(r => { - let preview_data = r.message; - frm.events.show_import_preview(frm, preview_data); - }); - } else { - frm.get_field('import_preview').$wrapper.empty(); - } frm.toggle_display('section_import_preview', frm.doc.import_file); + if (!frm.doc.import_file) { + frm.get_field('import_preview').$wrapper.empty(); + return; + } + + // load import preview + $('') + .html(__('Loading import file...')) + .appendTo(frm.get_field('import_preview').$wrapper); + + frm + .call({ + doc: frm.doc, + method: 'get_preview_from_template', + freeze: true, + freeze_message: __('Preparing Preview...') + }) + .then(r => { + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); + }); }, show_import_preview(frm, preview_data) { + let import_log = JSON.parse(frm.doc.import_log || '[]'); + + if (frm.import_preview) { + frm.import_preview.preview_data = preview_data; + frm.import_preview.import_log = import_log; + frm.import_preview.refresh(); + return; + } + frappe.require('/assets/js/data_import_tools.min.js', () => { frm.import_preview = new frappe.data_import.ImportPreview({ wrapper: frm.get_field('import_preview').$wrapper, doctype: frm.doc.reference_doctype, preview_data, + import_log, events: { remap_column(header_row_index, fieldname) { let template_options = JSON.parse(frm.doc.template_options || '{}'); @@ -123,9 +135,15 @@ frappe.ui.form.on('Data Import Beta', { let import_log = JSON.parse(frm.doc.import_log); let rows = import_log .map(log => { + if (log.inserted) { + return ` + ${log.name} + ${log.inserted ? 'Inserted' : ''} + `; + } return ` - ${log.name} - ${log.inserted ? 'Inserted' : ''} + Failed + 
${log.exception}
`; }) .join(''); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 0b3b946a58..5170e2340f 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -8,10 +8,14 @@ from frappe.model.document import Document from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter + class DataImportBeta(Document): def validate(self): - if not self.import_file: - self.import_json = '' + doc_before_save = self.get_doc_before_save() + if not self.import_file or ( + doc_before_save and doc_before_save.import_file != self.import_file + ): + self.template_options = "" def get_preview_from_template(self): if not self.import_file: @@ -44,7 +48,7 @@ class DataImportBeta(Document): return docs @frappe.whitelist() -def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type='CSV'): +def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type="CSV"): """ Download template from Exporter :param doctype: Document Type @@ -57,10 +61,11 @@ def download_template(doctype, export_fields=None, export_records=None, export_f export_fields = frappe.parse_json(export_fields) export_filters = frappe.parse_json(export_filters) - e = Exporter(doctype, + e = Exporter( + doctype, export_fields=export_fields, export_data=True, export_filters=export_filters, - file_type=file_type + file_type=file_type, ) e.build_csv_response() diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index f580293af9..0d4a09de2c 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -4,26 +4,41 @@ import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); +const SVG_ICONS = { + 'checkbox-circle-line': ` + + + + + ` +}; + frappe.data_import.ImportPreview = class ImportPreview { - constructor({ wrapper, doctype, preview_data, events = {} }) { + constructor({ wrapper, doctype, preview_data, import_log, events = {} }) { frappe.import_preview = this; this.wrapper = wrapper; this.doctype = doctype; - this.header_row = preview_data.header_row; - this.preview_fields = preview_data.fields; - this.preview_data = preview_data.data; - this.preview_warnings = preview_data.warnings; + this.preview_data = preview_data; this.events = events; + this.import_log = import_log; frappe.model.with_doctype(doctype, () => { this.make_wrapper(); - this.prepare_columns(); - this.prepare_data(); - this.render_warnings(this.preview_warnings); - this.render_datatable(); + this.refresh(); }); } + refresh() { + this.header_row = this.preview_data.header_row; + this.fields = this.preview_data.fields; + this.data = this.preview_data.data; + this.warnings = this.preview_data.warnings; + this.prepare_columns(); + this.prepare_data(); + this.render_warnings(this.warnings); + this.render_datatable(); + } + make_wrapper() { this.wrapper.html(`
@@ -43,7 +58,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { - this.columns = this.preview_fields.map((df, i) => { + this.columns = this.fields.map((df, i) => { let header_row_index = i - 1; if (df.skip_import) { return { @@ -54,7 +69,13 @@ frappe.data_import.ImportPreview = class ImportPreview { focusable: false, header_row_index, format: (value, row, column, data) => { - return `
${value}
`; + let html = `
${value}
`; + if (df.label === 'Sr. No' && this.is_row_imported(row)) { + html = ` +
${SVG_ICONS['checkbox-circle-line'] + html}
+ `; + } + return html; } }; } @@ -79,7 +100,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_data() { - this.preview_data = this.preview_data.map(row => { + this.data = this.data.map(row => { return row.map(cell => { if (cell == null) { return ''; @@ -104,7 +125,7 @@ frappe.data_import.ImportPreview = class ImportPreview { let self = this; this.datatable = new DataTable(this.$table_preview.get(0), { - data: this.preview_data, + data: this.data, columns: this.columns, layout: 'fixed', cellHeight: 35, @@ -144,7 +165,9 @@ frappe.data_import.ImportPreview = class ImportPreview { columns.forEach(col => { if (!col.skip_import && col.df) { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ + col.colIndex + } .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'green', @@ -156,7 +179,9 @@ frappe.data_import.ImportPreview = class ImportPreview { } if (col.skip_import && col.name !== 'Sr. No') { this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`, + `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ + col.colIndex + } .dt-dropdown__toggle`, { backgroundColor: frappe.ui.color.get_color_shade( 'orange', @@ -170,11 +195,15 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } }); + this.datatable.style.setStyle(`svg.import-success`, { + width: '16px', + fill: frappe.ui.color.get_color_shade('green', 'dark') + }); } add_row() { - this.preview_data.push([]); - this.datatable.refresh(this.preview_data); + this.data.push([]); + this.datatable.refresh(this.data); } remap_column(col) { @@ -204,4 +233,11 @@ frappe.data_import.ImportPreview = class ImportPreview { skip_import(col) { this.events.skip_import(col.header_row_index); } + + is_row_imported(row) { + let serial_no = row[0].content; + return this.import_log.find(log => { + return log.success && log.row_indexes.includes(serial_no); + }); + } }; From b486d5abddf7cb6059394b8655d3bdc34fb0511b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 26 Aug 2019 19:48:48 +0530 Subject: [PATCH 13/83] fix: Missing values Handle DocTypes that have more than one mandatory field. --- .../core/doctype/data_import/importer_new.py | 21 +++++--- .../data_import_beta/data_import_beta.js | 49 ++++++++++++------- .../data_import_beta/data_import_beta.py | 8 ++- 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 170627a949..e2980370a7 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -430,25 +430,30 @@ class Importer: def has_one_mandatory_field(doctype): meta = frappe.get_meta(doctype) # get mandatory fields with default not set - # mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] - mandatory_fields = [df for df in meta.fields if df.reqd] + mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] mandatory_fields_count = len(mandatory_fields) if meta.autoname.lower() == "prompt": mandatory_fields_count += 1 return mandatory_fields_count == 1 - missing_values_map = {} + missing_values_payload = [] for index in link_column_indexes: df = fields[index] column_values = [row[index] for row in data] values = set([v for v in column_values if v not in INVALID_VALUES]) doctype = df.options - if has_one_mandatory_field(doctype): - missing_values = [value for value in values if not frappe.db.exists(doctype, value)] - if missing_values: - missing_values_map[doctype] = missing_values - return missing_values_map + missing_values = [value for value in values if not frappe.db.exists(doctype, value)] + if missing_values: + missing_values_payload.append( + frappe._dict( + doctype=doctype, + missing_values=missing_values, + has_one_mandatory_field=has_one_mandatory_field(doctype), + ) + ) + + return missing_values_payload DATE_FORMATS = [ diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index a9439494d6..46f10f3e55 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -159,28 +159,39 @@ frappe.ui.form.on('Data Import Beta', { }, show_missing_link_values(frm, missing_link_values) { - let html = Object.keys(missing_link_values) - .map(doctype => { - let values = missing_link_values[doctype]; + let can_be_created_automatically = missing_link_values.every( + d => d.has_one_mandatory_field + ); + + let html = missing_link_values + .map(d => { + let doctype = d.doctype; + let values = d.missing_values; return ` -
${doctype}
-
    ${values.map(v => `
  • ${v}
    • `).join('')}
- `; +
${doctype}
+
    ${values.map(v => `
  • ${v}
    • `).join('')}
+ `; }) .join(''); - let message = __('There are some linked records which needs to be created before we can import your file. Do you want to create the following missing link records?'); - frappe.confirm(message + html, () => { - frm - .call('create_missing_link_values', { - missing_link_values - }) - .then(r => { - let records = r.message; - frappe.msgprint( - __('Created {0} records successfully.', [records.length]) - ); - }); - }); + if (can_be_created_automatically) { + let message = __('There are some linked records which needs to be created before we can import your file. Do you want to create the following missing records automatically?'); + frappe.confirm(message + html, () => { + frm + .call('create_missing_link_values', { + missing_link_values + }) + .then(r => { + let records = r.message; + frappe.msgprint( + __('Created {0} records successfully.', [records.length]) + ); + }); + }); + } else { + frappe.msgprint( + __('The following records needs to be created before we can import your file.') + html + ); + } } }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 5170e2340f..f101438a24 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -33,7 +33,13 @@ class DataImportBeta(Document): def create_missing_link_values(self, missing_link_values): docs = [] - for doctype, values in missing_link_values.items(): + for d in missing_link_values: + d = frappe._dict(d) + if not d.has_one_mandatory_field: + continue + + doctype = d.doctype + values = d.missing_values meta = frappe.get_meta(doctype) # find the autoname field if meta.autoname and meta.autoname.startswith('field:'): From fe7db55679dd3b04d1197feafa96da729c5cbe13 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 26 Aug 2019 19:54:14 +0530 Subject: [PATCH 14/83] fix: Importer - Skip already imported rows - Make imported rows read only - Publish progress and show progress bar - Use Savepoint - Status field to track Pending or Success - Add more date formats for parsing --- .../core/doctype/data_import/importer_new.py | 71 +++++++++++++-- .../data_import_beta/data_import_beta.js | 48 ++++++++-- .../data_import_beta/data_import_beta.json | 12 ++- .../js/frappe/data_import/import_preview.js | 91 +++++++++++-------- 4 files changed, 169 insertions(+), 53 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index e2980370a7..75766cfcef 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -290,7 +290,6 @@ class Importer: out = self.get_data_for_import_preview() fields = out["fields"] data = out["data"] - import_log = [] # validate link field values missing_link_values = self.get_missing_link_field_values(fields, data) @@ -307,18 +306,67 @@ class Importer: if warnings: return {"warnings": warnings} + if self.data_import.import_log: + import_log = frappe.parse_json(self.data_import.import_log) + else: + import_log = [] + + # get successfully imported rows + imported_rows = [] + for log in import_log: + log = frappe._dict(log) + if log.success: + imported_rows += log.row_indexes + # start import print("Importing {0} rows...".format(len(data))) - for payload in payloads: + # mark savepoint + frappe.db.sql("SAVEPOINT import") + + total_payload_count = len(payloads) + for i, payload in enumerate(payloads): doc = payload.doc row_indexes = [row[0] for row in payload.rows] + current_index = i + 1 + + if set(row_indexes).intersection(set(imported_rows)): + print("Skipping imported rows", row_indexes) + frappe.publish_realtime( + "data_import_progress", + {"current": current_index, "total": total_payload_count, "skipping": True}, + ) + continue + try: + print("Importing", doc) doc = self.process_doc(doc) - import_log.append({"success": True, "docname": doc.name, "row_indexes": row_indexes}) + frappe.publish_realtime( + "data_import_progress", + { + "current": current_index, + "total": total_payload_count, + "docname": doc.name, + "success": True, + "row_indexes": row_indexes, + }, + ) + import_log.append( + {"success": True, "docname": doc.name, "row_indexes": row_indexes} + ) + except Exception as e: - import_log.append({"success": False, "exception": frappe.get_traceback(), "row_indexes": row_indexes}) + import_log.append( + {"success": False, "exception": frappe.get_traceback(), "row_indexes": row_indexes} + ) + + # rollback to savepoint if something went wrong + # frappe.db.sql('ROLLBACK TO SAVEPOINT import') + + # release savepoint if everything is ok + frappe.db.sql("RELEASE SAVEPOINT import") self.data_import.db_set("import_log", json.dumps(import_log)) + self.data_import.db_set("status", "Success") def get_payloads_for_import(self, fields, data): payloads = [] @@ -457,17 +505,24 @@ class Importer: DATE_FORMATS = [ - r"%Y-%m-%d", r"%d-%m-%Y", r"%m-%d-%Y", - r"%Y/%m/%d", + r"%Y-%m-%d", + r"%d-%m-%y", + r"%m-%d-%y", + r"%y-%m-%d", r"%d/%m/%Y", r"%m/%d/%Y", - r"%m/%d/%y", + r"%Y/%m/%d", r"%d/%m/%y", - r"%Y.%m.%d", + r"%m/%d/%y", + r"%y/%m/%d", r"%d.%m.%Y", r"%m.%d.%Y", + r"%Y.%m.%d", + r"%d.%m.%y", + r"%m.%d.%y", + r"%y.%m.%d", ] TIME_FORMATS = [ diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 46f10f3e55..1a23700cec 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -2,20 +2,52 @@ // For license information, please see license.txt frappe.ui.form.on('Data Import Beta', { + setup(frm) { + frappe.realtime.on('data_import_progress', data => { + let percent = Math.floor((data.current * 100) / data.total); + let message; + if (data.success) { + message = __('Importing {0} ({1} of {2})', [data.docname, data.current, data.total]); + } + if (data.skipping) { + message = __('Skipping ({1} of {2})', [data.current, data.total]); + } + frm.dashboard.show_progress(__('Import Progress'), percent, message); + + // hide progress when complete + if (data.current === data.total) { + setTimeout(() => { + frm.dashboard.hide(); + frm.refresh(); + }, 2000); + } + }); + }, + refresh(frm) { frm.page.hide_icon_group(); frm.trigger('import_file'); - frm.trigger('reference_doctype'); - // frm.trigger('show_import_log'); - if (!frm.is_new()) { - frm.page.set_primary_action(__('Start Import'), () => - frm.events.start_import(frm) - ); + + if (frm.doc.status === 'Pending') { + if (!frm.is_new()) { + frm.page.set_primary_action(__('Start Import'), () => + frm.events.start_import(frm) + ); + } else { + frm.page.set_primary_action(__('Save'), () => frm.save()); + } } else { - frm.page.set_primary_action(__('Save'), () => frm.save()); + frm.disable_save(); + frm.events.after_success(frm); } }, + after_success(frm) { + let import_log = JSON.parse(frm.doc.import_log || '[]'); + let successful_records = import_log.filter(log => log.success); + frm.dashboard.set_headline(__('Successfully imported {0} records', [successful_records.length])); + }, + start_import(frm) { let csv_array = frm.import_preview.get_rows_as_csv_array(); let template_options = JSON.parse(frm.doc.template_options || '{}'); @@ -30,6 +62,8 @@ frappe.ui.form.on('Data Import Beta', { frm.import_preview.render_warnings(warnings); } else if (missing_link_values) { frm.events.show_missing_link_values(frm, missing_link_values); + } else { + frm.refresh(); } }) ); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index e413a9c455..2df5d67fe0 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -11,6 +11,7 @@ "download_sample_file", "import_file", "column_break_5", + "status", "skip_rows_with_errors", "section_break_7", "date_format", @@ -100,16 +101,25 @@ { "fieldname": "import_log_section", "fieldtype": "Section Break", + "hidden": 1, "label": "Import Log" }, { "fieldname": "import_log_preview", "fieldtype": "HTML", "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "label": "Status", + "options": "Pending\nSuccess\nPartial Success", + "read_only": 1 } ], "hide_toolbar": 1, - "modified": "2019-08-22 18:26:28.939065", + "modified": "2019-08-26 18:27:25.663674", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 0d4a09de2c..ee15f47df1 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -23,7 +23,6 @@ frappe.data_import.ImportPreview = class ImportPreview { this.import_log = import_log; frappe.model.with_doctype(doctype, () => { - this.make_wrapper(); this.refresh(); }); } @@ -33,10 +32,12 @@ frappe.data_import.ImportPreview = class ImportPreview { this.fields = this.preview_data.fields; this.data = this.preview_data.data; this.warnings = this.preview_data.warnings; + this.make_wrapper(); this.prepare_columns(); this.prepare_data(); this.render_warnings(this.warnings); this.render_datatable(); + this.setup_styles(); } make_wrapper() { @@ -58,6 +59,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { + let column_width = 120; this.columns = this.fields.map((df, i) => { let header_row_index = i - 1; if (df.skip_import) { @@ -67,12 +69,15 @@ frappe.data_import.ImportPreview = class ImportPreview { skip_import: true, editable: false, focusable: false, + align: 'left', header_row_index, + width: column_width, format: (value, row, column, data) => { let html = `
${value}
`; if (df.label === 'Sr. No' && this.is_row_imported(row)) { html = ` -
${SVG_ICONS['checkbox-circle-line'] + html}
+
${SVG_ICONS['checkbox-circle-line'] + + html}
`; } return html; @@ -94,7 +99,8 @@ frappe.data_import.ImportPreview = class ImportPreview { df: df, editable: true, align: 'left', - header_row_index + header_row_index, + width: column_width }; }); } @@ -111,18 +117,23 @@ frappe.data_import.ImportPreview = class ImportPreview { } render_warnings(warnings) { - let warning_html = warnings - .map(warning => { - return `
${warning}
`; - }) - .join(''); + let html = ''; + if (warnings.length > 0) { + let warning_html = warnings + .map(warning => { + return `
${warning}
`; + }) + .join(''); - let html = `
${warning_html}
`; + html = `
${warning_html}
`; + } this.$warnings.html(html); } render_datatable() { - let self = this; + if (this.datatable) { + this.datatable.destroy(); + } this.datatable = new DataTable(this.$table_preview.get(0), { data: this.data, @@ -150,8 +161,6 @@ frappe.data_import.ImportPreview = class ImportPreview { this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { display: 'none' }); - - this.add_color_to_column_header(); } get_rows_as_csv_array() { @@ -160,45 +169,53 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - add_color_to_column_header() { + setup_styles() { let columns = this.datatable.getColumns(); columns.forEach(col => { + let class_name = [ + `.dt-header .dt-cell--col-${col.colIndex}`, + `.dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle` + ].join(','); + if (!col.skip_import && col.df) { - this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ - col.colIndex - } .dt-dropdown__toggle`, - { - backgroundColor: frappe.ui.color.get_color_shade( - 'green', - 'extra-light' - ), - color: frappe.ui.color.get_color_shade('green', 'dark') - } - ); + this.datatable.style.setStyle(class_name, { + backgroundColor: frappe.ui.color.get_color_shade( + 'green', + 'extra-light' + ), + color: frappe.ui.color.get_color_shade('green', 'dark') + }); } if (col.skip_import && col.name !== 'Sr. No') { - this.datatable.style.setStyle( - `.dt-header .dt-cell--col-${col.colIndex}, .dt-header .dt-cell--col-${ - col.colIndex - } .dt-dropdown__toggle`, - { - backgroundColor: frappe.ui.color.get_color_shade( - 'orange', - 'extra-light' - ), - color: frappe.ui.color.get_color_shade('orange', 'dark') - } - ); + this.datatable.style.setStyle(class_name, { + backgroundColor: frappe.ui.color.get_color_shade( + 'orange', + 'extra-light' + ), + color: frappe.ui.color.get_color_shade('orange', 'dark') + }); this.datatable.style.setStyle(`.dt-cell--col-${col.colIndex}`, { backgroundColor: frappe.ui.color.get_color_shade('white', 'light') }); } }); + // import success checkbox this.datatable.style.setStyle(`svg.import-success`, { width: '16px', fill: frappe.ui.color.get_color_shade('green', 'dark') }); + // make successfully imported rows readonly + let row_classes = this.datatable + .getRows() + .filter(row => this.is_row_imported(row)) + .map(row => row.meta.rowIndex) + .map(i => `.dt-row-${i} .dt-cell`) + .join(','); + this.datatable.style.setStyle(row_classes, { + pointerEvents: 'none', + backgroundColor: frappe.ui.color.get_color_shade('white', 'light'), + color: frappe.ui.color.get_color_shade('black', 'extra-light'), + }); } add_row() { From b05159f3495f4c1ab788367bd9b4951b939851d3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 26 Aug 2019 19:54:26 +0530 Subject: [PATCH 15/83] fix: Remove event listener before attaching --- frappe/public/js/frappe/utils/utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index a285264272..6821e3d410 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -727,7 +727,10 @@ Object.assign(frappe.utils, { return ["ar", "he", "fa"].includes(frappe.boot.lang); }, bind_actions_with_class($el, class_instance) { - $($el).on('click', '[data-action]', e => { + // remove previously bound event + $($el).off('click.class_actions'); + // attach new event + $($el).on('click.class_actions', '[data-action]', e => { let $target = $(e.currentTarget); let action = $target.data('action'); let method = class_instance[action]; From ce5f6a0f128ee3555053c2c444f108fcdfc02eed Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 1 Sep 2019 22:41:56 +0530 Subject: [PATCH 16/83] fix: Parse child rows with non blank values --- .../core/doctype/data_import/importer_new.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 75766cfcef..39b5dcf79a 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -376,24 +376,32 @@ class Importer: return payloads def parse_next_row_for_import(self, fields, data): + """ + Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. + Returns the doc, rows, data without the rows and warnings. + """ doc = {} warnings = [] doctypes = set([df.parent for df in fields if df.parent]) - # get all rows to make this doc # first row is included by default - rows = [data[0]] + first_row = data[0] + rows = [first_row] # if there are child doctypes, find the subsequent rows if len(doctypes) > 1: - # subsequent rows dont have any parent value set - # so we use it to check if any value is set - # if not we include that row + # subsequent rows either dont have any parent value set + # or have the same value as the parent + # we include a row if either of conditions match parent_column_index = self.get_first_parent_column_index(fields) - for d in data[1:]: + parent_value = first_row[parent_column_index] + data_without_first_row = data[1:] + for d in data_without_first_row: value = d[parent_column_index] - # if cell value exists then it is the next doc - if value: + # if value is blank then it's a child row + # if value is same as parent value it's a child row + # if value is different than the parent value, it's the next doc + if value not in INVALID_VALUES and value != parent_value: break rows.append(d) @@ -421,6 +429,11 @@ class Importer: parsed_docs = {} for row_index, row in enumerate(rows): for doctype in doctypes: + if doctype == self.doctype and parsed_docs.get(doctype): + # if parent doc is already parsed from the first row + # then skip + continue + column_indexes = get_column_indexes(doctype) values = [row[i] for i in column_indexes] From e8b78a397802d295941e68c82123ff1e079a3518 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 00:24:15 +0530 Subject: [PATCH 17/83] feat: Handle failures - Capture messages and show them in import log with traceback - Set status as Partial Success for Retry --- .../core/doctype/data_import/importer_new.py | 26 ++++++- .../data_import_beta/data_import_beta.js | 77 +++++++++++++------ .../data_import_beta/data_import_beta.json | 4 +- 3 files changed, 79 insertions(+), 28 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 39b5dcf79a..e1ad8cf739 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -287,6 +287,8 @@ class Importer: return self._guessed_date_formats[fieldname] def import_data(self): + frappe.flags.in_import = True + out = self.get_data_for_import_preview() fields = out["fields"] data = out["data"] @@ -311,6 +313,9 @@ class Importer: else: import_log = [] + # remove failures + import_log = [l for l in import_log if l.get("success") == True] + # get successfully imported rows imported_rows = [] for log in import_log: @@ -351,13 +356,19 @@ class Importer: }, ) import_log.append( - {"success": True, "docname": doc.name, "row_indexes": row_indexes} + frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) ) except Exception as e: import_log.append( - {"success": False, "exception": frappe.get_traceback(), "row_indexes": row_indexes} + frappe._dict( + success=False, + exception=frappe.get_traceback(), + messages=frappe.local.message_log, + row_indexes=row_indexes, + ) ) + frappe.clear_messages() # rollback to savepoint if something went wrong # frappe.db.sql('ROLLBACK TO SAVEPOINT import') @@ -365,8 +376,17 @@ class Importer: # release savepoint if everything is ok frappe.db.sql("RELEASE SAVEPOINT import") + # set status + failures = [l for l in import_log if l.get("success") == False] + if len(failures) > 0: + status = "Partial Success" + else: + status = "Success" + + self.data_import.db_set("status", status) self.data_import.db_set("import_log", json.dumps(import_log)) - self.data_import.db_set("status", "Success") + + frappe.flags.in_import = False def get_payloads_for_import(self, fields, data): payloads = [] diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 1a23700cec..dd99f27c81 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -7,7 +7,11 @@ frappe.ui.form.on('Data Import Beta', { let percent = Math.floor((data.current * 100) / data.total); let message; if (data.success) { - message = __('Importing {0} ({1} of {2})', [data.docname, data.current, data.total]); + message = __('Importing {0} ({1} of {2})', [ + data.docname, + data.current, + data.total + ]); } if (data.skipping) { message = __('Skipping ({1} of {2})', [data.current, data.total]); @@ -27,25 +31,36 @@ frappe.ui.form.on('Data Import Beta', { refresh(frm) { frm.page.hide_icon_group(); frm.trigger('import_file'); + frm.trigger('show_import_log'); - if (frm.doc.status === 'Pending') { + if (frm.doc.status === 'Success') { + // set form as readonly + frm.doc.docstatus = 1; + frm.page.clear_secondary_action(); + frm.disable_save(); + frm.events.show_success_message(frm); + } else { if (!frm.is_new()) { - frm.page.set_primary_action(__('Start Import'), () => + let label = frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); + frm.page.set_primary_action(label, () => frm.events.start_import(frm) ); } else { frm.page.set_primary_action(__('Save'), () => frm.save()); } - } else { - frm.disable_save(); - frm.events.after_success(frm); } }, - after_success(frm) { + show_success_message(frm) { let import_log = JSON.parse(frm.doc.import_log || '[]'); let successful_records = import_log.filter(log => log.success); - frm.dashboard.set_headline(__('Successfully imported {0} records', [successful_records.length])); + let link = `${__('{0} List', [frm.doc.reference_doctype])}`; + frm.dashboard.set_headline( + __('Successfully imported {0} records. Go to {1}', [ + successful_records.length, + link + ]) + ); }, start_import(frm) { @@ -55,7 +70,8 @@ frappe.ui.form.on('Data Import Beta', { frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => { - frm.trigger('import_file').then(() => + frm.trigger('import_file').then(() => { + console.log('import_file ') frm.call('start_import').then(r => { let { warnings, missing_link_values } = r.message || {}; if (warnings) { @@ -66,7 +82,7 @@ frappe.ui.form.on('Data Import Beta', { frm.refresh(); } }) - ); + }); }); }, @@ -161,31 +177,46 @@ frappe.ui.form.on('Data Import Beta', { }, show_import_log(frm) { + let import_log = JSON.parse(frm.doc.import_log || '[]'); + let failures = import_log.filter(log => !log.success); frm.toggle_display('import_log', false); - if (!frm.doc.import_log) { + frm.toggle_display('import_log_preview', failures.length > 0); + + if (failures.length === 0) { frm.get_field('import_log_preview').$wrapper.empty(); return; } - let import_log = JSON.parse(frm.doc.import_log); - let rows = import_log + + let rows = failures .map(log => { - if (log.inserted) { - return ` - ${log.name} - ${log.inserted ? 'Inserted' : ''} - `; - } + let messages = log.messages.map(JSON.parse).map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `

${m.message}

` : ''; + return title + message; + }).join(''); + let id = frappe.dom.get_unique_id(); return ` - Failed - 
${log.exception}
+ ${log.row_indexes.join(', ')} + + ${messages} + +
+
+ 
${log.exception}
+
+
+ `; }) .join(''); + frm.get_field('import_log_preview').$wrapper.html(` - - + + ${rows}
${__('Document Name')}${__('Status')}${__('Row Number')}${__('Error Message')}
diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 2df5d67fe0..5d02bacba7 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -74,6 +74,7 @@ }, { "collapsible": 1, + "depends_on": "eval:!doc.__islocal", "fieldname": "section_break_7", "fieldtype": "Section Break", "label": "Import Options" @@ -101,7 +102,6 @@ { "fieldname": "import_log_section", "fieldtype": "Section Break", - "hidden": 1, "label": "Import Log" }, { @@ -119,7 +119,7 @@ } ], "hide_toolbar": 1, - "modified": "2019-08-26 18:27:25.663674", + "modified": "2019-09-01 22:11:16.057751", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From fff2c0cded4eb9e39d7d5f1b4a97994d3b07125b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 01:24:07 +0530 Subject: [PATCH 18/83] feat: Update Existing Records --- .../core/doctype/data_import/importer_new.py | 20 ++++++++- .../data_import_beta/data_import_beta.js | 42 +++++++++---------- .../data_import_beta/data_import_beta.json | 4 +- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index e1ad8cf739..5b4b052a28 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -497,7 +497,7 @@ class Importer: if import_type == "Insert New Records": return self.insert_record(doc) elif import_type == "Update Existing Records": - pass + return self.update_record(doc) def insert_record(self, doc): # name shouldn't be set when inserting a new record @@ -505,6 +505,15 @@ class Importer: new_doc = frappe.get_doc(doc) return new_doc.insert() + def update_record(self, doc): + id_fieldname = self.get_id_fieldname() + id_value = doc[id_fieldname] + existing_doc = frappe.get_doc(self.doctype, { id_fieldname: id_value }) + existing_doc.flags.via_data_import = self.data_import.name + existing_doc.update(doc) + existing_doc.save() + return existing_doc + def get_missing_link_field_values(self, fields, data): link_column_indexes = [i for i, df in enumerate(fields) if df.fieldtype == "Link"] @@ -536,6 +545,15 @@ class Importer: return missing_values_payload + def get_id_fieldname(self): + autoname = self.meta.autoname + if autoname and autoname.startswith("field:"): + fieldname = autoname[len("field:") :] + autoname_field = self.meta.get_field(fieldname) + if autoname_field: + return autoname_field.fieldname + return 'name' + DATE_FORMATS = [ r"%d-%m-%Y", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index dd99f27c81..cbd2ef6cee 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -40,7 +40,7 @@ frappe.ui.form.on('Data Import Beta', { frm.disable_save(); frm.events.show_success_message(frm); } else { - if (!frm.is_new()) { + if (!frm.is_new() && frm.doc.import_file) { let label = frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); frm.page.set_primary_action(label, () => frm.events.start_import(frm) @@ -54,13 +54,15 @@ frappe.ui.form.on('Data Import Beta', { show_success_message(frm) { let import_log = JSON.parse(frm.doc.import_log || '[]'); let successful_records = import_log.filter(log => log.success); - let link = `${__('{0} List', [frm.doc.reference_doctype])}`; - frm.dashboard.set_headline( - __('Successfully imported {0} records. Go to {1}', [ - successful_records.length, - link - ]) - ); + let link = ` + ${__('{0} List', [frm.doc.reference_doctype])} + `; + let message_args = [successful_records.length, link]; + let message = + frm.doc.import_type === 'Insert New Records' + ? __('Successfully imported {0} records. Go to {1}', message_args) + : __('Successfully updated {0} records. Go to {1}', message_args); + frm.dashboard.set_headline(message); }, start_import(frm) { @@ -70,18 +72,16 @@ frappe.ui.form.on('Data Import Beta', { frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => { - frm.trigger('import_file').then(() => { - console.log('import_file ') - frm.call('start_import').then(r => { - let { warnings, missing_link_values } = r.message || {}; - if (warnings) { - frm.import_preview.render_warnings(warnings); - } else if (missing_link_values) { - frm.events.show_missing_link_values(frm, missing_link_values); - } else { - frm.refresh(); - } - }) + console.log('import_file ') + frm.call('start_import').then(r => { + let { warnings, missing_link_values } = r.message || {}; + if (warnings) { + frm.import_preview.render_warnings(warnings); + } else if (missing_link_values) { + frm.events.show_missing_link_values(frm, missing_link_values); + } else { + frm.refresh(); + } }); }); }, @@ -180,7 +180,7 @@ frappe.ui.form.on('Data Import Beta', { let import_log = JSON.parse(frm.doc.import_log || '[]'); let failures = import_log.filter(log => !log.success); frm.toggle_display('import_log', false); - frm.toggle_display('import_log_preview', failures.length > 0); + frm.toggle_display('import_log_section', failures.length > 0); if (failures.length === 0) { frm.get_field('import_log_preview').$wrapper.empty(); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 5d02bacba7..8e939465c3 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -36,7 +36,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Import Type", - "options": "Insert New Records\nUpdate Existing Records", + "options": "\nInsert New Records\nUpdate Existing Records", "reqd": 1 }, { @@ -119,7 +119,7 @@ } ], "hide_toolbar": 1, - "modified": "2019-09-01 22:11:16.057751", + "modified": "2019-09-02 00:52:17.043355", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From 163d974b0b63ec87914cca3bde2021b7bd358d8b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 01:24:57 +0530 Subject: [PATCH 19/83] fix: Show changes via Data Import in timeline --- frappe/core/doctype/version/version.py | 6 ++- .../public/js/frappe/form/footer/timeline.js | 48 ++++++++++++++----- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 284e79cf62..05cc102ab9 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -43,7 +43,9 @@ def get_diff(old, new, for_child=False): if not new: return None - out = frappe._dict(changed = [], added = [], removed = [], row_changed = []) + # capture data import if set + data_import = new.flags.via_data_import + out = frappe._dict(changed = [], added = [], removed = [], row_changed = [], data_import=data_import) for df in new.meta.fields: if df.fieldtype in no_value_fields and df.fieldtype not in table_fields: continue @@ -91,4 +93,4 @@ def get_diff(old, new, for_child=False): return None def on_doctype_update(): - frappe.db.add_index("Version", ["ref_doctype", "docname"]) \ No newline at end of file + frappe.db.add_index("Version", ["ref_doctype", "docname"]) diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js index fbb047fbbd..96481136b4 100644 --- a/frappe/public/js/frappe/form/footer/timeline.js +++ b/frappe/public/js/frappe/form/footer/timeline.js @@ -571,15 +571,28 @@ frappe.ui.form.Timeline = class Timeline { return; } + let data_import_link = frappe.utils.get_form_link( + 'Data Import Beta', + data.data_import, + true, + __('via Data Import') + ); + // value changed in parent - if (data.changed && data.changed.length) { - const parts = []; - data.changed.every(function (p) { - if (p[0] === 'docstatus') { - if (p[2] == 1) { - out.push(me.get_version_comment(version, __('submitted this document'))); - } else if (p[2] == 2) { - out.push(me.get_version_comment(version, __('cancelled this document'))); + if(data.changed && data.changed.length) { + var parts = []; + data.changed.every(function(p) { + if(p[0]==='docstatus') { + if(p[2]==1) { + let message = data.data_import + ? __('submitted this document {0}', [data_import_link]) + : __('submitted this document'); + out.push(me.get_version_comment(version, message)); + } else if (p[2]==2) { + let message = data.data_import + ? __('cancelled this document {0}', [data_import_link]) + : __('cancelled this document'); + out.push(me.get_version_comment(version, message)); } } else { p = p.map(frappe.utils.escape_html); @@ -598,8 +611,14 @@ frappe.ui.form.Timeline = class Timeline { } return parts.length < 3; }); - if (parts.length) { - out.push(me.get_version_comment(version, __('changed value of {0}', [parts.join(', ')]))); + if(parts.length) { + let message; + if (data.data_import) { + message = __("changed value of {0} {1}", [parts.join(', ').bold(), data_import_link]); + } else { + message = __("changed value of {0}", [parts.join(', ').bold()]); + } + out.push(me.get_version_comment(version, message)); } } @@ -631,8 +650,13 @@ frappe.ui.form.Timeline = class Timeline { return parts.length < 3; }); if(parts.length) { - out.push(me.get_version_comment(version, __("changed values for {0}", - [parts.join(', ')]))); + let message; + if (data.data_import) { + message = __("changed values for {0} {1}", [parts.join(', '), data_import_link]) + } else { + message = __("changed values for {0}", [parts.join(', ')]) + } + out.push(me.get_version_comment(version, message)); } } From 1dab6ae90b39b3780d75161ac8e9b229b86b117a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 01:40:01 +0530 Subject: [PATCH 20/83] fix: Singular messages --- .../doctype/data_import_beta/data_import_beta.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index cbd2ef6cee..6df7bf1922 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -58,10 +58,18 @@ frappe.ui.form.on('Data Import Beta', { ${__('{0} List', [frm.doc.reference_doctype])} `; let message_args = [successful_records.length, link]; - let message = - frm.doc.import_type === 'Insert New Records' - ? __('Successfully imported {0} records. Go to {1}', message_args) - : __('Successfully updated {0} records. Go to {1}', message_args); + let message; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records.length > 1 + ? __('Successfully imported {0} records. Go to {1}', message_args) + : __('Successfully imported {0} record. Go to {1}', message_args); + } else { + message = + successful_records.length > 1 + ? __('Successfully updated {0} records. Go to {1}', message_args) + : __('Successfully updated {0} record. Go to {1}', message_args); + } frm.dashboard.set_headline(message); }, From b099414cf8e5d1feec52cd5180a3af7b85a2fab9 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 02:21:13 +0530 Subject: [PATCH 21/83] fix: Set readonly without docstatus --- frappe/core/doctype/data_import_beta/data_import_beta.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 6df7bf1922..63e9d67928 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -35,8 +35,7 @@ frappe.ui.form.on('Data Import Beta', { if (frm.doc.status === 'Success') { // set form as readonly - frm.doc.docstatus = 1; - frm.page.clear_secondary_action(); + frm.fields.forEach(f => f.df.read_only = 1); frm.disable_save(); frm.events.show_success_message(frm); } else { From c3d85cb4bddfd082e49f7bc8ff57593c17be88c0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 12:57:21 +0530 Subject: [PATCH 22/83] fix: Only show importable doctypes --- frappe/core/doctype/data_import_beta/data_import_beta.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 63e9d67928..938a67fe8c 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -26,6 +26,14 @@ frappe.ui.form.on('Data Import Beta', { }, 2000); } }); + + frm.set_query('reference_doctype', () => { + return { + filters: { + allow_import: 1 + } + } + }); }, refresh(frm) { From 7d2c9feb8e5a9b63029cf3ac4758070f970b3f6f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 2 Sep 2019 12:59:25 +0530 Subject: [PATCH 23/83] fix: Hide unused fields --- .../doctype/data_import_beta/data_import_beta.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 8e939465c3..a54ccfbfcb 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -12,7 +12,6 @@ "import_file", "column_break_5", "status", - "skip_rows_with_errors", "section_break_7", "date_format", "template_options", @@ -66,17 +65,12 @@ "fieldname": "column_break_5", "fieldtype": "Column Break" }, - { - "default": "0", - "fieldname": "skip_rows_with_errors", - "fieldtype": "Check", - "label": "Skip rows with errors" - }, { "collapsible": 1, "depends_on": "eval:!doc.__islocal", "fieldname": "section_break_7", "fieldtype": "Section Break", + "hidden": 1, "label": "Import Options" }, { @@ -119,7 +113,7 @@ } ], "hide_toolbar": 1, - "modified": "2019-09-02 00:52:17.043355", + "modified": "2019-09-02 12:58:49.053073", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From 421e93457babe2fc9f53aef5a223f857ac7fa658 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 3 Sep 2019 18:05:22 +0530 Subject: [PATCH 24/83] fix: Option to export blank template --- frappe/core/doctype/data_import/exporter_new.py | 4 ++-- frappe/core/doctype/data_import_beta/data_import_beta.js | 6 ++++++ frappe/core/doctype/data_import_beta/data_import_beta.py | 5 +++-- frappe/public/js/frappe/data_import/data_exporter.js | 7 ++++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index aad4ecc3f2..36e2d87800 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -245,7 +245,7 @@ class Exporter: csv_array = self.csv_array if not self.data: - # add 5 empty rows - csv_array += [[]] * 5 + # add 2 empty rows + csv_array += [[]] * 2 build_csv_response(csv_array, self.doctype) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 938a67fe8c..08e848b9f2 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -34,6 +34,12 @@ frappe.ui.form.on('Data Import Beta', { } } }); + + frm.get_field('import_file').df.options = { + restrictions: { + allowed_file_types: ['.csv', '.xls', '.xlsx'] + } + }; }, refresh(frm) { diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index f101438a24..be8b8b8c38 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -59,18 +59,19 @@ def download_template(doctype, export_fields=None, export_records=None, export_f Download template from Exporter :param doctype: Document Type :param export_fields=None: Fields to export as dict {'Sales Invoice': ['name', 'customer'], 'Sales Invoice Item': ['item_code']} - :param export_records=None: One of 'all', 'last_10_records', 'by_filter' + :param export_records=None: One of 'all', 'by_filter', 'blank_template' :param export_filters: Filter dict :param file_type: File type to export into """ export_fields = frappe.parse_json(export_fields) export_filters = frappe.parse_json(export_filters) + export_data = export_records != 'blank_template' e = Exporter( doctype, export_fields=export_fields, - export_data=True, + export_data=export_data, export_filters=export_filters, file_type=file_type, ) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 0532d5b002..b4121411ea 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -30,6 +30,10 @@ frappe.data_import.DataExporter = class DataExporter { { label: __('Export Filtered Records'), value: 'by_filter' + }, + { + label: __('Export Blank Template'), + value: 'blank_template' } ], default: 'all', @@ -200,7 +204,8 @@ frappe.data_import.DataExporter = class DataExporter { by_filter: () => frappe.db.count(this.doctype, { filters: this.get_filters() - }) + }), + blank_template: () => Promise.resolve(0) }; count_method[export_records]().then(value => { From fc708655a069e356e8f9f1292b1e04a09aa65189 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 3 Sep 2019 18:05:44 +0530 Subject: [PATCH 25/83] fix: Remove Add Row --- frappe/public/js/frappe/data_import/import_preview.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index ee15f47df1..d007865a44 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -46,9 +46,11 @@ frappe.data_import.ImportPreview = class ImportPreview {
+
`); From 84362550cb813218256b78aca0c9f0e99f8aa5d3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 3 Sep 2019 18:19:08 +0530 Subject: [PATCH 26/83] fix: Allow exporting hidden fields --- frappe/core/doctype/data_import/exporter_new.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index 36e2d87800..c4f98d25ba 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -82,7 +82,6 @@ class Exporter: return ( df.fieldtype not in display_fieldtypes and df.fieldtype not in no_value_fields - and not df.hidden ) meta = frappe.get_meta(doctype) From 9d770cf7e670a603cda65ff5e652a4374308d069 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:04:29 +0530 Subject: [PATCH 27/83] fix: Support request level error handlers --- frappe/public/js/frappe/request.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index a96e8c2acb..87bf29996d 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -98,6 +98,7 @@ frappe.call = function(opts) { freeze: opts.freeze, freeze_message: opts.freeze_message, headers: opts.headers || {}, + error_handlers: opts.error_handlers || {}, // show_spinner: !opts.no_spinner, async: opts.async, url, @@ -324,9 +325,12 @@ frappe.request.cleanup = function(opts, r) { return; } - // global error handlers + // error handlers + let global_handlers = frappe.request.error_handlers[r.exc_type] || []; + let request_handler = opts.error_handlers ? opts.error_handlers[r.exc_type] : null; + let handlers = [].concat(global_handlers, request_handler).filter(Boolean); + if (r.exc_type) { - let handlers = frappe.request.error_handlers[r.exc_type] || []; handlers.forEach(handler => { handler(r); }); @@ -334,9 +338,8 @@ frappe.request.cleanup = function(opts, r) { // show messages if(r._server_messages && !opts.silent) { - let handlers = frappe.request.error_handlers[r.exc_type] || []; - // dont show server messages if their handlers exist - if (!handlers.length) { + // show server messages if no handlers exist + if (handlers.length === 0) { r._server_messages = JSON.parse(r._server_messages); frappe.hide_msgprint(); frappe.msgprint(r._server_messages); From 2795ab6b40869be092fb42d8fec64b0bbf3b222a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:04:59 +0530 Subject: [PATCH 28/83] fix: Silently ignore Timestamp Mismatch Error --- frappe/core/doctype/data_import_beta/data_import_beta.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 08e848b9f2..67a8a334e4 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -130,7 +130,12 @@ frappe.ui.form.on('Data Import Beta', { doc: frm.doc, method: 'get_preview_from_template', freeze: true, - freeze_message: __('Preparing Preview...') + freeze_message: __('Preparing Preview...'), + error_handlers: { + TimestampMismatchError() { + // ignore this error + } + } }) .then(r => { let preview_data = r.message; From b973a5d1019de9baf7760b99b34fd567531632d0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:05:45 +0530 Subject: [PATCH 29/83] fix: Process data in batches --- frappe/core/doctype/data_import/importer_new.py | 5 ++++- frappe/utils/__init__.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 5b4b052a28..64b5d12dbf 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -329,7 +329,10 @@ class Importer: frappe.db.sql("SAVEPOINT import") total_payload_count = len(payloads) - for i, payload in enumerate(payloads): + batch_size = frappe.conf.data_import_batch_size or 1000 + + for batched_payloads in frappe.utils.create_batch(payloads, batch_size): + for i, payload in enumerate(batched_payloads): doc = payload.doc row_indexes = [row[0] for row in payload.rows] current_index = i + 1 diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index df96111f20..80bcad3ddb 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -670,3 +670,11 @@ def get_safe_filters(filters): pass return filters + +def create_batch(iterable, batch_size): + """ + Convert an iterable to multiple batches of constant size of batch_size + """ + total_count = len(iterable) + for i in range(0, total_count, batch_size): + yield iterable[i:min(i + batch_size, total_count)] From a1216c929ef646eed90667f7477c57f0fefa1ae7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:06:31 +0530 Subject: [PATCH 30/83] fix: name column matching --- frappe/core/doctype/data_import/importer_new.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 64b5d12dbf..4fff226c63 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -203,6 +203,17 @@ class Importer: key = "{0}:{1}".format(doctype, df.fieldname) out[key] = df + # name field + out["name"] = frappe._dict( + { + "fieldtype": "Data", + "fieldname": "name", + "label": "ID", + "reqd": self.data_import.import_type == "Update Existing Records", + "parent": self.doctype, + } + ) + # if autoname is based on field # add an entry for "ID (Autoname Field)" autoname = self.meta.autoname From d960affe199037c1fbb4ba8d83aae79ec618e351 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:07:01 +0530 Subject: [PATCH 31/83] fix: Realtime update message for import types --- .../core/doctype/data_import_beta/data_import_beta.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 67a8a334e4..dc7a1a1901 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -7,11 +7,11 @@ frappe.ui.form.on('Data Import Beta', { let percent = Math.floor((data.current * 100) / data.total); let message; if (data.success) { - message = __('Importing {0} ({1} of {2})', [ - data.docname, - data.current, - data.total - ]); + let message_args = [data.docname, data.current, data.total]; + message = + frm.doc.import_type === 'Insert New Records' + ? __('Importing {0} ({1} of {2})', message_args) + : __('Updating {0} ({1} of {2})', message_args); } if (data.skipping) { message = __('Skipping ({1} of {2})', [data.current, data.total]); From 87e525d40c78a68de57f76ae540e84b24cd4fcb2 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 02:56:10 +0530 Subject: [PATCH 32/83] fix: Run Import in background job --- .../core/doctype/data_import/importer_new.py | 80 ++++++++++--------- .../data_import_beta/data_import_beta.js | 6 ++ .../data_import_beta/data_import_beta.py | 35 ++++++-- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 4fff226c63..d5c634bf2b 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -319,12 +319,13 @@ class Importer: if warnings: return {"warnings": warnings} + # setup import log if self.data_import.import_log: import_log = frappe.parse_json(self.data_import.import_log) else: import_log = [] - # remove failures + # remove previous failures from import log import_log = [l for l in import_log if l.get("success") == True] # get successfully imported rows @@ -342,47 +343,47 @@ class Importer: total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 - for batched_payloads in frappe.utils.create_batch(payloads, batch_size): + for batch_index, batched_payloads in enumerate(frappe.utils.create_batch(payloads, batch_size)): for i, payload in enumerate(batched_payloads): - doc = payload.doc - row_indexes = [row[0] for row in payload.rows] - current_index = i + 1 + doc = payload.doc + row_indexes = [row[0] for row in payload.rows] + current_index = (i + 1) + (batch_index * batch_size) - if set(row_indexes).intersection(set(imported_rows)): - print("Skipping imported rows", row_indexes) - frappe.publish_realtime( - "data_import_progress", - {"current": current_index, "total": total_payload_count, "skipping": True}, - ) - continue - - try: - print("Importing", doc) - doc = self.process_doc(doc) - frappe.publish_realtime( - "data_import_progress", - { - "current": current_index, - "total": total_payload_count, - "docname": doc.name, - "success": True, - "row_indexes": row_indexes, - }, - ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) - ) - - except Exception as e: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, + if set(row_indexes).intersection(set(imported_rows)): + print("Skipping imported rows", row_indexes) + frappe.publish_realtime( + "data_import_progress", + {"current": current_index, "total": total_payload_count, "skipping": True}, ) - ) - frappe.clear_messages() + continue + + try: + print("Importing", doc) + doc = self.process_doc(doc) + frappe.publish_realtime( + "data_import_progress", + { + "current": current_index, + "total": total_payload_count, + "docname": doc.name, + "success": True, + "row_indexes": row_indexes, + }, + ) + import_log.append( + frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) + ) + + except Exception as e: + import_log.append( + frappe._dict( + success=False, + exception=frappe.get_traceback(), + messages=frappe.local.message_log, + row_indexes=row_indexes, + ) + ) + frappe.clear_messages() # rollback to savepoint if something went wrong # frappe.db.sql('ROLLBACK TO SAVEPOINT import') @@ -401,6 +402,7 @@ class Importer: self.data_import.db_set("import_log", json.dumps(import_log)) frappe.flags.in_import = False + frappe.publish_realtime("data_import_refresh") def get_payloads_for_import(self, fields, data): payloads = [] diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index dc7a1a1901..3fb3f2e6ec 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -3,6 +3,12 @@ frappe.ui.form.on('Data Import Beta', { setup(frm) { + frappe.realtime.on('data_import_refresh', () => { + frappe.model.clear_doc('Data Import Beta', frm.doc.name); + frappe.model.with_doc('Data Import Beta', frm.doc.name).then(() => { + frm.refresh(); + }); + }); frappe.realtime.on('data_import_progress', data => { let percent = Math.floor((data.current * 100) / data.total); let message; diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index be8b8b8c38..dc22f662b2 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -7,6 +7,8 @@ import frappe from frappe.model.document import Document from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter +from frappe.core.page.background_jobs.background_jobs import get_info +from frappe.utils.background_jobs import enqueue class DataImportBeta(Document): @@ -25,8 +27,17 @@ class DataImportBeta(Document): return i.get_data_for_import_preview() def start_import(self): - i = self.get_importer() - return i.import_data() + enqueued_jobs = [d.get("job_name") for d in get_info()] + + if self.name not in enqueued_jobs: + enqueue( + start_import, + queue="default", + timeout=6000, + event="data_import", + job_name=self.name, + data_import=self.name, + ) def get_importer(self): return Importer(self.reference_doctype, data_import=self) @@ -42,10 +53,10 @@ class DataImportBeta(Document): values = d.missing_values meta = frappe.get_meta(doctype) # find the autoname field - if meta.autoname and meta.autoname.startswith('field:'): - autoname_field = meta.autoname[len('field:') :] + if meta.autoname and meta.autoname.startswith("field:"): + autoname_field = meta.autoname[len("field:") :] else: - autoname_field = 'name' + autoname_field = "name" for value in values: new_doc = frappe.new_doc(doctype) @@ -53,8 +64,18 @@ class DataImportBeta(Document): docs.append(new_doc.insert()) return docs + +def start_import(data_import): + """This method runs in background job""" + data_import = frappe.get_doc("Data Import Beta", data_import) + i = Importer(data_import.reference_doctype, data_import=data_import) + return i.import_data() + + @frappe.whitelist() -def download_template(doctype, export_fields=None, export_records=None, export_filters=None, file_type="CSV"): +def download_template( + doctype, export_fields=None, export_records=None, export_filters=None, file_type="CSV" +): """ Download template from Exporter :param doctype: Document Type @@ -66,7 +87,7 @@ def download_template(doctype, export_fields=None, export_records=None, export_f export_fields = frappe.parse_json(export_fields) export_filters = frappe.parse_json(export_filters) - export_data = export_records != 'blank_template' + export_data = export_records != "blank_template" e = Exporter( doctype, From fc28a7eb727df835bec6bbc11247eda9d3a7ec5e Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 6 Sep 2019 03:30:24 +0530 Subject: [PATCH 33/83] fix: Show ETA --- frappe/core/doctype/data_import/importer_new.py | 13 +++++++++++++ .../doctype/data_import_beta/data_import_beta.js | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index d5c634bf2b..911d6a2501 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -5,6 +5,7 @@ import io import csv import json +import timeit import frappe from datetime import datetime from frappe import _ @@ -42,6 +43,7 @@ class Importer: self.data = None # used to store date formats guessed from data rows per column self._guessed_date_formats = {} + self.last_eta = 0 self.meta = frappe.get_meta(doctype) self.prepare_content(file_path, content) @@ -359,7 +361,10 @@ class Importer: try: print("Importing", doc) + start = timeit.default_timer() doc = self.process_doc(doc) + processing_time = timeit.default_timer() - start + eta = self.get_eta(current_index, total_payload_count, processing_time) frappe.publish_realtime( "data_import_progress", { @@ -368,6 +373,7 @@ class Importer: "docname": doc.name, "success": True, "row_indexes": row_indexes, + "eta": eta }, ) import_log.append( @@ -570,6 +576,13 @@ class Importer: return autoname_field.fieldname return 'name' + def get_eta(self, current, total, processing_time): + remaining = total - current + eta = processing_time * remaining + if not self.last_eta or eta < self.last_eta: + self.last_eta = eta + return self.last_eta + DATE_FORMATS = [ r"%d-%m-%Y", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 3fb3f2e6ec..a3b4b40133 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -11,6 +11,10 @@ frappe.ui.form.on('Data Import Beta', { }); frappe.realtime.on('data_import_progress', data => { let percent = Math.floor((data.current * 100) / data.total); + let eta_message = + data.eta < 60 + ? __('ETA {0} seconds', [Math.floor(data.eta)]) + : __('ETA {0} minutes', [Math.floor(data.eta / 60)]); let message; if (data.success) { let message_args = [data.docname, data.current, data.total]; @@ -23,6 +27,7 @@ frappe.ui.form.on('Data Import Beta', { message = __('Skipping ({1} of {2})', [data.current, data.total]); } frm.dashboard.show_progress(__('Import Progress'), percent, message); + frm.page.set_indicator(eta_message, 'orange'); // hide progress when complete if (data.current === data.total) { @@ -68,6 +73,7 @@ frappe.ui.form.on('Data Import Beta', { frm.page.set_primary_action(__('Save'), () => frm.save()); } } + frm.page.set_indicator(__(frm.doc.status), frm.doc.status === 'Success' ? 'green' : 'grey'); }, show_success_message(frm) { From 6f2fdbf106ef42e4ea19d284f577c96e3c1b5213 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 16 Sep 2019 01:30:32 +0530 Subject: [PATCH 34/83] fix: Ignore non reqd tables in Select Mandatory --- .../js/frappe/data_import/data_exporter.js | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index b4121411ea..3c5c72fd93 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -11,10 +11,6 @@ frappe.data_import.DataExporter = class DataExporter { } make_dialog() { - let doctypes = [this.doctype].concat( - ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) - ); - this.dialog = new frappe.ui.Dialog({ title: __('Export Data'), fields: [ @@ -60,23 +56,33 @@ frappe.data_import.DataExporter = class DataExporter { fieldtype: 'HTML', fieldname: 'select_all_buttons' }, - ...doctypes.map(doctype => { - return { - label: __(doctype), - fieldname: doctype, - fieldtype: 'MultiCheck', - columns: 2, - on_change: () => { - this.update_primary_action(); - }, - options: this.get_multicheck_options(doctype) - }; - }) + { + label: __(this.doctype), + fieldname: this.doctype, + fieldtype: 'MultiCheck', + columns: 2, + on_change: () => this.update_primary_action(), + options: this.get_multicheck_options(this.doctype) + }, + ...frappe.meta.get_table_fields(this.doctype) + .map(df => { + let doctype = df.options; + let label = df.reqd + ? __('{0} (1 row mandatory)', [doctype]) + : __(doctype); + return { + label, + fieldname: doctype, + fieldtype: 'MultiCheck', + columns: 2, + on_change: () => this.update_primary_action(), + options: this.get_multicheck_options(doctype) + }; + }) ], primary_action_label: __('Export'), - primary_action: values => { - this.export_records(values); - } + primary_action: values => this.export_records(values), + on_page_show: () => this.select_mandatory() }); this.make_filter_area(); @@ -159,9 +165,16 @@ frappe.data_import.DataExporter = class DataExporter { } select_mandatory() { + let mandatory_table_doctypes = frappe.meta + .get_table_fields(this.doctype) + .filter(df => df.reqd) + .map(df => df.options); + mandatory_table_doctypes.push(this.doctype); + let multicheck_fields = this.dialog.fields .filter(df => df.fieldtype === 'MultiCheck') - .map(df => df.fieldname); + .map(df => df.fieldname) + .filter(doctype => mandatory_table_doctypes.includes(doctype)); let checkboxes = [].concat( ...multicheck_fields.map(fieldname => { @@ -281,7 +294,7 @@ frappe.data_import.DataExporter = class DataExporter { label, value: df.fieldname, danger: df.reqd, - checked: df.reqd, + checked: false, description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}` }; }); From dca054aa0cefa74db9458d0921588ef9edfae2cb Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 16 Sep 2019 01:35:55 +0530 Subject: [PATCH 35/83] fix: Persistent template warnings --- .../core/doctype/data_import/importer_new.py | 40 ++++++++++++++----- .../data_import_beta/data_import_beta.js | 4 ++ .../data_import_beta/data_import_beta.json | 9 +++++ .../js/frappe/data_import/import_preview.js | 10 ++--- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 911d6a2501..f4cc8fbe94 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -305,21 +305,26 @@ class Importer: out = self.get_data_for_import_preview() fields = out["fields"] data = out["data"] + warnings = [] # validate link field values missing_link_values = self.get_missing_link_field_values(fields, data) - if missing_link_values: - return {"missing_link_values": missing_link_values} + for d in missing_link_values: + msg = _('The following linked values are missing for the Document Type {0}:').format(frappe.bold(_(d.doctype))) + msg += ' ' + ', '.join(d.missing_values) + warnings.append(msg) # parse import data payloads = self.get_payloads_for_import(fields, data) # collect warnings - warnings = [] for payload in payloads: warnings += payload.warnings + if warnings: - return {"warnings": warnings} + self.data_import.db_set("template_warnings", json.dumps(warnings)) + frappe.publish_realtime("data_import_refresh") + return # setup import log if self.data_import.import_log: @@ -424,6 +429,7 @@ class Importer: """ doc = {} warnings = [] + mandatory_fields = [] doctypes = set([df.parent for df in fields if df.parent]) # first row is included by default @@ -458,14 +464,12 @@ class Importer: if value in INVALID_VALUES: if df.reqd: - warnings.append( - _("Row {0}: {1} is a mandatory field").format(row_index, frappe.bold(df.label)) - ) + mandatory_fields.append(frappe._dict(row_number=row_number, df=df)) continue else: value = None - doc[df.fieldname] = value = self.parse_value(value, df) + doc[df.fieldname] = self.parse_value(value, df) return doc parsed_docs = {} @@ -476,6 +480,7 @@ class Importer: # then skip continue + row_number = row[0] column_indexes = get_column_indexes(doctype) values = [row[i] for i in column_indexes] @@ -484,7 +489,7 @@ class Importer: continue docfields = [fields[i] for i in column_indexes] - doc = parse_doc(doctype, docfields, values, row_index) + doc = parse_doc(doctype, docfields, values, row_number) parsed_docs[doctype] = parsed_docs.get(doctype, []) parsed_docs[doctype].append(doc) @@ -499,6 +504,23 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs + if mandatory_fields: + df_by_row_number = {} + for d in mandatory_fields: + df_by_row_number.setdefault(d.row_number, []) + df_by_row_number[d.row_number].append(d.df) + + for row_number, fields in df_by_row_number.items(): + if len(fields) == 1: + warnings.append( + _("Row {0}: {1} is a mandatory field").format(row_number, fields[0].label) + ) + else: + fields_string = ', '.join([df.label for df in fields]) + warnings.append( + _("Row {0}: {1} are mandatory fields").format(row_number, fields_string) + ) + return doc, rows, data[len(rows) :], warnings def get_first_parent_column_index(self, fields): diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index a3b4b40133..a14820ce6f 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -157,11 +157,14 @@ frappe.ui.form.on('Data Import Beta', { show_import_preview(frm, preview_data) { let import_log = JSON.parse(frm.doc.import_log || '[]'); + let warnings = JSON.parse(frm.doc.template_warnings || '[]'); + warnings = warnings.concat(preview_data.warnings || []); if (frm.import_preview) { frm.import_preview.preview_data = preview_data; frm.import_preview.import_log = import_log; frm.import_preview.refresh(); + frm.import_preview.render_warnings(warnings); return; } @@ -171,6 +174,7 @@ frappe.ui.form.on('Data Import Beta', { doctype: frm.doc.reference_doctype, preview_data, import_log, + warnings, events: { remap_column(header_row_index, fieldname) { let template_options = JSON.parse(frm.doc.template_options || '{}'); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index a54ccfbfcb..83e63d2e36 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -15,6 +15,7 @@ "section_break_7", "date_format", "template_options", + "template_warnings", "section_import_preview", "import_preview", "import_log_section", @@ -107,9 +108,17 @@ "default": "Pending", "fieldname": "status", "fieldtype": "Select", + "hidden": 1, "label": "Status", "options": "Pending\nSuccess\nPartial Success", "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "label": "Template Warnings", + "options": "JSON" + }, } ], "hide_toolbar": 1, diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index d007865a44..17dab0692c 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -14,12 +14,13 @@ const SVG_ICONS = { }; frappe.data_import.ImportPreview = class ImportPreview { - constructor({ wrapper, doctype, preview_data, import_log, events = {} }) { + constructor({ wrapper, doctype, preview_data, import_log, warnings, events = {} }) { frappe.import_preview = this; this.wrapper = wrapper; this.doctype = doctype; this.preview_data = preview_data; this.events = events; + this.warnings = warnings; this.import_log = import_log; frappe.model.with_doctype(doctype, () => { @@ -31,7 +32,6 @@ frappe.data_import.ImportPreview = class ImportPreview { this.header_row = this.preview_data.header_row; this.fields = this.preview_data.fields; this.data = this.preview_data.data; - this.warnings = this.preview_data.warnings; this.make_wrapper(); this.prepare_columns(); this.prepare_data(); @@ -73,7 +73,7 @@ frappe.data_import.ImportPreview = class ImportPreview { focusable: false, align: 'left', header_row_index, - width: column_width, + width: df.label === 'Sr. No' ? 60 : column_width, format: (value, row, column, data) => { let html = `
${value}
`; if (df.label === 'Sr. No' && this.is_row_imported(row)) { @@ -123,11 +123,11 @@ frappe.data_import.ImportPreview = class ImportPreview { if (warnings.length > 0) { let warning_html = warnings .map(warning => { - return `
${warning}
`; + return `
  • ${warning}
    • `; }) .join(''); - html = `
    ${warning_html}
    `; + html = `
      ${warning_html}
    `; } this.$warnings.html(html); } From 70faaff8b48c000bc266d687f53763a553032f2b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 16 Sep 2019 01:40:41 +0530 Subject: [PATCH 36/83] fix: Submit after import --- .../core/doctype/data_import/importer_new.py | 5 +++- .../data_import_beta/data_import_beta.js | 30 ++++++++++--------- .../data_import_beta/data_import_beta.json | 8 ++++- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index f4cc8fbe94..a7e1e58a70 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -547,7 +547,10 @@ class Importer: # name shouldn't be set when inserting a new record doc.update({"doctype": self.doctype, "name": None}) new_doc = frappe.get_doc(doc) - return new_doc.insert() + new_doc.insert() + if self.meta.is_submittable and self.data_import.submit_after_import: + new_doc.submit() + return new_doc def update_record(self, doc): id_fieldname = self.get_id_fieldname() diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index a14820ce6f..ded954d18c 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -103,20 +103,7 @@ frappe.ui.form.on('Data Import Beta', { let template_options = JSON.parse(frm.doc.template_options || '{}'); template_options.edited_rows = csv_array; frm.set_value('template_options', JSON.stringify(template_options)); - - frm.save().then(() => { - console.log('import_file ') - frm.call('start_import').then(r => { - let { warnings, missing_link_values } = r.message || {}; - if (warnings) { - frm.import_preview.render_warnings(warnings); - } else if (missing_link_values) { - frm.events.show_missing_link_values(frm, missing_link_values); - } else { - frm.refresh(); - } - }); - }); + frm.save().then(() => frm.call('start_import')); }, download_sample_file(frm) { @@ -125,6 +112,21 @@ frappe.ui.form.on('Data Import Beta', { }); }, + reference_doctype(frm) { + frm.trigger('toggle_submit_after_import'); + }, + + toggle_submit_after_import(frm) { + frm.toggle_display('submit_after_import', false); + let doctype = frm.doc.reference_doctype; + if (doctype) { + frappe.model.with_doctype(doctype, () => { + let meta = frappe.get_meta(doctype); + frm.toggle_display('submit_after_import', meta.is_submittable); + }); + } + }, + import_file(frm) { frm.toggle_display('section_import_preview', frm.doc.import_file); if (!frm.doc.import_file) { diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 83e63d2e36..ca953f23cb 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -12,6 +12,7 @@ "import_file", "column_break_5", "status", + "submit_after_import", "section_break_7", "date_format", "template_options", @@ -119,10 +120,15 @@ "label": "Template Warnings", "options": "JSON" }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import" } ], "hide_toolbar": 1, - "modified": "2019-09-02 12:58:49.053073", + "modified": "2019-09-09 17:53:28.398802", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From 9bd3933f531706712f76097f99f5e291a90763c5 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 16 Sep 2019 01:41:07 +0530 Subject: [PATCH 37/83] fix: Match standard fields in import template --- .../core/doctype/data_import/importer_new.py | 90 +++++++++++++------ .../data_import_beta/data_import_beta.py | 2 + 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index a7e1e58a70..797009084d 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -14,13 +14,6 @@ from frappe.utils.csvutils import read_csv_content from frappe.exceptions import ValidationError, MandatoryError from frappe.model import display_fieldtypes, no_value_fields, table_fields -# set user lang -# set flags: frappe.flags.in_import = True - -# during import -# check empty row -# validate naming - INVALID_VALUES = ["", None] @@ -176,11 +169,11 @@ class Importer: out = {} table_doctypes = [df.options for df in self.meta.get_table_fields()] - doctypes = [self.doctype] + table_doctypes + doctypes = table_doctypes + [self.doctype] for doctype in doctypes: # name field name_key = "ID" if self.doctype == doctype else "ID ({})".format(doctype) - out[name_key] = frappe._dict( + name_df = frappe._dict( { "fieldtype": "Data", "fieldname": "name", @@ -189,13 +182,19 @@ class Importer: "parent": doctype, } ) + out[name_key] = name_df + out["name"] = name_df + # other fields meta = frappe.get_meta(doctype) - for df in meta.fields: - if df.fieldtype not in no_value_fields: + fields = self.get_standard_fields(doctype) + meta.fields + for df in fields: + fieldtype = df.fieldtype or 'Data' + parent = df.parent or self.doctype + if fieldtype not in no_value_fields: # label as key label = ( - df.label if self.doctype == doctype else "{0} ({1})".format(df.label, df.parent) + df.label if self.doctype == doctype else "{0} ({1})".format(df.label, parent) ) out[label] = df # fieldname as key @@ -205,17 +204,6 @@ class Importer: key = "{0}:{1}".format(doctype, df.fieldname) out[key] = df - # name field - out["name"] = frappe._dict( - { - "fieldtype": "Data", - "fieldname": "name", - "label": "ID", - "reqd": self.data_import.import_type == "Update Existing Records", - "parent": self.doctype, - } - ) - # if autoname is based on field # add an entry for "ID (Autoname Field)" autoname = self.meta.autoname @@ -226,9 +214,51 @@ class Importer: out["ID ({})".format(autoname_field.label)] = autoname_field # ID field should also map to the autoname field out["ID"] = autoname_field + out["name"] = autoname_field return out + def get_standard_fields(self, doctype): + meta = frappe.get_meta(doctype) + if meta.istable: + standard_fields = [ + { + 'label': 'Parent', + 'fieldname': 'parent' + }, + { + 'label': 'Parent Type', + 'fieldname': 'parenttype' + }, + { + 'label': 'Parent Field', + 'fieldname': 'parentfield' + }, + { + 'label': 'Row Index', + 'fieldname': 'idx' + } + ] + else: + standard_fields = [ + { + 'label': 'Owner', + 'fieldname': 'owner' + }, + { + 'label': 'Document Status', + 'fieldname': 'docstatus', + 'fieldtype': 'Int' + } + ] + + out = [] + for df in standard_fields: + df = frappe._dict(df) + df.parent = doctype + out.append(df) + return out + def parse_formats_from_first_10_rows(self): """ Returns a list of column descriptors for columns that might need parsing. @@ -271,7 +301,9 @@ class Importer: def parse_date_format(self, value, df): date_format = self.guess_date_format_for_column(df.fieldname) - return datetime.strptime(value, date_format) + if date_format: + return datetime.strptime(value, date_format) + return value def guess_date_format_for_column(self, fieldname): """ Guesses date format for a column by parsing the first 10 values in the column, @@ -293,13 +325,19 @@ class Importer: column_values = map(lambda x: x[column_index], self.data[:PARSE_ROW_COUNT]) column_values = filter(lambda x: bool(x), column_values) date_formats = list(map(lambda x: guess_date_format(x), column_values)) + if not date_formats: + return max_occurred_date_format = max(set(date_formats), key=date_formats.count) - self._guessed_date_formats[fieldname] = max_occurred_date_format return self._guessed_date_formats[fieldname] def import_data(self): + # set user lang for translations + frappe.cache().hdel("lang", frappe.session.user) + frappe.set_user_lang(frappe.session.user) + + # set flag frappe.flags.in_import = True out = self.get_data_for_import_preview() @@ -569,7 +607,7 @@ class Importer: # get mandatory fields with default not set mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] mandatory_fields_count = len(mandatory_fields) - if meta.autoname.lower() == "prompt": + if meta.autoname and meta.autoname.lower() == "prompt": mandatory_fields_count += 1 return mandatory_fields_count == 1 diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index dc22f662b2..21c94f2cd8 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -18,6 +18,7 @@ class DataImportBeta(Document): doc_before_save and doc_before_save.import_file != self.import_file ): self.template_options = "" + self.template_warnings = "" def get_preview_from_template(self): if not self.import_file: @@ -37,6 +38,7 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, + now=True ) def get_importer(self): From 6d192d07665d9c6e3db98229b368fe4df689550c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 17 Sep 2019 16:06:48 +0530 Subject: [PATCH 38/83] fix: Validate values by fieldtype --- .../core/doctype/data_import/importer_new.py | 98 +++++++++---------- frappe/core/doctype/docfield/docfield.py | 5 + 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 797009084d..ae4553624c 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -189,7 +189,7 @@ class Importer: meta = frappe.get_meta(doctype) fields = self.get_standard_fields(doctype) + meta.fields for df in fields: - fieldtype = df.fieldtype or 'Data' + fieldtype = df.fieldtype or "Data" parent = df.parent or self.doctype if fieldtype not in no_value_fields: # label as key @@ -222,34 +222,15 @@ class Importer: meta = frappe.get_meta(doctype) if meta.istable: standard_fields = [ - { - 'label': 'Parent', - 'fieldname': 'parent' - }, - { - 'label': 'Parent Type', - 'fieldname': 'parenttype' - }, - { - 'label': 'Parent Field', - 'fieldname': 'parentfield' - }, - { - 'label': 'Row Index', - 'fieldname': 'idx' - } + {"label": "Parent", "fieldname": "parent"}, + {"label": "Parent Type", "fieldname": "parenttype"}, + {"label": "Parent Field", "fieldname": "parentfield"}, + {"label": "Row Index", "fieldname": "idx"}, ] else: standard_fields = [ - { - 'label': 'Owner', - 'fieldname': 'owner' - }, - { - 'label': 'Document Status', - 'fieldname': 'docstatus', - 'fieldtype': 'Int' - } + {"label": "Owner", "fieldname": "owner"}, + {"label": "Document Status", "fieldname": "docstatus", "fieldtype": "Int"}, ] out = [] @@ -345,12 +326,8 @@ class Importer: data = out["data"] warnings = [] - # validate link field values - missing_link_values = self.get_missing_link_field_values(fields, data) - for d in missing_link_values: - msg = _('The following linked values are missing for the Document Type {0}:').format(frappe.bold(_(d.doctype))) - msg += ' ' + ', '.join(d.missing_values) - warnings.append(msg) + # prepare a map for missing link field values + self.prepare_missing_link_field_values(fields, data) # parse import data payloads = self.get_payloads_for_import(fields, data) @@ -388,7 +365,9 @@ class Importer: total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 - for batch_index, batched_payloads in enumerate(frappe.utils.create_batch(payloads, batch_size)): + for batch_index, batched_payloads in enumerate( + frappe.utils.create_batch(payloads, batch_size) + ): for i, payload in enumerate(batched_payloads): doc = payload.doc row_indexes = [row[0] for row in payload.rows] @@ -416,7 +395,7 @@ class Importer: "docname": doc.name, "success": True, "row_indexes": row_indexes, - "eta": eta + "eta": eta, }, ) import_log.append( @@ -494,6 +473,30 @@ class Importer: def get_column_indexes(doctype): return [i for i, df in enumerate(fields) if df.parent == doctype] + def validate_value(value, df): + local_warnings = [] + + if df.fieldtype == "Select" and value not in df.get_select_options(): + options_string = ", ".join([frappe.bold(d) for d in df.get_select_options()]) + msg = _("Row {0}, Column {1}: Value must be one of {2}").format( + row_number, df.label, options_string + ) + local_warnings.append(msg) + + elif df.fieldtype == "Link": + missing_link_values = self.get_missing_link_field_values(df.options) + if value in missing_link_values: + msg = _("Row {0}, Column {1}: Value {2} missing for Document Type {3}").format( + row_number, df.label, frappe.bold(value), frappe.bold(df.options) + ) + local_warnings.append(msg) + + if local_warnings: + warnings.extend(local_warnings) + return False + + return True + def parse_doc(doctype, docfields, values, row_index): doc = {} for index, (df, value) in enumerate(zip(docfields, values)): @@ -507,7 +510,8 @@ class Importer: else: value = None - doc[df.fieldname] = self.parse_value(value, df) + if validate_value(value, df): + doc[df.fieldname] = self.parse_value(value, df) return doc parsed_docs = {} @@ -554,7 +558,7 @@ class Importer: _("Row {0}: {1} is a mandatory field").format(row_number, fields[0].label) ) else: - fields_string = ', '.join([df.label for df in fields]) + fields_string = ", ".join([df.label for df in fields]) warnings.append( _("Row {0}: {1} are mandatory fields").format(row_number, fields_string) ) @@ -593,13 +597,16 @@ class Importer: def update_record(self, doc): id_fieldname = self.get_id_fieldname() id_value = doc[id_fieldname] - existing_doc = frappe.get_doc(self.doctype, { id_fieldname: id_value }) + existing_doc = frappe.get_doc(self.doctype, {id_fieldname: id_value}) existing_doc.flags.via_data_import = self.data_import.name existing_doc.update(doc) existing_doc.save() return existing_doc - def get_missing_link_field_values(self, fields, data): + def get_missing_link_field_values(self, doctype): + return self.missing_link_values.get(doctype, []) + + def prepare_missing_link_field_values(self, fields, data): link_column_indexes = [i for i, df in enumerate(fields) if df.fieldtype == "Link"] def has_one_mandatory_field(doctype): @@ -611,7 +618,7 @@ class Importer: mandatory_fields_count += 1 return mandatory_fields_count == 1 - missing_values_payload = [] + self.missing_link_values = {} for index in link_column_indexes: df = fields[index] column_values = [row[index] for row in data] @@ -619,16 +626,7 @@ class Importer: doctype = df.options missing_values = [value for value in values if not frappe.db.exists(doctype, value)] - if missing_values: - missing_values_payload.append( - frappe._dict( - doctype=doctype, - missing_values=missing_values, - has_one_mandatory_field=has_one_mandatory_field(doctype), - ) - ) - - return missing_values_payload + self.missing_link_values[doctype] = missing_values def get_id_fieldname(self): autoname = self.meta.autoname @@ -637,7 +635,7 @@ class Importer: autoname_field = self.meta.get_field(fieldname) if autoname_field: return autoname_field.fieldname - return 'name' + return "name" def get_eta(self, current, total, processing_time): remaining = total - current diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index bce92557ba..b6e2d9b67d 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -26,3 +26,8 @@ class DocField(Document): }, 'options') return link_doctype + + def get_select_options(self): + if self.fieldtype == 'Select': + options = self.options or '' + return [d for d in options.split('\n') if d] From 2b42838917ccd3a4546b0de7827a4c4a15566a59 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 17 Sep 2019 16:47:09 +0530 Subject: [PATCH 39/83] fix: Export errored rows into a separate file --- .../core/doctype/data_import/importer_new.py | 25 +++++++++++++++++++ .../data_import_beta/data_import_beta.js | 7 ++++++ .../data_import_beta/data_import_beta.py | 10 +++++++- .../js/frappe/data_import/import_preview.js | 13 ++++------ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index ae4553624c..ce6b284860 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -603,6 +603,31 @@ class Importer: existing_doc.save() return existing_doc + def export_errored_rows(self): + from frappe.utils.csvutils import build_csv_response + + if not self.data_import: + return + + import_log = frappe.parse_json(self.data_import.import_log or "[]") + failures = [l for l in import_log if l.get("success") == False] + row_indexes = [] + for f in failures: + row_indexes.extend(f.get("row_indexes", [])) + + # de duplicate + row_indexes = list(set(row_indexes)) + row_indexes.sort() + + out = self.get_data_for_import_preview() + header_row = out["header_row"] + data = out["data"] + + rows = [header_row] + rows += [row[1: ] for row in data if row[0] in row_indexes] + + build_csv_response(rows, self.doctype) + def get_missing_link_field_values(self, doctype): return self.missing_link_values.get(doctype, []) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index ded954d18c..38d45d7b18 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -57,6 +57,7 @@ frappe.ui.form.on('Data Import Beta', { frm.page.hide_icon_group(); frm.trigger('import_file'); frm.trigger('show_import_log'); + frm.trigger('toggle_submit_after_import'); if (frm.doc.status === 'Success') { // set form as readonly @@ -214,6 +215,12 @@ frappe.ui.form.on('Data Import Beta', { frm.save().then(() => { frm.trigger('import_file'); }); + }, + + export_errored_rows() { + open_url_post('/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_errored_template', { + data_import_name: frm.doc.name + }) } } }); diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 21c94f2cd8..d6c414b6c7 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -38,7 +38,7 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, - now=True + now=True, ) def get_importer(self): @@ -66,6 +66,9 @@ class DataImportBeta(Document): docs.append(new_doc.insert()) return docs + def export_errored_rows(self): + return self.get_importer().export_errored_rows() + def start_import(data_import): """This method runs in background job""" @@ -99,3 +102,8 @@ def download_template( file_type=file_type, ) e.build_csv_response() + +@frappe.whitelist() +def download_errored_template(data_import_name): + data_import = frappe.get_doc('Data Import Beta', data_import_name) + data_import.export_errored_rows() diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 17dab0692c..3677b49bcc 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -43,14 +43,12 @@ frappe.data_import.ImportPreview = class ImportPreview { make_wrapper() { this.wrapper.html(`
    -
    +
    -
    `); @@ -220,9 +218,8 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - add_row() { - this.data.push([]); - this.datatable.refresh(this.data); + export_errored_rows() { + this.events.export_errored_rows(); } remap_column(col) { From 5b193f5672eafd69cd9e2498df7cd2a3b82d7f4f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 17 Sep 2019 18:22:19 +0530 Subject: [PATCH 40/83] fix: Dont load preview for more than 500 rows --- .../core/doctype/data_import/importer_new.py | 18 ++++++++--- .../js/frappe/data_import/import_preview.js | 31 ++++++++++++++++--- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index ce6b284860..5d9b972eb4 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -15,6 +15,7 @@ from frappe.exceptions import ValidationError, MandatoryError from frappe.model import display_fieldtypes, no_value_fields, table_fields INVALID_VALUES = ["", None] +MAX_ROWS_IN_PREVIEW = 500 class Importer: @@ -106,6 +107,13 @@ class Importer: self.header_row = header_row def get_data_for_import_preview(self): + out = self.get_parsed_data_from_template() + if len(out.data) > MAX_ROWS_IN_PREVIEW: + out.data = [] + out.max_rows_exceeded = True + return out + + def get_parsed_data_from_template(self): fields, fields_warnings = self.parse_fields_from_header_row() formats, formats_warnings = self.parse_formats_from_first_10_rows() fields, data = self.add_serial_no_column(fields, self.data) @@ -115,7 +123,9 @@ class Importer: warnings = fields_warnings + formats_warnings - return dict(header_row=self.header_row, fields=fields, data=data, warnings=warnings) + return frappe._dict( + header_row=self.header_row, fields=fields, data=data, warnings=warnings + ) def parse_fields_from_header_row(self): remap_column = self.template_options.remap_column @@ -321,7 +331,7 @@ class Importer: # set flag frappe.flags.in_import = True - out = self.get_data_for_import_preview() + out = self.get_parsed_data_from_template() fields = out["fields"] data = out["data"] warnings = [] @@ -619,12 +629,12 @@ class Importer: row_indexes = list(set(row_indexes)) row_indexes.sort() - out = self.get_data_for_import_preview() + out = self.get_parsed_data_from_template() header_row = out["header_row"] data = out["data"] rows = [header_row] - rows += [row[1: ] for row in data if row[0] in row_indexes] + rows += [row[1:] for row in data if row[0] in row_indexes] build_csv_response(rows, self.doctype) diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 3677b49bcc..38100da3ce 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -32,12 +32,14 @@ frappe.data_import.ImportPreview = class ImportPreview { this.header_row = this.preview_data.header_row; this.fields = this.preview_data.fields; this.data = this.preview_data.data; + this.max_rows_exceeded = this.preview_data.max_rows_exceeded; this.make_wrapper(); this.prepare_columns(); this.prepare_data(); this.render_warnings(this.warnings); this.render_datatable(); this.setup_styles(); + this.add_actions(); } make_wrapper() { @@ -45,11 +47,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
    -
    - -
    +
    `); frappe.utils.bind_actions_with_class(this.wrapper, this); @@ -135,6 +133,11 @@ frappe.data_import.ImportPreview = class ImportPreview { this.datatable.destroy(); } + let no_data_message = this.max_rows_exceeded + ? __('Cannot load preview for more than 500 rows. You can still remap or skip columns.') + : __('No Data'); + no_data_message = `${no_data_message}`; + this.datatable = new DataTable(this.$table_preview.get(0), { data: this.data, columns: this.columns, @@ -143,6 +146,7 @@ frappe.data_import.ImportPreview = class ImportPreview { serialNoColumn: false, checkboxColumn: false, pasteFromClipboard: true, + noDataMessage: no_data_message, headerDropdown: [ { label: __('Remap Column'), @@ -158,6 +162,12 @@ frappe.data_import.ImportPreview = class ImportPreview { } }); + if (this.data.length === 0) { + this.datatable.style.setStyle('.dt-scrollable', { + height: 'auto' + }); + } + this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { display: 'none' }); @@ -218,6 +228,17 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } + add_actions() { + let failures = this.import_log.filter(log => !log.success); + if (failures.length > 0) { + this.wrapper.find('.table-actions').append( + ` + `); + } + } + export_errored_rows() { this.events.export_errored_rows(); } From 07eedad5bb15f834b6e4208f66c97e62f210a681 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 17 Sep 2019 18:56:42 +0530 Subject: [PATCH 41/83] fix: Parse .xls and .xlsx files in import --- frappe/core/doctype/data_import/importer_new.py | 17 ++++++++++++++--- frappe/utils/xlsxutils.py | 11 +++++++++++ requirements.txt | 1 + 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 5d9b972eb4..7b63b6b0ad 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -11,6 +11,10 @@ from datetime import datetime from frappe import _ from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT from frappe.utils.csvutils import read_csv_content +from frappe.utils.xlsxutils import ( + read_xlsx_file_from_attached_file, + read_xls_file_from_attached_file, +) from frappe.exceptions import ValidationError, MandatoryError from frappe.model import display_fieldtypes, no_value_fields, table_fields @@ -45,11 +49,12 @@ class Importer: if self.data_import: file_doc = frappe.get_doc("File", {"file_url": self.data_import.import_file}) content = file_doc.get_content() + extension = file_doc.file_name.split(".")[1] if file_path: self.read_file(file_path) elif content: - self.read_content(content) + self.read_content(content, extension) self.remove_empty_rows_and_columns() def read_file(self, file_path): @@ -64,8 +69,14 @@ class Importer: self.header_row = data[0] self.data = data[1:] - def read_content(self, content): - data = read_csv_content(content) + def read_content(self, content, extension): + if extension == "csv": + data = read_csv_content(content) + elif extension == "xlsx": + data = read_xlsx_file_from_attached_file(fcontent=content) + elif extension == "xls": + data = read_xls_file_from_attached_file(content) + self.header_row = data[0] self.data = data[1:] diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 82d631af4c..67df187165 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import frappe import openpyxl +import xlrd import re from openpyxl.styles import Font from openpyxl import load_workbook @@ -95,3 +96,13 @@ def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=Non tmp_list.append(cell.value) rows.append(tmp_list) return rows + +def read_xls_file_from_attached_file(content): + book = xlrd.open_workbook(file_contents=content) + sheets = book.sheets() + sheet = sheets[0] + rows = [] + for i in range(sheet.nrows): + rows.append(sheet.row_values(i)) + rows = rows[1:] + return rows diff --git a/requirements.txt b/requirements.txt index da0c87d655..c11b023c54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -64,3 +64,4 @@ sqlparse==0.2.4 Pygments==2.2.0 frontmatter PyYAML==3.13 +xlrd From d07ac06e5c76d979877fccc4a0def1e0fae5a5c7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 18 Sep 2019 14:39:54 +0530 Subject: [PATCH 42/83] fix: Export to Excel --- .../core/doctype/data_import/exporter_new.py | 18 ++++++++++++++++-- .../data_import_beta/data_import_beta.py | 2 +- frappe/utils/xlsxutils.py | 7 +++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index c4f98d25ba..d4224b08d4 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -6,6 +6,7 @@ import frappe from frappe import _ from frappe.model import display_fieldtypes, no_value_fields, table_fields from frappe.utils.csvutils import build_csv_response +from frappe.utils.xlsxutils import build_xlsx_response from .importer_new import INVALID_VALUES @@ -30,6 +31,7 @@ class Exporter: self.meta = frappe.get_meta(doctype) self.export_fields = export_fields self.export_filters = export_filters + self.file_type = file_type # this will contain the csv content self.csv_array = [] @@ -240,11 +242,23 @@ class Exporter: def get_csv_array(self): return self.csv_array - def build_csv_response(self): + def get_csv_array_for_export(self): csv_array = self.csv_array if not self.data: # add 2 empty rows csv_array += [[]] * 2 - build_csv_response(csv_array, self.doctype) + return csv_array + + def build_response(self): + if self.file_type == 'CSV': + self.build_csv_response() + elif self.file_type == 'Excel': + self.build_xlsx_response() + + def build_csv_response(self): + build_csv_response(self.get_csv_array_for_export(), self.doctype) + + def build_xlsx_response(self): + build_xlsx_response(self.get_csv_array_for_export(), self.doctype) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index d6c414b6c7..734c00d1f1 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -101,7 +101,7 @@ def download_template( export_filters=export_filters, file_type=file_type, ) - e.build_csv_response() + e.build_response() @frappe.whitelist() def download_errored_template(data_import_name): diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 67df187165..9545722e9a 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -106,3 +106,10 @@ def read_xls_file_from_attached_file(content): rows.append(sheet.row_values(i)) rows = rows[1:] return rows + +def build_xlsx_response(data, filename): + xlsx_file = make_xlsx(data, filename) + # write out response as a xlsx type + frappe.response['filename'] = filename + '.xlsx' + frappe.response['filecontent'] = xlsx_file.getvalue() + frappe.response['type'] = 'binary' From e09b45ed81f10defb47114ef1b2e5bf9bf336bde Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 14:03:39 +0530 Subject: [PATCH 43/83] fix: List view formatting --- .../data_import_beta/data_import_beta_list.js | 19 +++++++++++++++++++ frappe/public/js/frappe/list/list_view.js | 15 ++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 frappe/core/doctype/data_import_beta/data_import_beta_list.js diff --git a/frappe/core/doctype/data_import_beta/data_import_beta_list.js b/frappe/core/doctype/data_import_beta/data_import_beta_list.js new file mode 100644 index 0000000000..b7674b432d --- /dev/null +++ b/frappe/core/doctype/data_import_beta/data_import_beta_list.js @@ -0,0 +1,19 @@ +frappe.listview_settings['Data Import Beta'] = { + get_indicator: function(doc) { + var colors = { + "Pending": "orange", + "Partial Success": "orange", + "Success": "green", + } + return [__(doc.status), colors[doc.status], "status,=," + doc.status]; + }, + formatters: { + import_type(value) { + return { + 'Insert New Records': __('Insert'), + 'Update Existing Records': __('Update') + }[value]; + } + }, + hide_name_column: true +}; diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 5c4a23962b..19079e1c24 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -525,13 +525,8 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { const fieldname = df.fieldname; const value = doc[fieldname] || ''; - // listview_setting formatter - const formatters = this.settings.formatters; - const format = () => { - if (formatters && formatters[fieldname]) { - return formatters[fieldname](value, df, doc); - } else if (df.fieldtype === 'Code') { + if (df.fieldtype === 'Code') { return value; } else if (df.fieldtype === 'Percent') { return `
    @@ -547,7 +542,13 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { const field_html = () => { let html; - const _value = typeof value === 'string' ? frappe.utils.escape_html(value) : value; + let _value; + // listview_setting formatter + if (this.settings.formatters && this.settings.formatters[fieldname]) { + _value = this.settings.formatters[fieldname](value, df, doc); + } else { + _value = typeof value === 'string' ? frappe.utils.escape_html(value) : value; + } if (df.fieldtype === 'Image') { html = df.options ? From 1bfd7d3731ef54582bbc8a1977010233cfdb79f0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 14:12:27 +0530 Subject: [PATCH 44/83] fix: Column Mapping - Remove double headers - Indicate match with indicators - Column Mapping Dialog - Handle untitled columns --- .../core/doctype/data_import/importer_new.py | 61 +++++--- .../data_import_beta/data_import_beta.js | 23 ++- .../data_import/custom_column_manager.js | 38 ----- .../js/frappe/data_import/import_preview.js | 139 +++++++++++------- 4 files changed, 143 insertions(+), 118 deletions(-) delete mode 100644 frappe/public/js/frappe/data_import/custom_column_manager.js diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 7b63b6b0ad..e2c690ebed 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -9,6 +9,7 @@ import timeit import frappe from datetime import datetime from frappe import _ +from frappe.core.doctype.docfield.docfield import DocField from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT from frappe.utils.csvutils import read_csv_content from frappe.utils.xlsxutils import ( @@ -119,6 +120,23 @@ class Importer: def get_data_for_import_preview(self): out = self.get_parsed_data_from_template() + + # prepare fields + fields = [] + for df in out.fields: + header_title = df.header_title + skip_import = df.skip_import + if isinstance(df, DocField): + field = df.as_dict() + else: + field = df + field.update({ + 'header_title': header_title, + 'skip_import': skip_import + }) + fields.append(field) + out.fields = fields + if len(out.data) > MAX_ROWS_IN_PREVIEW: out.data = [] out.max_rows_exceeded = True @@ -146,30 +164,37 @@ class Importer: df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() - for i, value in enumerate(self.header_row): + for i, header_title in enumerate(self.header_row): header_row_index = str(i) if remap_column.get(header_row_index): - column_name = value - value = remap_column.get(header_row_index) + fieldname = remap_column.get(header_row_index) + df = df_by_labels_and_fieldnames.get(fieldname) warnings.append( _("Column {0}: Mapping column {1} to field {2}").format( - i, frappe.bold(column_name), frappe.bold(value) + i, frappe.bold(header_title or 'Untitled Column'), frappe.bold(df.label) ) ) + else: + df = df_by_labels_and_fieldnames.get(header_title) - field = df_by_labels_and_fieldnames.get(value) - if not field or i in skip_import: - field = frappe._dict({"label": value, "skip_import": True}) - if value and i not in skip_import: - warnings.append( - _("Column {0}: Cannot match column {1} with any field").format( - i, frappe.bold(value) - ) + if not df: + field = frappe._dict(header_title=header_title, skip_import=True) + else: + field = df + field.header_title = header_title + field.skip_import = False + + if i in skip_import: + field.skip_import = True + warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(header_title))) + elif header_title and not df: + warnings.append( + _("Column {0}: Cannot match column {1} with any field").format( + i, frappe.bold(header_title) ) - elif i in skip_import: - warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(value))) - else: - warnings.append(_("Column {0}: Skipping untitled column").format(i)) + ) + elif not header_title and not df: + warnings.append(_("Column {0}: Skipping untitled column").format(i)) fields.append(field) return fields, warnings @@ -442,7 +467,9 @@ class Importer: # set status failures = [l for l in import_log if l.get("success") == False] - if len(failures) > 0: + if len(failures) == total_payload_count: + status = "Pending" + elif len(failures) > 0: status = "Partial Success" else: status = "Success" diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 38d45d7b18..26ef2f812c 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -59,10 +59,13 @@ frappe.ui.form.on('Data Import Beta', { frm.trigger('show_import_log'); frm.trigger('toggle_submit_after_import'); - if (frm.doc.status === 'Success') { + if (frm.doc.import_log && frm.doc.import_log !== '[]') { // set form as readonly frm.fields.forEach(f => f.df.read_only = 1); frm.disable_save(); + } + + if (frm.doc.status === 'Success') { frm.events.show_success_message(frm); } else { if (!frm.is_new() && frm.doc.import_file) { @@ -144,8 +147,6 @@ frappe.ui.form.on('Data Import Beta', { .call({ doc: frm.doc, method: 'get_preview_from_template', - freeze: true, - freeze_message: __('Preparing Preview...'), error_handlers: { TimestampMismatchError() { // ignore this error @@ -179,23 +180,19 @@ frappe.ui.form.on('Data Import Beta', { import_log, warnings, events: { - remap_column(header_row_index, fieldname) { + remap_column(changed_map) { let template_options = JSON.parse(frm.doc.template_options || '{}'); template_options.remap_column = template_options.remap_column || {}; - template_options.remap_column[header_row_index] = fieldname; + Object.assign(template_options.remap_column, changed_map); + // if the column is remapped, remove it from skip_import - if ( - template_options.skip_import && - template_options.skip_import.includes(header_row_index) - ) { + if (template_options.skip_import) { template_options.skip_import = template_options.skip_import.filter( - d => d !== header_row_index + d => !Object.keys(template_options.remap_column).includes(cstr(d)) ); } frm.set_value('template_options', JSON.stringify(template_options)); - frm.save().then(() => { - frm.trigger('import_file'); - }); + frm.save().then(() => frm.trigger('import_file')); }, skip_import(header_row_index) { diff --git a/frappe/public/js/frappe/data_import/custom_column_manager.js b/frappe/public/js/frappe/data_import/custom_column_manager.js deleted file mode 100644 index 2e829d349c..0000000000 --- a/frappe/public/js/frappe/data_import/custom_column_manager.js +++ /dev/null @@ -1,38 +0,0 @@ -import ColumnManager from 'frappe-datatable/src/columnmanager'; - -export default function(header_row) { - return class CustomColumnManager extends ColumnManager { - getHeaderHTML(columns) { - let html = super.getHeaderHTML(columns); - - let header_row_columns = [ - { - id: '_checkbox', - colIndex: 0, - format: () => '' - } - // { - // id: 'Sr. No', - // colIndex: 1, - // format: () => '' - // } - ].concat( - ...header_row.map((col, i) => { - return { - id: col, - name: col, - align: 'left', - dropdown: false, - content: col, - colIndex: i + 1 - }; - }) - ); - - let header_row_html = this.rowmanager.getRowHTML(header_row_columns, { - rowIndex: 'header-row' - }); - return header_row_html + html; - } - }; -} diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 38100da3ce..993f163a0c 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -1,5 +1,4 @@ import DataTable from 'frappe-datatable'; -import get_custom_column_manager from './custom_column_manager'; import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); @@ -47,7 +46,11 @@ frappe.data_import.ImportPreview = class ImportPreview {
    -
    +
    + +
    `); frappe.utils.bind_actions_with_class(this.wrapper, this); @@ -61,18 +64,23 @@ frappe.data_import.ImportPreview = class ImportPreview { this.columns = this.fields.map((df, i) => { let header_row_index = i - 1; if (df.skip_import) { + let is_sr = df.label === 'Sr. No'; + let column_title = is_sr + ? df.label + : `${df.header_title || `${__('Untitled Column')}`}`; return { id: frappe.utils.get_random(6), name: df.label, + content: column_title, skip_import: true, editable: false, focusable: false, align: 'left', header_row_index, - width: df.label === 'Sr. No' ? 60 : column_width, + width: is_sr ? 60 : column_width, format: (value, row, column, data) => { let html = `
    ${value}
    `; - if (df.label === 'Sr. No' && this.is_row_imported(row)) { + if (is_sr && this.is_row_imported(row)) { html = `
    ${SVG_ICONS['checkbox-circle-line'] + html}
    @@ -94,6 +102,7 @@ frappe.data_import.ImportPreview = class ImportPreview { return { id: df.fieldname, name: column_title, + content: `${df.header_title || df.label}`, df: df, editable: true, align: 'left', @@ -146,20 +155,7 @@ frappe.data_import.ImportPreview = class ImportPreview { serialNoColumn: false, checkboxColumn: false, pasteFromClipboard: true, - noDataMessage: no_data_message, - headerDropdown: [ - { - label: __('Remap Column'), - action: col => this.remap_column(col) - }, - { - label: __('Skip Import'), - action: col => this.skip_import(col) - } - ], - overrideComponents: { - ColumnManager: get_custom_column_manager(this.header_row) - } + noDataMessage: no_data_message }); if (this.data.length === 0) { @@ -168,7 +164,7 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', { + this.datatable.style.setStyle('.dt-dropdown', { display: 'none' }); } @@ -180,35 +176,6 @@ frappe.data_import.ImportPreview = class ImportPreview { } setup_styles() { - let columns = this.datatable.getColumns(); - columns.forEach(col => { - let class_name = [ - `.dt-header .dt-cell--col-${col.colIndex}`, - `.dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle` - ].join(','); - - if (!col.skip_import && col.df) { - this.datatable.style.setStyle(class_name, { - backgroundColor: frappe.ui.color.get_color_shade( - 'green', - 'extra-light' - ), - color: frappe.ui.color.get_color_shade('green', 'dark') - }); - } - if (col.skip_import && col.name !== 'Sr. No') { - this.datatable.style.setStyle(class_name, { - backgroundColor: frappe.ui.color.get_color_shade( - 'orange', - 'extra-light' - ), - color: frappe.ui.color.get_color_shade('orange', 'dark') - }); - this.datatable.style.setStyle(`.dt-cell--col-${col.colIndex}`, { - backgroundColor: frappe.ui.color.get_color_shade('white', 'light') - }); - } - }); // import success checkbox this.datatable.style.setStyle(`svg.import-success`, { width: '16px', @@ -232,8 +199,8 @@ frappe.data_import.ImportPreview = class ImportPreview { let failures = this.import_log.filter(log => !log.success); if (failures.length > 0) { this.wrapper.find('.table-actions').append( - ` `); } @@ -243,6 +210,78 @@ frappe.data_import.ImportPreview = class ImportPreview { this.events.export_errored_rows(); } + show_column_mapper() { + let column_picker_fields = new ColumnPickerFields({ + doctype: this.doctype + }); + let changed = []; + let fields = this.fields.map((df, i) => { + if (df.label === 'Sr. No') return []; + + let fieldname; + if (df.skip_import) { + fieldname = null; + } else { + fieldname = df.parent === this.doctype + ? df.fieldname + : `${df.parent}:${df.fieldname}`; + } + return [ + { + label: __('Column {0}', [i]), + fieldtype: 'Data', + default: df.header_title, + fieldname: `Column ${i}`, + read_only: 1 + }, + { + fieldtype: 'Button', + label: 'Skip Column', + fieldname: 'skip_' + i, + click: () => { + let header_row_index = i - 1; + this.events.skip_import(header_row_index); + } + }, + { + fieldtype: 'Column Break' + }, + { + fieldtype: 'Autocomplete', + fieldname: i, + label: __('Select field'), + max_items: Infinity, + options: column_picker_fields.get_fields_as_options(), + default: fieldname, + change() { + changed.push(i); + } + }, + { + fieldtype: 'Section Break' + } + ]; + }); + // flatten the array + fields = fields.reduce((acc, curr) => [...acc, ...curr]); + let dialog = new frappe.ui.Dialog({ + title: __('Column Mapper'), + fields, + primary_action: (values) => { + let changed_map = {}; + changed.map(i => { + let header_row_index = i - 1; + changed_map[header_row_index] = values[i]; + }); + if (changed.length > 0) { + this.events.remap_column(changed_map); + } + dialog.hide(); + } + }); + dialog.show(); + } + remap_column(col) { let column_picker_fields = new ColumnPickerFields({ doctype: this.doctype From 91554f43df39d6148f7ea98082744ba3a555a001 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 14:30:09 +0530 Subject: [PATCH 45/83] fix: Show successful imports in Import Log --- .../data_import_beta/data_import_beta.js | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 26ef2f812c..65eeb7fc87 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -226,27 +226,30 @@ frappe.ui.form.on('Data Import Beta', { show_import_log(frm) { let import_log = JSON.parse(frm.doc.import_log || '[]'); - let failures = import_log.filter(log => !log.success); + let logs = import_log; frm.toggle_display('import_log', false); - frm.toggle_display('import_log_section', failures.length > 0); + frm.toggle_display('import_log_section', logs.length > 0); - if (failures.length === 0) { + if (logs.length === 0) { frm.get_field('import_log_preview').$wrapper.empty(); return; } - let rows = failures + let rows = logs .map(log => { - let messages = log.messages.map(JSON.parse).map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `

    ${m.message}

    ` : ''; - return title + message; - }).join(''); - let id = frappe.dom.get_unique_id(); - return ` - ${log.row_indexes.join(', ')} - - ${messages} + let html; + if (log.success) { + html = __('Successfully imported {0}', [ + `${frappe.utils.get_form_link(frm.doc.doctype, log.docname, true)}` + ]); + } else { + let messages = log.messages.map(JSON.parse).map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `

    ${m.message}

    ` : ''; + return title + message; + }).join(''); + let id = frappe.dom.get_unique_id(); + html = `${messages} @@ -254,17 +257,20 @@ frappe.ui.form.on('Data Import Beta', { 
    ${log.exception}
    -
    - + ` + } + return ` + ${log.row_indexes.join(', ')} + ${html} `; }) .join(''); frm.get_field('import_log_preview').$wrapper.html(` - + - + ${rows}
    ${__('Row Number')}${__('Error Message')}${__('Message')}
    From cd3ee902621c4a075e50f33ab77f65d3b536ff5e Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 15:43:13 +0530 Subject: [PATCH 46/83] fix(Form API): scroll_to_field --- frappe/public/js/frappe/form/form.js | 22 ++++++++++++++++++++++ frappe/public/js/frappe/form/toolbar.js | 20 +------------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 4052d2debb..dfa00d099f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1391,6 +1391,28 @@ frappe.ui.form.Form = class FrappeForm { } return sum; } + + scroll_to_field(fieldname) { + let field = this.get_field(fieldname); + if (!field) return; + + let $el = field.$wrapper; + + // uncollapse section + if (field.section.is_collapsed()) { + field.section.collapse(false); + } + + // scroll to input + frappe.utils.scroll_to($el); + + // highlight input + $el.addClass('has-error'); + setTimeout(() => { + $el.removeClass('has-error'); + $el.find('input, select, textarea').focus(); + }, 1000); + } }; frappe.validated = 0; diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 46be4d7d09..7f5882f517 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -411,25 +411,7 @@ frappe.ui.form.Toolbar = Class.extend({ primary_action_label: __('Go'), primary_action: ({ fieldname }) => { dialog.hide(); - let field = this.frm.get_field(fieldname); - if (!field) return; - - let $el = field.$wrapper; - - // uncollapse section - if (field.section.is_collapsed()) { - field.section.collapse(false); - } - - // scroll to input - frappe.utils.scroll_to($el); - - // highlight input - $el.addClass('has-error'); - setTimeout(() => { - $el.removeClass('has-error'); - $el.find('input, select, textarea').focus(); - }, 1000); + this.frm.scroll_to_field(fieldname); } }); From fbfa0781475a484bcbd0845832764fbedc08fb9c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 16:30:19 +0530 Subject: [PATCH 47/83] fix: as_dict - convert_dates_to_str should work for non df fields --- frappe/model/base_document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 3d5b002c75..1bde485ac4 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -233,8 +233,8 @@ class BaseDocument(object): if isinstance(d[fieldname], list) and df.fieldtype not in table_fields: frappe.throw(_('Value for {0} cannot be a list').format(_(df.label))) - if convert_dates_to_str and isinstance(d[fieldname], (datetime.datetime, datetime.time, datetime.timedelta)): - d[fieldname] = str(d[fieldname]) + if convert_dates_to_str and isinstance(d[fieldname], (datetime.datetime, datetime.time, datetime.timedelta)): + d[fieldname] = str(d[fieldname]) if d[fieldname] == None and ignore_nulls: del d[fieldname] From 626c2b4070251cf503ac7fea0a7697045e2663bf Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 16:30:32 +0530 Subject: [PATCH 48/83] fix: Structured warnings grouped by rows --- .../core/doctype/data_import/importer_new.py | 60 ++++++---- .../data_import_beta/data_import_beta.js | 109 +++++++++++++++--- .../data_import_beta/data_import_beta.json | 14 ++- .../js/frappe/data_import/import_preview.js | 78 +++++++------ 4 files changed, 185 insertions(+), 76 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index e2c690ebed..46d0b1df0f 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -130,10 +130,7 @@ class Importer: field = df.as_dict() else: field = df - field.update({ - 'header_title': header_title, - 'skip_import': skip_import - }) + field.update({"header_title": header_title, "skip_import": skip_import}) fields.append(field) out.fields = fields @@ -166,13 +163,17 @@ class Importer: for i, header_title in enumerate(self.header_row): header_row_index = str(i) + column_number = str(i + 1) if remap_column.get(header_row_index): fieldname = remap_column.get(header_row_index) df = df_by_labels_and_fieldnames.get(fieldname) warnings.append( - _("Column {0}: Mapping column {1} to field {2}").format( - i, frappe.bold(header_title or 'Untitled Column'), frappe.bold(df.label) - ) + { + "col": column_number, + "message": _("Mapping column {0} to field {1}").format( + frappe.bold(header_title or "Untitled Column"), frappe.bold(df.label) + ), + } ) else: df = df_by_labels_and_fieldnames.get(header_title) @@ -186,15 +187,20 @@ class Importer: if i in skip_import: field.skip_import = True - warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(header_title))) + warnings.append( + {"col": column_number, "message": _("Skipping column {0}").format(frappe.bold(header_title))} + ) elif header_title and not df: warnings.append( - _("Column {0}: Cannot match column {1} with any field").format( - i, frappe.bold(header_title) - ) + { + "col": column_number, + "message": _("Cannot match column {0} with any field").format( + frappe.bold(header_title) + ), + } ) elif not header_title and not df: - warnings.append(_("Column {0}: Skipping untitled column").format(i)) + warnings.append({"col": column_number, "message": _("Skipping Untitled Column")}) fields.append(field) return fields, warnings @@ -522,25 +528,27 @@ class Importer: return [i for i, df in enumerate(fields) if df.parent == doctype] def validate_value(value, df): - local_warnings = [] + validate_warnings = [] if df.fieldtype == "Select" and value not in df.get_select_options(): options_string = ", ".join([frappe.bold(d) for d in df.get_select_options()]) - msg = _("Row {0}, Column {1}: Value must be one of {2}").format( - row_number, df.label, options_string + msg = _("Value must be one of {0}").format(options_string) + validate_warnings.append( + {"row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg} ) - local_warnings.append(msg) elif df.fieldtype == "Link": missing_link_values = self.get_missing_link_field_values(df.options) if value in missing_link_values: - msg = _("Row {0}, Column {1}: Value {2} missing for Document Type {3}").format( - row_number, df.label, frappe.bold(value), frappe.bold(df.options) + msg = _("Value {0} missing for Document Type {1}").format( + frappe.bold(value), frappe.bold(df.options) + ) + validate_warnings.append( + {"row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg} ) - local_warnings.append(msg) - if local_warnings: - warnings.extend(local_warnings) + if validate_warnings: + warnings.extend(validate_warnings) return False return True @@ -603,12 +611,18 @@ class Importer: for row_number, fields in df_by_row_number.items(): if len(fields) == 1: warnings.append( - _("Row {0}: {1} is a mandatory field").format(row_number, fields[0].label) + { + "row": row_number, + "message": _("{0} is a mandatory field").format(fields[0].label), + } ) else: fields_string = ", ".join([df.label for df in fields]) warnings.append( - _("Row {0}: {1} are mandatory fields").format(row_number, fields_string) + { + "row": row_number, + "message": _("{0} are mandatory fields").format(fields_string), + } ) return doc, rows, data[len(rows) :], warnings diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 65eeb7fc87..86cb540745 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -43,7 +43,7 @@ frappe.ui.form.on('Data Import Beta', { filters: { allow_import: 1 } - } + }; }); frm.get_field('import_file').df.options = { @@ -57,11 +57,12 @@ frappe.ui.form.on('Data Import Beta', { frm.page.hide_icon_group(); frm.trigger('import_file'); frm.trigger('show_import_log'); + frm.trigger('show_import_warnings'); frm.trigger('toggle_submit_after_import'); if (frm.doc.import_log && frm.doc.import_log !== '[]') { // set form as readonly - frm.fields.forEach(f => f.df.read_only = 1); + frm.fields.forEach(f => (f.df.read_only = 1)); frm.disable_save(); } @@ -70,14 +71,15 @@ frappe.ui.form.on('Data Import Beta', { } else { if (!frm.is_new() && frm.doc.import_file) { let label = frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); - frm.page.set_primary_action(label, () => - frm.events.start_import(frm) - ); + frm.page.set_primary_action(label, () => frm.events.start_import(frm)); } else { frm.page.set_primary_action(__('Save'), () => frm.save()); } } - frm.page.set_indicator(__(frm.doc.status), frm.doc.status === 'Success' ? 'green' : 'grey'); + frm.page.set_indicator( + __(frm.doc.status), + frm.doc.status === 'Success' ? 'green' : 'grey' + ); }, show_success_message(frm) { @@ -139,6 +141,7 @@ frappe.ui.form.on('Data Import Beta', { } // load import preview + frm.get_field('import_preview').$wrapper.empty(); $('') .html(__('Loading import file...')) .appendTo(frm.get_field('import_preview').$wrapper); @@ -156,19 +159,17 @@ frappe.ui.form.on('Data Import Beta', { .then(r => { let preview_data = r.message; frm.events.show_import_preview(frm, preview_data); + frm.events.show_import_warnings(frm, preview_data); }); }, show_import_preview(frm, preview_data) { let import_log = JSON.parse(frm.doc.import_log || '[]'); - let warnings = JSON.parse(frm.doc.template_warnings || '[]'); - warnings = warnings.concat(preview_data.warnings || []); if (frm.import_preview) { frm.import_preview.preview_data = preview_data; frm.import_preview.import_log = import_log; frm.import_preview.refresh(); - frm.import_preview.render_warnings(warnings); return; } @@ -178,7 +179,7 @@ frappe.ui.form.on('Data Import Beta', { doctype: frm.doc.reference_doctype, preview_data, import_log, - warnings, + frm, events: { remap_column(changed_map) { let template_options = JSON.parse(frm.doc.template_options || '{}'); @@ -217,13 +218,78 @@ frappe.ui.form.on('Data Import Beta', { export_errored_rows() { open_url_post('/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_errored_template', { data_import_name: frm.doc.name - }) + }); + }, + + show_warnings() { + frm.scroll_to_field('import_warnings'); } } }); }); }, + show_import_warnings(frm, preview_data) { + frm.toggle_display('import_warnings_section', + preview_data.warnings && preview_data.warnings.length); + if (!preview_data) { + frm.get_field('import_warnings').$wrapper.html(''); + return; + } + let warnings = JSON.parse(frm.doc.template_warnings || '[]'); + warnings = warnings.concat(preview_data.warnings || []); + + // group warnings by row + let warnings_by_row = {}; + let other_warnings = []; + for (let warning of warnings) { + if (warning.row) { + warnings_by_row[warning.row] = warnings_by_row[warning.row] || []; + warnings_by_row[warning.row].push(warning); + } else { + other_warnings.push(warning); + } + } + + let html = ''; + html += Object.keys(warnings_by_row).map(row_number => { + let message = warnings_by_row[row_number] + .map(w => { + if (w.field) { + return `
  • ${w.field.label}: ${w.message}
    • `; + } + return w.message; + }) + .join(''); + return ` +
    +
    ${__('Row {0}', [row_number])}
    +
      ${message}
    +
    + `; + }).join(''); + + html += other_warnings + .map(warning => { + let header = ''; + if (warning.col) { + header = __('Column {0}', [warning.col]); + } + return ` +
    +
    ${header}
    +
    ${warning.message}
    +
    + `; + }) + .join(''); + frm.get_field('import_warnings').$wrapper.html(` +
    +
    ${html}
    +
    + `); + }, + show_import_log(frm) { let import_log = JSON.parse(frm.doc.import_log || '[]'); let logs = import_log; @@ -240,14 +306,21 @@ frappe.ui.form.on('Data Import Beta', { let html; if (log.success) { html = __('Successfully imported {0}', [ - `${frappe.utils.get_form_link(frm.doc.doctype, log.docname, true)}` + `${frappe.utils.get_form_link( + frm.doc.doctype, + log.docname, + true + )}` ]); } else { - let messages = log.messages.map(JSON.parse).map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `

    ${m.message}

    ` : ''; - return title + message; - }).join(''); + let messages = log.messages + .map(JSON.parse) + .map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `

    ${m.message}

    ` : ''; + return title + message; + }) + .join(''); let id = frappe.dom.get_unique_id(); html = `${messages} +
    +
    +
    +
    +
    +
    `); frappe.utils.bind_actions_with_class(this.wrapper, this); - this.$warnings = this.wrapper.find('.warnings'); this.$table_preview = this.wrapper.find('.table-preview'); } @@ -65,9 +62,14 @@ frappe.data_import.ImportPreview = class ImportPreview { let header_row_index = i - 1; if (df.skip_import) { let is_sr = df.label === 'Sr. No'; + let show_warnings_button = ``; let column_title = is_sr ? df.label - : `${df.header_title || `${__('Untitled Column')}`}`; + : ` + ${df.header_title || `${__('Untitled Column')}`} + ${!df.parent ? show_warnings_button : ''} + `; return { id: frappe.utils.get_random(6), name: df.label, @@ -123,20 +125,6 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - render_warnings(warnings) { - let html = ''; - if (warnings.length > 0) { - let warning_html = warnings - .map(warning => { - return `
  • ${warning}
    • `; - }) - .join(''); - - html = `
      ${warning_html}
    `; - } - this.$warnings.html(html); - } - render_datatable() { if (this.datatable) { this.datatable.destroy(); @@ -150,7 +138,7 @@ frappe.data_import.ImportPreview = class ImportPreview { this.datatable = new DataTable(this.$table_preview.get(0), { data: this.data, columns: this.columns, - layout: 'fixed', + layout: this.columns.length < 10 ? 'fluid' : 'fixed', cellHeight: 35, serialNoColumn: false, checkboxColumn: false, @@ -196,20 +184,42 @@ frappe.data_import.ImportPreview = class ImportPreview { } add_actions() { - let failures = this.import_log.filter(log => !log.success); - if (failures.length > 0) { - this.wrapper.find('.table-actions').append( - ` - `); - } + `; + }); + + this.wrapper.find('.table-actions').html(html); } export_errored_rows() { this.events.export_errored_rows(); } + show_warnings() { + this.events.show_warnings(); + } + show_column_mapper() { let column_picker_fields = new ColumnPickerFields({ doctype: this.doctype From 12f2d03a70f621ac30d12b2050c02349e7343e51 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 16:42:19 +0530 Subject: [PATCH 49/83] fix: Scroll to warning box --- .../js/frappe/data_import/import_preview.js | 15 +++++++++++++-- frappe/public/js/frappe/utils/utils.js | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 7880a82966..866d3d9f69 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -57,13 +57,17 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { - let column_width = 120; this.columns = this.fields.map((df, i) => { + let column_width = 120; let header_row_index = i - 1; if (df.skip_import) { let is_sr = df.label === 'Sr. No'; - let show_warnings_button = ``; + if (!df.parent) { + // increase column width for unidentified columns + column_width += 50 + } let column_title = is_sr ? df.label : ` @@ -220,6 +224,13 @@ frappe.data_import.ImportPreview = class ImportPreview { this.events.show_warnings(); } + show_column_warning(_, $target) { + let $warning = this.frm + .get_field('import_warnings').$wrapper + .find(`[data-col=${$target.data('col')}]`); + frappe.utils.scroll_to($warning, true, 30); + } + show_column_mapper() { let column_picker_fields = new ColumnPickerFields({ doctype: this.doctype diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 6821e3d410..c8aa2e3beb 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -734,7 +734,7 @@ Object.assign(frappe.utils, { let $target = $(e.currentTarget); let action = $target.data('action'); let method = class_instance[action]; - method ? class_instance[action]() : null; + method ? class_instance[action](e, $target) : null; }); return $el; From 08214d2d63c9cb0dfddd2557a724a8dc391e8b84 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Sep 2019 16:43:38 +0530 Subject: [PATCH 50/83] fix: Remove "Select mandatory without children" --- .../public/js/frappe/data_import/data_exporter.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 3c5c72fd93..c196acc025 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -143,9 +143,6 @@ frappe.data_import.DataExporter = class DataExporter { - @@ -191,18 +188,6 @@ frappe.data_import.DataExporter = class DataExporter { .trigger('change'); } - select_mandatory_without_children() { - let field = this.dialog.get_field(this.doctype); - let checkboxes = field.options - .filter(option => option.danger) - .map(option => option.$checkbox.find('input').get(0)); - - this.unselect_all(); - $(checkboxes) - .prop('checked', true) - .trigger('change'); - } - unselect_all() { this.dialog.$wrapper .find(':checkbox') From 1cccb33bada07b6d58e3f563a2fc9972ce020623 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 12:01:04 +0530 Subject: [PATCH 51/83] feat: Reload attached file --- .../js/frappe/data_import/data_exporter.js | 2 +- .../js/frappe/data_import/import_preview.js | 2 +- .../public/js/frappe/form/controls/attach.js | 23 +++++++++++++++---- frappe/public/js/frappe/utils/utils.js | 6 ++--- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index c196acc025..fd5fb980ea 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -148,7 +148,7 @@ frappe.data_import.DataExporter = class DataExporter { `); - frappe.utils.bind_actions_with_class($select_all_buttons, this); + frappe.utils.bind_actions_with_object($select_all_buttons, this); this.dialog .get_field('select_all_buttons') .$wrapper.html($select_all_buttons); diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 866d3d9f69..8986c4260d 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -51,7 +51,7 @@ frappe.data_import.ImportPreview = class ImportPreview { `); - frappe.utils.bind_actions_with_class(this.wrapper, this); + frappe.utils.bind_actions_with_object(this.wrapper, this); this.$table_preview = this.wrapper.find('.table-preview'); } diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js index ef9aa1e05c..7db843d5a0 100644 --- a/frappe/public/js/frappe/form/controls/attach.js +++ b/frappe/public/js/frappe/form/controls/attach.js @@ -13,7 +13,10 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ - ${__('Clear')} + `) .prependTo(me.input_area) .toggle(false); @@ -21,9 +24,8 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ this.set_input_attributes(); this.has_input = true; - this.$value.find(".clear-file").on("click", function() { - me.clear_attachment(); - }); + frappe.utils.bind_actions_with_object(this.$value, this); + this.toggle_reload_button(); }, clear_attachment: function() { var me = this; @@ -41,15 +43,21 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ this.refresh(); } }, + reload_attachment() { + if (this.file_uploader) { + this.file_uploader.uploader.upload_files(); + } + }, on_attach_click() { this.set_upload_options(); - new frappe.ui.FileUploader(this.upload_options); + this.file_uploader = new frappe.ui.FileUploader(this.upload_options); }, set_upload_options() { let options = { allow_multiple: false, on_success: file => { this.on_upload_complete(file); + this.toggle_reload_button(); } }; @@ -95,4 +103,9 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ } this.set_value(attachment.file_url); }, + + toggle_reload_button() { + this.$value.find('[data-action="reload_attachment"]') + .toggle(this.file_uploader && this.file_uploader.uploader.files.length > 0); + } }); diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index c8aa2e3beb..00b6f95f06 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -726,15 +726,15 @@ Object.assign(frappe.utils, { is_rtl() { return ["ar", "he", "fa"].includes(frappe.boot.lang); }, - bind_actions_with_class($el, class_instance) { + bind_actions_with_object($el, object) { // remove previously bound event $($el).off('click.class_actions'); // attach new event $($el).on('click.class_actions', '[data-action]', e => { let $target = $(e.currentTarget); let action = $target.data('action'); - let method = class_instance[action]; - method ? class_instance[action](e, $target) : null; + let method = object[action]; + method ? object[action](e, $target) : null; }); return $el; From 5c533dc90de4c41f720604677edecc8b9287caf3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 12:46:06 +0530 Subject: [PATCH 52/83] fix: Filter styling --- .../js/frappe/ui/filters/edit_filter.html | 21 +++++++++---------- frappe/public/less/filters.less | 18 +++++++++------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/frappe/public/js/frappe/ui/filters/edit_filter.html b/frappe/public/js/frappe/ui/filters/edit_filter.html index 50b1e1b4e6..3908c63fa1 100644 --- a/frappe/public/js/frappe/ui/filters/edit_filter.html +++ b/frappe/public/js/frappe/ui/filters/edit_filter.html @@ -1,6 +1,6 @@
    -
    +
    -
    -
    -
    - - - - - - +
    +
    +
    +
    +
    + + {%= __("Apply") %} + + {%= __("Remove") %}
    -
    diff --git a/frappe/public/less/filters.less b/frappe/public/less/filters.less index 30894eea19..95580857e7 100644 --- a/frappe/public/less/filters.less +++ b/frappe/public/less/filters.less @@ -31,9 +31,17 @@ float: none; } -.filter-box { +.frappe-list .filter-box { border-bottom: 1px solid @border-color; - padding: 10px 15px 3px; + padding: 10px 15px; +} + +.filter-box { + .form-group { + @media (min-width: @screen-xs) { + margin-bottom: 0; + } + } .remove-filter { margin-top: 6px; @@ -41,11 +49,9 @@ } .filter-field { - padding-right: 15px; - width: calc(100% - 36px); - .frappe-control { position: relative; + margin-bottom: 0; } } } @@ -56,8 +62,6 @@ padding-right: 0px; } .filter-field { - width: 65% !important; - .frappe-control { position: relative; } From b33d202a7a228085945867a2b79933223480dd3b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 12:46:42 +0530 Subject: [PATCH 53/83] fix: Download Template button --- .../doctype/data_import_beta/data_import_beta.js | 3 +-- .../data_import_beta/data_import_beta.json | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 86cb540745..80bb3ea6fa 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -112,8 +112,7 @@ frappe.ui.form.on('Data Import Beta', { frm.save().then(() => frm.call('start_import')); }, - download_sample_file(frm) { - frappe.require('/assets/js/data_import_tools.min.js', () => { + download_template(frm) { new frappe.data_import.DataExporter(frm.doc.reference_doctype); }); }, diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 4757a4b7da..cdb6ee8cfc 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -8,7 +8,7 @@ "field_order": [ "reference_doctype", "import_type", - "download_sample_file", + "download_template", "import_file", "column_break_5", "status", @@ -59,12 +59,6 @@ "fieldtype": "Section Break", "label": "Import Preview" }, - { - "depends_on": "reference_doctype", - "fieldname": "download_sample_file", - "fieldtype": "Button", - "label": "Download Sample File" - }, { "fieldname": "column_break_5", "fieldtype": "Column Break" @@ -137,10 +131,16 @@ "fieldname": "import_warnings", "fieldtype": "HTML", "label": "Import Warnings" + }, + { + "depends_on": "reference_doctype", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" } ], "hide_toolbar": 1, - "modified": "2019-09-24 15:08:25.984923", + "modified": "2019-09-25 12:41:46.836826", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From df3236a98a55861489b92660987b7f6a4fe7f9d2 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 12:47:04 +0530 Subject: [PATCH 54/83] fix: Set Export Type based on Import Type --- .../data_import_beta/data_import_beta.js | 19 +++++++++++++++++-- .../js/frappe/data_import/data_exporter.js | 11 +++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 80bb3ea6fa..72ab7a7172 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -113,8 +113,23 @@ frappe.ui.form.on('Data Import Beta', { }, download_template(frm) { - new frappe.data_import.DataExporter(frm.doc.reference_doctype); - }); + if (frm.data_exporter) { + frm.data_exporter.dialog.show(); + set_export_records(); + } else { + frappe.require('/assets/js/data_import_tools.min.js', () => { + frm.data_exporter = new frappe.data_import.DataExporter(frm.doc.reference_doctype); + set_export_records(); + }); + } + + function set_export_records() { + if (frm.doc.import_type === 'Insert New Records') { + frm.data_exporter.dialog.set_value('export_records', 'blank_template'); + } else { + frm.data_exporter.dialog.set_value('export_records', 'all'); + } + } }, reference_doctype(frm) { diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index fd5fb980ea..8c49690b4d 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -3,7 +3,6 @@ frappe.provide('frappe.data_import'); frappe.data_import.DataExporter = class DataExporter { constructor(doctype) { - frappe.data_exporter = this; this.doctype = doctype; frappe.model.with_doctype(doctype, () => { this.make_dialog(); @@ -17,22 +16,22 @@ frappe.data_import.DataExporter = class DataExporter { { fieldtype: 'Select', fieldname: 'export_records', - label: __('Export Records'), + label: __('Export Type'), options: [ { - label: __('Export All Records'), + label: __('All Records'), value: 'all' }, { - label: __('Export Filtered Records'), + label: __('Filtered Records'), value: 'by_filter' }, { - label: __('Export Blank Template'), + label: __('Blank Template'), value: 'blank_template' } ], - default: 'all', + default: 'blank_template', change: () => { this.update_record_count_message(); } From 2fc34effb4ff4a1a81d9755dd7a2c061cdf6a644 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 12:56:44 +0530 Subject: [PATCH 55/83] fix: Show message when scheduler in inactive --- frappe/core/doctype/data_import_beta/data_import_beta.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 734c00d1f1..97959ed308 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -9,6 +9,7 @@ from frappe.core.doctype.data_import.importer_new import Importer from frappe.core.doctype.data_import.exporter_new import Exporter from frappe.core.page.background_jobs.background_jobs import get_info from frappe.utils.background_jobs import enqueue +from frappe import _ class DataImportBeta(Document): @@ -28,6 +29,9 @@ class DataImportBeta(Document): return i.get_data_for_import_preview() def start_import(self): + if frappe.utils.scheduler.is_scheduler_inactive(): + frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Error")) + enqueued_jobs = [d.get("job_name") for d in get_info()] if self.name not in enqueued_jobs: @@ -38,7 +42,6 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, - now=True, ) def get_importer(self): @@ -103,7 +106,8 @@ def download_template( ) e.build_response() + @frappe.whitelist() def download_errored_template(data_import_name): - data_import = frappe.get_doc('Data Import Beta', data_import_name) + data_import = frappe.get_doc("Data Import Beta", data_import_name) data_import.export_errored_rows() From ebb111aaf897a6f77b25e2ec865cb67d25738ff3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 14:11:21 +0530 Subject: [PATCH 56/83] fix: Set only once Document Type and Import Type --- frappe/core/doctype/data_import_beta/data_import_beta.js | 4 +--- .../core/doctype/data_import_beta/data_import_beta.json | 8 +++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 72ab7a7172..56d8d570a3 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -61,8 +61,6 @@ frappe.ui.form.on('Data Import Beta', { frm.trigger('toggle_submit_after_import'); if (frm.doc.import_log && frm.doc.import_log !== '[]') { - // set form as readonly - frm.fields.forEach(f => (f.df.read_only = 1)); frm.disable_save(); } @@ -321,7 +319,7 @@ frappe.ui.form.on('Data Import Beta', { if (log.success) { html = __('Successfully imported {0}', [ `${frappe.utils.get_form_link( - frm.doc.doctype, + frm.doc.reference_doctype, log.docname, true )}` diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index cdb6ee8cfc..1994e55045 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -32,7 +32,8 @@ "in_list_view": 1, "label": "Document Type", "options": "DocType", - "reqd": 1 + "reqd": 1, + "set_only_once": 1 }, { "fieldname": "import_type", @@ -40,7 +41,8 @@ "in_list_view": 1, "label": "Import Type", "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1 + "reqd": 1, + "set_only_once": 1 }, { "depends_on": "eval:!doc.__islocal", @@ -140,7 +142,7 @@ } ], "hide_toolbar": 1, - "modified": "2019-09-25 12:41:46.836826", + "modified": "2019-09-25 13:03:25.463692", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From b024d0f89257a8ed8656cc8405b1d39b55768ced Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 14:56:37 +0530 Subject: [PATCH 57/83] fix: Check for mandatory fields --- .../core/doctype/data_import/importer_new.py | 71 +++++++++++-------- .../data_import_beta/data_import_beta.js | 2 +- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 46d0b1df0f..354a649793 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -188,7 +188,10 @@ class Importer: if i in skip_import: field.skip_import = True warnings.append( - {"col": column_number, "message": _("Skipping column {0}").format(frappe.bold(header_title))} + { + "col": column_number, + "message": _("Skipping column {0}").format(frappe.bold(header_title)), + } ) elif header_title and not df: warnings.append( @@ -392,6 +395,8 @@ class Importer: self.data_import.db_set("template_warnings", json.dumps(warnings)) frappe.publish_realtime("data_import_refresh") return + else: + self.data_import.db_set("template_warnings", "") # setup import log if self.data_import.import_log: @@ -544,7 +549,11 @@ class Importer: frappe.bold(value), frappe.bold(df.options) ) validate_warnings.append( - {"row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg} + { + "row": row_number, + "field": df.as_dict(convert_dates_to_str=True), + "message": msg, + } ) if validate_warnings: @@ -553,23 +562,46 @@ class Importer: return True - def parse_doc(doctype, docfields, values, row_index): + def parse_doc(doctype, docfields, values, row_number): doc = {} for index, (df, value) in enumerate(zip(docfields, values)): if df.get("skip_import", False): continue if value in INVALID_VALUES: - if df.reqd: - mandatory_fields.append(frappe._dict(row_number=row_number, df=df)) - continue - else: - value = None + value = None if validate_value(value, df): doc[df.fieldname] = self.parse_value(value, df) + + check_mandatory_fields(doctype, doc, row_number) + return doc + def check_mandatory_fields(doctype, doc, row_number): + meta = frappe.get_meta(doctype) + fields = [df for df in meta.fields if df.reqd and doc.get(df.fieldname) in INVALID_VALUES] + + if not fields: + return + + if len(fields) == 1: + warnings.append( + { + "row": row_number, + "message": _("{0} is a mandatory field").format(fields[0].label), + } + ) + else: + fields_string = ", ".join([df.label for df in fields]) + warnings.append( + { + "row": row_number, + "message": _("{0} are mandatory fields").format(fields_string), + } + ) + + parsed_docs = {} for row_index, row in enumerate(rows): for doctype in doctypes: @@ -602,29 +634,6 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs - if mandatory_fields: - df_by_row_number = {} - for d in mandatory_fields: - df_by_row_number.setdefault(d.row_number, []) - df_by_row_number[d.row_number].append(d.df) - - for row_number, fields in df_by_row_number.items(): - if len(fields) == 1: - warnings.append( - { - "row": row_number, - "message": _("{0} is a mandatory field").format(fields[0].label), - } - ) - else: - fields_string = ", ".join([df.label for df in fields]) - warnings.append( - { - "row": row_number, - "message": _("{0} are mandatory fields").format(fields_string), - } - ) - return doc, rows, data[len(rows) :], warnings def get_first_parent_column_index(self, fields): diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 56d8d570a3..df29482404 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -270,7 +270,7 @@ frappe.ui.form.on('Data Import Beta', { if (w.field) { return `
  • ${w.field.label}: ${w.message}
    • `; } - return w.message; + return `
  • ${w.message}
    • `; }) .join(''); return ` From d2fe00717715cd0c377ee5afb077174cb2d108a3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 15:05:07 +0530 Subject: [PATCH 58/83] fix(xls): Don't remove first row --- frappe/utils/xlsxutils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 9545722e9a..2814c5ff40 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -104,7 +104,6 @@ def read_xls_file_from_attached_file(content): rows = [] for i in range(sheet.nrows): rows.append(sheet.row_values(i)) - rows = rows[1:] return rows def build_xlsx_response(data, filename): From f86a5af7296ce4b7d3411e093905e1cdba1866a3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 15:06:23 +0530 Subject: [PATCH 59/83] fix: Update attach immediately --- frappe/public/js/frappe/form/controls/attach.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js index 7db843d5a0..a34c57b38f 100644 --- a/frappe/public/js/frappe/form/controls/attach.js +++ b/frappe/public/js/frappe/form/controls/attach.js @@ -30,6 +30,8 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({ clear_attachment: function() { var me = this; if(this.frm) { + me.parse_validate_and_set_in_model(null); + me.refresh(); me.frm.attachments.remove_attachment_by_filename(me.value, function() { me.parse_validate_and_set_in_model(null); me.refresh(); From 7927f5c8c4b0e343007f563be0804edb273e1b7a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 15:59:33 +0530 Subject: [PATCH 60/83] fix(progress): Always show green progress --- frappe/public/js/frappe/form/dashboard.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index 67befa1441..3e9dacbd05 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -123,11 +123,7 @@ frappe.ui.form.Dashboard = Class.extend({ format_percent: function(title, percent) { var width = cint(percent) < 1 ? 1 : cint(percent); - var progress_class = ""; - if(width < 10) - progress_class = "progress-bar-danger"; - if(width > 99.9) - progress_class = "progress-bar-success"; + var progress_class = "progress-bar-success"; return [{ title: title, From cd961cb9ae213f84683d42cccb2625246b4ae456 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 16:01:05 +0530 Subject: [PATCH 61/83] fix: Validate template on attach --- frappe/core/doctype/data_import/importer_new.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 354a649793..35eb298ee5 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -56,6 +56,8 @@ class Importer: self.read_file(file_path) elif content: self.read_content(content, extension) + + self.validate_template_content() self.remove_empty_rows_and_columns() def read_file(self, file_path): @@ -81,6 +83,13 @@ class Importer: self.header_row = data[0] self.data = data[1:] + def validate_template_content(self): + column_count = len(self.header_row) + if any([len(row) != column_count and len(row) != 0 for row in self.data]): + frappe.throw( + _("Number of columns does not match with data"), title=_("Invalid Template") + ) + def remove_empty_rows_and_columns(self): self.row_index_map = [] removed_rows = [] From 3b7b0f24dc366ba63d182ed2cdbcfbb597be03c2 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 16:01:15 +0530 Subject: [PATCH 62/83] fix: Commit after every doc import --- .../core/doctype/data_import/importer_new.py | 57 +++++++++---------- .../data_import_beta/data_import_beta.py | 5 ++ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 35eb298ee5..3d26f07033 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -382,6 +382,8 @@ class Importer: frappe.cache().hdel("lang", frappe.session.user) frappe.set_user_lang(frappe.session.user) + self.data_import.db_set("template_warnings", "") + # set flag frappe.flags.in_import = True @@ -404,8 +406,6 @@ class Importer: self.data_import.db_set("template_warnings", json.dumps(warnings)) frappe.publish_realtime("data_import_refresh") return - else: - self.data_import.db_set("template_warnings", "") # setup import log if self.data_import.import_log: @@ -425,8 +425,6 @@ class Importer: # start import print("Importing {0} rows...".format(len(data))) - # mark savepoint - frappe.db.sql("SAVEPOINT import") total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 @@ -441,10 +439,11 @@ class Importer: if set(row_indexes).intersection(set(imported_rows)): print("Skipping imported rows", row_indexes) - frappe.publish_realtime( - "data_import_progress", - {"current": current_index, "total": total_payload_count, "skipping": True}, - ) + if total_payload_count > 5: + frappe.publish_realtime( + "data_import_progress", + {"current": current_index, "total": total_payload_count, "skipping": True}, + ) continue try: @@ -453,20 +452,24 @@ class Importer: doc = self.process_doc(doc) processing_time = timeit.default_timer() - start eta = self.get_eta(current_index, total_payload_count, processing_time) - frappe.publish_realtime( - "data_import_progress", - { - "current": current_index, - "total": total_payload_count, - "docname": doc.name, - "success": True, - "row_indexes": row_indexes, - "eta": eta, - }, - ) + + if total_payload_count > 5: + frappe.publish_realtime( + "data_import_progress", + { + "current": current_index, + "total": total_payload_count, + "docname": doc.name, + "success": True, + "row_indexes": row_indexes, + "eta": eta, + }, + ) import_log.append( frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) ) + # commit after every successful import + frappe.db.commit() except Exception as e: import_log.append( @@ -479,12 +482,6 @@ class Importer: ) frappe.clear_messages() - # rollback to savepoint if something went wrong - # frappe.db.sql('ROLLBACK TO SAVEPOINT import') - - # release savepoint if everything is ok - frappe.db.sql("RELEASE SAVEPOINT import") - # set status failures = [l for l in import_log if l.get("success") == False] if len(failures) == total_payload_count: @@ -589,7 +586,9 @@ class Importer: def check_mandatory_fields(doctype, doc, row_number): meta = frappe.get_meta(doctype) - fields = [df for df in meta.fields if df.reqd and doc.get(df.fieldname) in INVALID_VALUES] + fields = [ + df for df in meta.fields if df.reqd and doc.get(df.fieldname) in INVALID_VALUES + ] if not fields: return @@ -604,13 +603,9 @@ class Importer: else: fields_string = ", ".join([df.label for df in fields]) warnings.append( - { - "row": row_number, - "message": _("{0} are mandatory fields").format(fields_string), - } + {"row": row_number, "message": _("{0} are mandatory fields").format(fields_string)} ) - parsed_docs = {} for row_index, row in enumerate(rows): for doctype in doctypes: diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 97959ed308..1efe794e0a 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -21,6 +21,10 @@ class DataImportBeta(Document): self.template_options = "" self.template_warnings = "" + if self.import_file: + # validate template + i = self.get_importer() + def get_preview_from_template(self): if not self.import_file: return @@ -42,6 +46,7 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, + # now=True ) def get_importer(self): From 23d6e27f80e1c56c957b5fb709eaec8fb00f5b88 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 16:27:07 +0530 Subject: [PATCH 63/83] fix: Option to export 5 records --- frappe/core/doctype/data_import/exporter_new.py | 4 +++- frappe/core/doctype/data_import_beta/data_import_beta.py | 1 + frappe/public/js/frappe/data_import/data_exporter.js | 7 ++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index d4224b08d4..fec813610d 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -17,6 +17,7 @@ class Exporter: export_fields=None, export_data=False, export_filters=None, + export_page_length=None, file_type="CSV", ): """ @@ -31,6 +32,7 @@ class Exporter: self.meta = frappe.get_meta(doctype) self.export_fields = export_fields self.export_filters = export_filters + self.export_page_length = export_page_length self.file_type = file_type # this will contain the csv content @@ -133,7 +135,7 @@ class Exporter: self.doctype, filters=filters, fields=fields, - limit_page_length=None, + limit_page_length=self.export_page_length, order_by=order_by, as_list=1, ) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 1efe794e0a..76fc608b42 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -108,6 +108,7 @@ def download_template( export_data=export_data, export_filters=export_filters, file_type=file_type, + export_page_length=5 if export_records == "5_records" else None, ) e.build_response() diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 8c49690b4d..39055aed7c 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -26,6 +26,10 @@ frappe.data_import.DataExporter = class DataExporter { label: __('Filtered Records'), value: 'by_filter' }, + { + label: __('5 Records'), + value: '5_records' + }, { label: __('Blank Template'), value: 'blank_template' @@ -202,7 +206,8 @@ frappe.data_import.DataExporter = class DataExporter { frappe.db.count(this.doctype, { filters: this.get_filters() }), - blank_template: () => Promise.resolve(0) + blank_template: () => Promise.resolve(0), + '5_records': () => Promise.resolve(5) }; count_method[export_records]().then(value => { From 5e8a56b44430be2b702ff240cad542f1aa1bf5bc Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 22:55:44 +0530 Subject: [PATCH 64/83] fix: Rollback on exception --- frappe/core/doctype/data_import/importer_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 3d26f07033..3f078f71cc 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -481,6 +481,8 @@ class Importer: ) ) frappe.clear_messages() + # rollback if exception + frappe.db.rollback() # set status failures = [l for l in import_log if l.get("success") == False] From 44028ce23a73ab78bb79de3e95efa066c7e50e94 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 25 Sep 2019 23:31:08 +0530 Subject: [PATCH 65/83] fix(test_runner): Load file if exists --- frappe/test_runner.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe/test_runner.py b/frappe/test_runner.py index cde743643f..76140e442c 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -250,10 +250,11 @@ def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): if os.path.basename(os.path.dirname(path))=="doctype": txt_file = os.path.join(path, filename[5:].replace(".py", ".json")) - with open(txt_file, 'r') as f: - doc = json.loads(f.read()) - doctype = doc["name"] - make_test_records(doctype, verbose) + if os.path.exists(txt_file): + with open(txt_file, 'r') as f: + doc = json.loads(f.read()) + doctype = doc["name"] + make_test_records(doctype, verbose) test_suite.addTest(unittest.TestLoader().loadTestsFromModule(module)) @@ -417,4 +418,4 @@ def get_test_record_log(): else: frappe.flags.test_record_log = [] - return frappe.flags.test_record_log \ No newline at end of file + return frappe.flags.test_record_log From a71f8f95b82b635a18380dd0895f108e4a1a5d79 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 26 Sep 2019 02:03:22 +0530 Subject: [PATCH 66/83] fix: Show template warnings --- .../core/doctype/data_import_beta/data_import_beta.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index df29482404..631f645b9c 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -242,14 +242,14 @@ frappe.ui.form.on('Data Import Beta', { }, show_import_warnings(frm, preview_data) { - frm.toggle_display('import_warnings_section', - preview_data.warnings && preview_data.warnings.length); - if (!preview_data) { + let warnings = JSON.parse(frm.doc.template_warnings || '[]'); + warnings = warnings.concat(preview_data.warnings || []); + + frm.toggle_display('import_warnings_section', warnings.length > 0); + if (warnings.length === 0) { frm.get_field('import_warnings').$wrapper.html(''); return; } - let warnings = JSON.parse(frm.doc.template_warnings || '[]'); - warnings = warnings.concat(preview_data.warnings || []); // group warnings by row let warnings_by_row = {}; From 056ad75bc2127008e114e1f0a312a83211c3dc11 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 26 Sep 2019 02:03:50 +0530 Subject: [PATCH 67/83] fix: Automatically create Link field dependencies --- .../core/doctype/data_import/importer_new.py | 61 ++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 3f078f71cc..2674ae24d0 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -541,34 +541,30 @@ class Importer: return [i for i, df in enumerate(fields) if df.parent == doctype] def validate_value(value, df): - validate_warnings = [] - if df.fieldtype == "Select" and value not in df.get_select_options(): options_string = ", ".join([frappe.bold(d) for d in df.get_select_options()]) msg = _("Value must be one of {0}").format(options_string) - validate_warnings.append( + warnings.append( {"row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg} ) + return False elif df.fieldtype == "Link": - missing_link_values = self.get_missing_link_field_values(df.options) - if value in missing_link_values: - msg = _("Value {0} missing for Document Type {1}").format( + d = self.get_missing_link_field_values(df.options) + if value in d.missing_values and not d.one_mandatory: + msg = _("Value {0} missing for {1}").format( frappe.bold(value), frappe.bold(df.options) ) - validate_warnings.append( + warnings.append( { "row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg, } ) + return value - if validate_warnings: - warnings.extend(validate_warnings) - return False - - return True + return value def parse_doc(doctype, docfields, values, row_number): doc = {} @@ -663,6 +659,7 @@ class Importer: return self.update_record(doc) def insert_record(self, doc): + self.create_missing_linked_records(doc) # name shouldn't be set when inserting a new record doc.update({"doctype": self.doctype, "name": None}) new_doc = frappe.get_doc(doc) @@ -671,6 +668,37 @@ class Importer: new_doc.submit() return new_doc + def create_missing_linked_records(self, doc): + """ + Finds fields that are of type Link, and creates the corresponding + document automatically if it has only one mandatory field + """ + link_values = [] + def get_link_fields(doc, doctype): + for fieldname, value in doc.items(): + meta = frappe.get_meta(doctype) + df = meta.get_field(fieldname) + if df.fieldtype == 'Link': + link_values.append([df.options, value]) + elif df.fieldtype in table_fields: + for row in value: + get_link_fields(row, df.options) + get_link_fields(doc, self.doctype) + + for link_doctype, link_value in link_values: + d = self.missing_link_values.get(link_doctype) + if d.one_mandatory and link_value in d.missing_values: + meta = frappe.get_meta(link_doctype) + # find the autoname field + if meta.autoname and meta.autoname.startswith("field:"): + autoname_field = meta.autoname[len("field:") :] + else: + autoname_field = "name" + new_doc = frappe.new_doc(link_doctype) + new_doc.set(autoname_field, link_value) + new_doc.insert() + d.missing_values.remove(link_value) + def update_record(self, doc): id_fieldname = self.get_id_fieldname() id_value = doc[id_fieldname] @@ -706,7 +734,7 @@ class Importer: build_csv_response(rows, self.doctype) def get_missing_link_field_values(self, doctype): - return self.missing_link_values.get(doctype, []) + return self.missing_link_values.get(doctype, {}) def prepare_missing_link_field_values(self, fields, data): link_column_indexes = [i for i, df in enumerate(fields) if df.fieldtype == "Link"] @@ -728,7 +756,12 @@ class Importer: doctype = df.options missing_values = [value for value in values if not frappe.db.exists(doctype, value)] - self.missing_link_values[doctype] = missing_values + if self.missing_link_values.get(doctype): + self.missing_link_values[doctype].missing_values += missing_values + else: + self.missing_link_values[doctype] = frappe._dict( + missing_values=missing_values, one_mandatory=has_one_mandatory_field(doctype), df=df + ) def get_id_fieldname(self): autoname = self.meta.autoname From addc20cde4c1275427f3b6fdd12f9e1a1e2d9f4c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 28 Sep 2019 13:35:47 +0530 Subject: [PATCH 68/83] fix: Update datatable --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7a5ccfe3e4..f2cc7b1cca 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "express": "^4.16.2", "fast-deep-equal": "^2.0.1", "frappe-charts": "^1.3.0", - "frappe-datatable": "^1.13.5", + "frappe-datatable": "^1.14.0", "frappe-gantt": "^0.1.0", "fuse.js": "^3.2.0", "highlight.js": "^9.12.0", diff --git a/yarn.lock b/yarn.lock index 3752d00516..0838f3f255 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1770,10 +1770,10 @@ frappe-charts@^1.3.0: resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-1.3.0.tgz#9ed033fa64833906bba16554187fa2f8a3a54ef6" integrity sha512-hdLv4fOIVgIL5eV9KYlsQaEpxkcJvuEVVDJewJL8PG0ySPy5EEiG5KZGL2uj7YegVWbtsqJ4Oq/74mjgQoMdag== -frappe-datatable@^1.13.5: - version "1.13.5" - resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.13.5.tgz#6f507fe7a84c22b1eab6b08e7b6fccbcdf7bb936" - integrity sha512-k3Y8ScfxSD6Kj3Ch98kY2EWBnHUm0oPuPZonkslq4w5689iUhduy/ZynmLgOYDVjXXajBZG3oh5ycnx1gCwY5Q== +frappe-datatable@^1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.14.0.tgz#8e5a0f61764fd634ae01f6767ce055b04ec5c3e1" + integrity sha512-rxePE/UpYFnWzAFIpiLrVGFHxh+fIbpDI98gAZfraZOgO4Dz6qDcJMaeSKDosQ1Zq6imt15KyKoaePXNpsCVfg== dependencies: hyperlist "^1.0.0-beta" lodash "^4.17.5" From d5c6a69e232b2237a5961559d41641a4608e9fa2 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 28 Sep 2019 13:46:49 +0530 Subject: [PATCH 69/83] fix: Select "Don't Import" to skip a column - Remove separate skip_column structure - Simpler Map Columns dialog --- .../core/doctype/data_import/importer_new.py | 9 ++- .../data_import_beta/data_import_beta.js | 26 ------- .../data_import_beta/data_import_beta.py | 4 +- .../js/frappe/data_import/import_preview.js | 69 ++++++++----------- frappe/public/less/form.less | 8 +++ 5 files changed, 41 insertions(+), 75 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 2674ae24d0..3cf798663a 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -27,7 +27,7 @@ class Importer: def __init__(self, doctype, data_import=None, file_path=None, content=None): self.doctype = doctype self.template_options = frappe._dict( - {"remap_column": {}, "skip_import": [], "edited_rows": []} + {"remap_column": {}, "edited_rows": []} ) if data_import: @@ -164,7 +164,6 @@ class Importer: def parse_fields_from_header_row(self): remap_column = self.template_options.remap_column - skip_import = self.template_options.skip_import fields = [] warnings = [] @@ -173,8 +172,8 @@ class Importer: for i, header_title in enumerate(self.header_row): header_row_index = str(i) column_number = str(i + 1) - if remap_column.get(header_row_index): - fieldname = remap_column.get(header_row_index) + fieldname = remap_column.get(header_row_index) + if fieldname and fieldname != "Don't Import": df = df_by_labels_and_fieldnames.get(fieldname) warnings.append( { @@ -194,7 +193,7 @@ class Importer: field.header_title = header_title field.skip_import = False - if i in skip_import: + if fieldname == "Don't Import": field.skip_import = True warnings.append( { diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 631f645b9c..3fe21a2ae6 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -197,36 +197,10 @@ frappe.ui.form.on('Data Import Beta', { let template_options = JSON.parse(frm.doc.template_options || '{}'); template_options.remap_column = template_options.remap_column || {}; Object.assign(template_options.remap_column, changed_map); - - // if the column is remapped, remove it from skip_import - if (template_options.skip_import) { - template_options.skip_import = template_options.skip_import.filter( - d => !Object.keys(template_options.remap_column).includes(cstr(d)) - ); - } frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => frm.trigger('import_file')); }, - skip_import(header_row_index) { - let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.skip_import = template_options.skip_import || []; - if (!template_options.skip_import.includes(header_row_index)) { - template_options.skip_import.push(header_row_index); - } - // if column is being skipped, remove it from remap_column - if ( - template_options.remap_column && - template_options.remap_column[header_row_index] - ) { - delete template_options.remap_column[header_row_index]; - } - frm.set_value('template_options', JSON.stringify(template_options)); - frm.save().then(() => { - frm.trigger('import_file'); - }); - }, - export_errored_rows() { open_url_post('/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_errored_template', { data_import_name: frm.doc.name diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 76fc608b42..e169ee06ac 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -34,7 +34,7 @@ class DataImportBeta(Document): def start_import(self): if frappe.utils.scheduler.is_scheduler_inactive(): - frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Error")) + frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive")) enqueued_jobs = [d.get("job_name") for d in get_info()] @@ -46,7 +46,7 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, - # now=True + now=True ) def get_importer(self): diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 8986c4260d..feb67d55c9 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -249,31 +249,27 @@ frappe.data_import.ImportPreview = class ImportPreview { } return [ { - label: __('Column {0}', [i]), + label: '', fieldtype: 'Data', default: df.header_title, fieldname: `Column ${i}`, read_only: 1 }, - { - fieldtype: 'Button', - label: 'Skip Column', - fieldname: 'skip_' + i, - click: () => { - let header_row_index = i - 1; - this.events.skip_import(header_row_index); - } - }, { fieldtype: 'Column Break' }, { fieldtype: 'Autocomplete', fieldname: i, - label: __('Select field'), + label: '', max_items: Infinity, - options: column_picker_fields.get_fields_as_options(), - default: fieldname, + options: [ + { + label: __("Don't Import"), + value: "Don't Import" + } + ].concat(column_picker_fields.get_fields_as_options()), + default: fieldname || "Don't Import", change() { changed.push(i); } @@ -285,8 +281,24 @@ frappe.data_import.ImportPreview = class ImportPreview { }); // flatten the array fields = fields.reduce((acc, curr) => [...acc, ...curr]); + let file_name = (this.frm.doc.import_file || '').split('/').pop(); + fields = [ + { + fieldtype: 'HTML', + fieldname: 'heading', + options: ` +
    + ${__('Map columns from {0} to fields in {1}', [file_name.bold(), this.doctype.bold()])} +
    + ` + }, + { + fieldtype: 'Section Break' + } + ].concat(fields); + let dialog = new frappe.ui.Dialog({ - title: __('Column Mapper'), + title: __('Map Columns'), fields, primary_action: (values) => { let changed_map = {}; @@ -300,37 +312,10 @@ frappe.data_import.ImportPreview = class ImportPreview { dialog.hide(); } }); + dialog.$body.addClass('map-columns'); dialog.show(); } - remap_column(col) { - let column_picker_fields = new ColumnPickerFields({ - doctype: this.doctype - }); - let dialog = new frappe.ui.Dialog({ - title: __('Remap Column: {0}', [col.name]), - fields: [ - { - fieldtype: 'Autocomplete', - fieldname: 'fieldname', - label: __('Select field'), - max_items: Infinity, - options: column_picker_fields.get_fields_as_options() - } - ], - primary_action: ({ fieldname }) => { - if (!fieldname) return; - this.events.remap_column(col.header_row_index, fieldname); - dialog.hide(); - } - }); - dialog.show(); - } - - skip_import(col) { - this.events.skip_import(col.header_row_index); - } - is_row_imported(row) { let serial_no = row[0].content; return this.import_log.find(log => { diff --git a/frappe/public/less/form.less b/frappe/public/less/form.less index fb8b9c3d49..77fe4b8f17 100644 --- a/frappe/public/less/form.less +++ b/frappe/public/less/form.less @@ -984,3 +984,11 @@ body[data-route^="Form/Communication"] textarea[data-fieldname="subject"] { .followed-by-label{ margin-top: 30px; } + +.map-columns .form-section { + padding: 0 7px 7px; +} + +.map-columns .form-section:first-child { + padding-top: 7px; +} From 13df26eaab230ca5cda3cf91b6d9ba661df63789 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 28 Sep 2019 23:56:55 +0530 Subject: [PATCH 70/83] fix: Show 10 rows in preview --- .../core/doctype/data_import/importer_new.py | 10 +++---- .../data_import_beta/data_import_beta.js | 4 --- .../data_import_beta/data_import_beta.json | 17 ++++------- .../js/frappe/data_import/import_preview.js | 28 +++++++++---------- 4 files changed, 22 insertions(+), 37 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 3cf798663a..f9c919387c 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -20,14 +20,14 @@ from frappe.exceptions import ValidationError, MandatoryError from frappe.model import display_fieldtypes, no_value_fields, table_fields INVALID_VALUES = ["", None] -MAX_ROWS_IN_PREVIEW = 500 +MAX_ROWS_IN_PREVIEW = 10 class Importer: def __init__(self, doctype, data_import=None, file_path=None, content=None): self.doctype = doctype self.template_options = frappe._dict( - {"remap_column": {}, "edited_rows": []} + {"remap_column": {}} ) if data_import: @@ -144,8 +144,9 @@ class Importer: out.fields = fields if len(out.data) > MAX_ROWS_IN_PREVIEW: - out.data = [] + out.data = out.data[:MAX_ROWS_IN_PREVIEW] out.max_rows_exceeded = True + out.max_rows_in_preview = MAX_ROWS_IN_PREVIEW return out def get_parsed_data_from_template(self): @@ -153,9 +154,6 @@ class Importer: formats, formats_warnings = self.parse_formats_from_first_10_rows() fields, data = self.add_serial_no_column(fields, self.data) - if self.template_options.edited_rows: - data = self.template_options.edited_rows - warnings = fields_warnings + formats_warnings return frappe._dict( diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 3fe21a2ae6..03086f1a1a 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -103,10 +103,6 @@ frappe.ui.form.on('Data Import Beta', { }, start_import(frm) { - let csv_array = frm.import_preview.get_rows_as_csv_array(); - let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.edited_rows = csv_array; - frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => frm.call('start_import')); }, diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 1994e55045..1c2be4ada0 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -12,14 +12,13 @@ "import_file", "column_break_5", "status", - "submit_after_import", "section_break_7", - "date_format", + "submit_after_import", "template_options", - "template_warnings", "section_import_preview", "import_preview", "import_warnings_section", + "template_warnings", "import_warnings", "import_log_section", "import_log", @@ -59,7 +58,7 @@ { "fieldname": "section_import_preview", "fieldtype": "Section Break", - "label": "Import Preview" + "label": "Preview" }, { "fieldname": "column_break_5", @@ -70,15 +69,8 @@ "depends_on": "eval:!doc.__islocal", "fieldname": "section_break_7", "fieldtype": "Section Break", - "hidden": 1, "label": "Import Options" }, - { - "fieldname": "date_format", - "fieldtype": "Select", - "label": "Date Format", - "options": "\nYYYY-MM-DD\nDD-MM-YYYY\nMM-DD-YYYY\nYYYY/MM/DD\nDD/MM/YYYY\nMM/DD/YYYY\nMM/DD/YY\nDD/MM/YY\nYYYY.MM.DD\nDD.MM.YYYY\nMM.DD.YYYY" - }, { "fieldname": "template_options", "fieldtype": "Code", @@ -115,6 +107,7 @@ { "fieldname": "template_warnings", "fieldtype": "Code", + "hidden": 1, "label": "Template Warnings", "options": "JSON" }, @@ -127,7 +120,7 @@ { "fieldname": "import_warnings_section", "fieldtype": "Section Break", - "label": "Import Warnings" + "label": "Warnings" }, { "fieldname": "import_warnings", diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index feb67d55c9..fd512172db 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -30,7 +30,6 @@ frappe.data_import.ImportPreview = class ImportPreview { this.header_row = this.preview_data.header_row; this.fields = this.preview_data.fields; this.data = this.preview_data.data; - this.max_rows_exceeded = this.preview_data.max_rows_exceeded; this.make_wrapper(); this.prepare_columns(); this.prepare_data(); @@ -47,6 +46,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
    +
    @@ -110,7 +110,7 @@ frappe.data_import.ImportPreview = class ImportPreview { name: column_title, content: `${df.header_title || df.label}`, df: df, - editable: true, + editable: false, align: 'left', header_row_index, width: column_width @@ -134,11 +134,6 @@ frappe.data_import.ImportPreview = class ImportPreview { this.datatable.destroy(); } - let no_data_message = this.max_rows_exceeded - ? __('Cannot load preview for more than 500 rows. You can still remap or skip columns.') - : __('No Data'); - no_data_message = `${no_data_message}`; - this.datatable = new DataTable(this.$table_preview.get(0), { data: this.data, columns: this.columns, @@ -146,10 +141,19 @@ frappe.data_import.ImportPreview = class ImportPreview { cellHeight: 35, serialNoColumn: false, checkboxColumn: false, - pasteFromClipboard: true, - noDataMessage: no_data_message + noDataMessage: __('No Data'), + disableReorderColumn: true }); + let { max_rows_exceeded, max_rows_in_preview } = this.preview_data; + if (max_rows_exceeded) { + this.wrapper.find('.table-message').html(` +
    + ${__('Showing only first {0} rows in preview', [max_rows_in_preview])} +
    + `); + } + if (this.data.length === 0) { this.datatable.style.setStyle('.dt-scrollable', { height: 'auto' @@ -161,12 +165,6 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } - get_rows_as_csv_array() { - return this.datatable.getRows().map(row => { - return row.map(cell => cell.content); - }); - } - setup_styles() { // import success checkbox this.datatable.style.setStyle(`svg.import-success`, { From 4116e17bb3a57da4719097527346213321568429 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 00:25:03 +0530 Subject: [PATCH 71/83] fix: Mute emails in during import --- frappe/core/doctype/data_import/importer_new.py | 4 +++- .../core/doctype/data_import_beta/data_import_beta.json | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index f9c919387c..934c7b8727 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -381,8 +381,9 @@ class Importer: self.data_import.db_set("template_warnings", "") - # set flag + # set flags frappe.flags.in_import = True + frappe.flags.mute_emails = self.data_import.mute_emails out = self.get_parsed_data_from_template() fields = out["fields"] @@ -494,6 +495,7 @@ class Importer: self.data_import.db_set("import_log", json.dumps(import_log)) frappe.flags.in_import = False + frappe.flags.mute_emails = False frappe.publish_realtime("data_import_refresh") def get_payloads_for_import(self, fields, data): diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 1c2be4ada0..30f730589a 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -14,6 +14,7 @@ "status", "section_break_7", "submit_after_import", + "mute_emails", "template_options", "section_import_preview", "import_preview", @@ -132,10 +133,16 @@ "fieldname": "download_template", "fieldtype": "Button", "label": "Download Template" + }, + { + "default": "0", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails" } ], "hide_toolbar": 1, - "modified": "2019-09-25 13:03:25.463692", + "modified": "2019-09-28 13:54:35.061730", "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", From 779084991ee95ef210ca76910cce2e6ebf0b2ccc Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 19:16:12 +0530 Subject: [PATCH 72/83] refactor - Parse template and build self.rows and self.columns - Store header_row data in columns along with df and skip_import - Use self.columns and self.rows without passing them explicitly - Remove the ability to edit rows - Show only first 10 rows as preview - Build doc with default values set - Show Dashboard progress when coming back from another view - Better ETA Message inspired from Apple - Action buttons "Export Errored Rows" and "Go to DocType List" - Import status "Imported x out of y records" - Success / Failure column in import log --- .../core/doctype/data_import/importer_new.py | 273 +++++++++--------- .../data_import_beta/data_import_beta.js | 124 +++++--- .../data_import_beta/data_import_beta.py | 34 +-- .../js/frappe/data_import/import_preview.js | 75 ++--- frappe/public/js/frappe/form/dashboard.js | 9 +- 5 files changed, 264 insertions(+), 251 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 934c7b8727..ca56dce16a 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -26,9 +26,7 @@ MAX_ROWS_IN_PREVIEW = 10 class Importer: def __init__(self, doctype, data_import=None, file_path=None, content=None): self.doctype = doctype - self.template_options = frappe._dict( - {"remap_column": {}} - ) + self.template_options = frappe._dict({"remap_column": {}}) if data_import: self.data_import = data_import @@ -42,9 +40,14 @@ class Importer: self.data = None # used to store date formats guessed from data rows per column self._guessed_date_formats = {} + # used to store eta during import self.last_eta = 0 + # used to collect warnings during template parsing + # and show them to user + self.warnings = [] self.meta = frappe.get_meta(doctype) self.prepare_content(file_path, content) + self.parse_data_from_template() def prepare_content(self, file_path, content): if self.data_import: @@ -128,91 +131,89 @@ class Importer: self.header_row = header_row def get_data_for_import_preview(self): - out = self.get_parsed_data_from_template() - - # prepare fields - fields = [] - for df in out.fields: - header_title = df.header_title - skip_import = df.skip_import - if isinstance(df, DocField): - field = df.as_dict() - else: - field = df - field.update({"header_title": header_title, "skip_import": skip_import}) - fields.append(field) - out.fields = fields - + out = frappe._dict() + out.data = list(self.rows) + out.columns = self.columns + out.warnings = self.warnings if len(out.data) > MAX_ROWS_IN_PREVIEW: out.data = out.data[:MAX_ROWS_IN_PREVIEW] out.max_rows_exceeded = True out.max_rows_in_preview = MAX_ROWS_IN_PREVIEW return out - def get_parsed_data_from_template(self): - fields, fields_warnings = self.parse_fields_from_header_row() - formats, formats_warnings = self.parse_formats_from_first_10_rows() - fields, data = self.add_serial_no_column(fields, self.data) + def parse_data_from_template(self): + columns = self.parse_columns_from_header_row() + columns, data = self.add_serial_no_column(columns, self.data) - warnings = fields_warnings + formats_warnings + self.columns = columns + self.rows = data - return frappe._dict( - header_row=self.header_row, fields=fields, data=data, warnings=warnings - ) - - def parse_fields_from_header_row(self): + def parse_columns_from_header_row(self): remap_column = self.template_options.remap_column - fields = [] - warnings = [] + columns = [] df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() for i, header_title in enumerate(self.header_row): header_row_index = str(i) column_number = str(i + 1) + skip_import = False fieldname = remap_column.get(header_row_index) + if fieldname and fieldname != "Don't Import": df = df_by_labels_and_fieldnames.get(fieldname) - warnings.append( + self.warnings.append( { "col": column_number, "message": _("Mapping column {0} to field {1}").format( frappe.bold(header_title or "Untitled Column"), frappe.bold(df.label) ), + "type": "info", } ) else: df = df_by_labels_and_fieldnames.get(header_title) if not df: - field = frappe._dict(header_title=header_title, skip_import=True) + skip_import = True else: - field = df - field.header_title = header_title - field.skip_import = False + skip_import = False if fieldname == "Don't Import": - field.skip_import = True - warnings.append( + skip_import = True + self.warnings.append( { "col": column_number, "message": _("Skipping column {0}").format(frappe.bold(header_title)), + "type": "info", } ) elif header_title and not df: - warnings.append( + self.warnings.append( { "col": column_number, "message": _("Cannot match column {0} with any field").format( frappe.bold(header_title) ), + "type": "info", } ) elif not header_title and not df: - warnings.append({"col": column_number, "message": _("Skipping Untitled Column")}) - fields.append(field) + self.warnings.append( + {"col": column_number, "message": _("Skipping Untitled Column"), "type": "info"} + ) - return fields, warnings + columns.append( + frappe._dict( + df=df, + skip_import=skip_import, + header_title=header_title, + column_number=column_number, + index=i, + ) + ) + + return columns def build_fields_dict_for_column_matching(self): """ @@ -301,30 +302,20 @@ class Importer: out.append(df) return out - def parse_formats_from_first_10_rows(self): - """ - Returns a list of column descriptors for columns that might need parsing. - For e.g if it is a Date column return the Date format - [ - [['Data']], - [['Date', '%m/%d/%y']], - [['Currency', '#,###.##']], - ... - ] - """ - formats = [] - return formats, [] + def add_serial_no_column(self, columns, data): + columns_with_serial_no = [ + frappe._dict({"header_title": "Sr. No", "skip_import": True}) + ] + columns - def add_serial_no_column(self, fields, data): - fields_with_serial_no = [ - frappe._dict({"label": "Sr. No", "skip_import": True, "parent": None}) - ] + fields + # update index for each column + for i, col in enumerate(columns_with_serial_no): + col.index = i data_with_serial_no = [] for i, row in enumerate(data): data_with_serial_no.append([self.row_index_map[i] + 1] + row) - return fields_with_serial_no, data_with_serial_no + return columns_with_serial_no, data_with_serial_no def parse_value(self, value, df): # convert boolean values to 0 or 1 @@ -385,24 +376,19 @@ class Importer: frappe.flags.in_import = True frappe.flags.mute_emails = self.data_import.mute_emails - out = self.get_parsed_data_from_template() - fields = out["fields"] - data = out["data"] - warnings = [] - # prepare a map for missing link field values - self.prepare_missing_link_field_values(fields, data) + self.prepare_missing_link_field_values() - # parse import data - payloads = self.get_payloads_for_import(fields, data) - - # collect warnings - for payload in payloads: - warnings += payload.warnings + # parse docs from rows + payloads = self.get_payloads_for_import() + # dont import if there are non-ignorable warnings + warnings = [w for w in self.warnings if w.get("type") != "info"] if warnings: self.data_import.db_set("template_warnings", json.dumps(warnings)) - frappe.publish_realtime("data_import_refresh") + frappe.publish_realtime( + "data_import_refresh", {"data_import": self.data_import.name} + ) return # setup import log @@ -422,7 +408,7 @@ class Importer: imported_rows += log.row_indexes # start import - print("Importing {0} rows...".format(len(data))) + print("Importing {0} rows...".format(len(self.rows))) total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 @@ -440,7 +426,12 @@ class Importer: if total_payload_count > 5: frappe.publish_realtime( "data_import_progress", - {"current": current_index, "total": total_payload_count, "skipping": True}, + { + "current": current_index, + "total": total_payload_count, + "skipping": True, + "data_import": self.data_import.name, + }, ) continue @@ -458,6 +449,7 @@ class Importer: "current": current_index, "total": total_payload_count, "docname": doc.name, + "data_import": self.data_import.name, "success": True, "row_indexes": row_indexes, "eta": eta, @@ -496,24 +488,25 @@ class Importer: frappe.flags.in_import = False frappe.flags.mute_emails = False - frappe.publish_realtime("data_import_refresh") + frappe.publish_realtime("data_import_refresh", {"data_import": self.data_import.name}) - def get_payloads_for_import(self, fields, data): + def get_payloads_for_import(self): payloads = [] + # make a copy + data = list(self.rows) while data: - doc, rows, data, warnings = self.parse_next_row_for_import(fields, data) - payloads.append(frappe._dict(doc=doc, rows=rows, warnings=warnings)) + doc, rows, data = self.parse_next_row_for_import(data) + payloads.append(frappe._dict(doc=doc, rows=rows)) return payloads - def parse_next_row_for_import(self, fields, data): + def parse_next_row_for_import(self, data): """ Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. - Returns the doc, rows, data without the rows and warnings. + Returns the doc, rows, and data without the rows. """ doc = {} - warnings = [] mandatory_fields = [] - doctypes = set([df.parent for df in fields if df.parent]) + doctypes = set([col.df.parent for col in self.columns if col.df and col.df.parent]) # first row is included by default first_row = data[0] @@ -524,7 +517,7 @@ class Importer: # subsequent rows either dont have any parent value set # or have the same value as the parent # we include a row if either of conditions match - parent_column_index = self.get_first_parent_column_index(fields) + parent_column_index = self.get_first_parent_column_index() parent_value = first_row[parent_column_index] data_without_first_row = data[1:] for d in data_without_first_row: @@ -537,16 +530,26 @@ class Importer: rows.append(d) def get_column_indexes(doctype): - return [i for i, df in enumerate(fields) if df.parent == doctype] + return [ + col.index + for col in self.columns + if not col.skip_import and col.df and col.df.parent == doctype + ] def validate_value(value, df): - if df.fieldtype == "Select" and value not in df.get_select_options(): - options_string = ", ".join([frappe.bold(d) for d in df.get_select_options()]) - msg = _("Value must be one of {0}").format(options_string) - warnings.append( - {"row": row_number, "field": df.as_dict(convert_dates_to_str=True), "message": msg} - ) - return False + if df.fieldtype == "Select": + select_options = df.get_select_options() + if select_options and value not in select_options: + options_string = ", ".join([frappe.bold(d) for d in select_options]) + msg = _("Value must be one of {0}").format(options_string) + self.warnings.append( + { + "row": row_number, + "field": df.as_dict(convert_dates_to_str=True), + "message": msg, + } + ) + return False elif df.fieldtype == "Link": d = self.get_missing_link_field_values(df.options) @@ -554,7 +557,7 @@ class Importer: msg = _("Value {0} missing for {1}").format( frappe.bold(value), frappe.bold(df.options) ) - warnings.append( + self.warnings.append( { "row": row_number, "field": df.as_dict(convert_dates_to_str=True), @@ -566,19 +569,21 @@ class Importer: return value def parse_doc(doctype, docfields, values, row_number): - doc = {} - for index, (df, value) in enumerate(zip(docfields, values)): - if df.get("skip_import", False): - continue + # new_doc returns a dict with default values set + doc = frappe.new_doc(doctype, as_dict=True) + # remove standard fields and __islocal + for key in frappe.model.default_fields + ('__islocal',): + doc.pop(key, None) + for index, (df, value) in enumerate(zip(docfields, values)): if value in INVALID_VALUES: value = None - if validate_value(value, df): + value = validate_value(value, df) + if value: doc[df.fieldname] = self.parse_value(value, df) check_mandatory_fields(doctype, doc, row_number) - return doc def check_mandatory_fields(doctype, doc, row_number): @@ -591,7 +596,7 @@ class Importer: return if len(fields) == 1: - warnings.append( + self.warnings.append( { "row": row_number, "message": _("{0} is a mandatory field").format(fields[0].label), @@ -599,7 +604,7 @@ class Importer: ) else: fields_string = ", ".join([df.label for df in fields]) - warnings.append( + self.warnings.append( {"row": row_number, "message": _("{0} are mandatory fields").format(fields_string)} ) @@ -619,7 +624,8 @@ class Importer: # skip values if all of them are empty continue - docfields = [fields[i] for i in column_indexes] + columns = [self.columns[i] for i in column_indexes] + docfields = [col.df for col in columns] doc = parse_doc(doctype, docfields, values, row_number) parsed_docs[doctype] = parsed_docs.get(doctype, []) parsed_docs[doctype].append(doc) @@ -635,17 +641,17 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs - return doc, rows, data[len(rows) :], warnings + return doc, rows, data[len(rows) :] - def get_first_parent_column_index(self, fields): + def get_first_parent_column_index(self): """ Returns the first column's index which must be one of the parent columns """ # find a parent column parent_column_index = -1 - for i, df in enumerate(fields): - if not df.get("skip_import", False) and df.parent == self.doctype: - parent_column_index = i + for col in self.columns: + if not col.skip_import and col.df and col.df.parent == self.doctype: + parent_column_index = col.index break return parent_column_index @@ -659,9 +665,11 @@ class Importer: def insert_record(self, doc): self.create_missing_linked_records(doc) + + new_doc = frappe.new_doc(self.doctype) + new_doc.update(doc) # name shouldn't be set when inserting a new record - doc.update({"doctype": self.doctype, "name": None}) - new_doc = frappe.get_doc(doc) + new_doc.set("name", None) new_doc.insert() if self.meta.is_submittable and self.data_import.submit_after_import: new_doc.submit() @@ -673,15 +681,17 @@ class Importer: document automatically if it has only one mandatory field """ link_values = [] + def get_link_fields(doc, doctype): for fieldname, value in doc.items(): meta = frappe.get_meta(doctype) df = meta.get_field(fieldname) - if df.fieldtype == 'Link': + if df.fieldtype == "Link": link_values.append([df.options, value]) elif df.fieldtype in table_fields: for row in value: get_link_fields(row, df.options) + get_link_fields(doc, self.doctype) for link_doctype, link_value in link_values: @@ -701,7 +711,7 @@ class Importer: def update_record(self, doc): id_fieldname = self.get_id_fieldname() id_value = doc[id_fieldname] - existing_doc = frappe.get_doc(self.doctype, {id_fieldname: id_value}) + existing_doc = frappe.get_doc(self.doctype, id_value) existing_doc.flags.via_data_import = self.data_import.name existing_doc.update(doc) existing_doc.save() @@ -723,43 +733,37 @@ class Importer: row_indexes = list(set(row_indexes)) row_indexes.sort() - out = self.get_parsed_data_from_template() - header_row = out["header_row"] - data = out["data"] - + header_row = [col.header_title for col in self.columns[1:]] rows = [header_row] - rows += [row[1:] for row in data if row[0] in row_indexes] + rows += [row[1:] for row in self.rows if row[0] in row_indexes] build_csv_response(rows, self.doctype) def get_missing_link_field_values(self, doctype): return self.missing_link_values.get(doctype, {}) - def prepare_missing_link_field_values(self, fields, data): - link_column_indexes = [i for i, df in enumerate(fields) if df.fieldtype == "Link"] - - def has_one_mandatory_field(doctype): - meta = frappe.get_meta(doctype) - # get mandatory fields with default not set - mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] - mandatory_fields_count = len(mandatory_fields) - if meta.autoname and meta.autoname.lower() == "prompt": - mandatory_fields_count += 1 - return mandatory_fields_count == 1 + def prepare_missing_link_field_values(self): + columns = self.columns + rows = self.rows + link_column_indexes = [ + col.index for col in columns if col.df and col.df.fieldtype == "Link" + ] self.missing_link_values = {} for index in link_column_indexes: - df = fields[index] - column_values = [row[index] for row in data] + col = columns[index] + column_values = [row[index] for row in rows] values = set([v for v in column_values if v not in INVALID_VALUES]) - doctype = df.options + doctype = col.df.options missing_values = [value for value in values if not frappe.db.exists(doctype, value)] if self.missing_link_values.get(doctype): self.missing_link_values[doctype].missing_values += missing_values else: self.missing_link_values[doctype] = frappe._dict( - missing_values=missing_values, one_mandatory=has_one_mandatory_field(doctype), df=df + missing_values=missing_values, + one_mandatory=self.has_one_mandatory_field(doctype), + df=col.df, ) def get_id_fieldname(self): @@ -778,6 +782,15 @@ class Importer: self.last_eta = eta return self.last_eta + def has_one_mandatory_field(self, doctype): + meta = frappe.get_meta(doctype) + # get mandatory fields with default not set + mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] + mandatory_fields_count = len(mandatory_fields) + if meta.autoname and meta.autoname.lower() == "prompt": + mandatory_fields_count += 1 + return mandatory_fields_count == 1 + DATE_FORMATS = [ r"%d-%m-%Y", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 03086f1a1a..f2ff1eb11b 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -3,31 +3,40 @@ frappe.ui.form.on('Data Import Beta', { setup(frm) { - frappe.realtime.on('data_import_refresh', () => { + frappe.realtime.on('data_import_refresh', ({ data_import }) => { + if (data_import !== frm.doc.name) return; frappe.model.clear_doc('Data Import Beta', frm.doc.name); frappe.model.with_doc('Data Import Beta', frm.doc.name).then(() => { frm.refresh(); }); }); frappe.realtime.on('data_import_progress', data => { + if (data.data_import !== frm.doc.name) { + return; + } let percent = Math.floor((data.current * 100) / data.total); + let seconds = Math.floor(data.eta); + let minutes = Math.floor(data.eta / 60); let eta_message = - data.eta < 60 - ? __('ETA {0} seconds', [Math.floor(data.eta)]) - : __('ETA {0} minutes', [Math.floor(data.eta / 60)]); + seconds < 60 + ? __('About {0} seconds remaining', [seconds]) + : minutes === 1 + ? __('About {0} minute remaining', [minutes]) + : __('About {0} minutes remaining', [minutes]); + let message; if (data.success) { - let message_args = [data.docname, data.current, data.total]; + let message_args = [data.current, data.total, eta_message]; message = frm.doc.import_type === 'Insert New Records' - ? __('Importing {0} ({1} of {2})', message_args) - : __('Updating {0} ({1} of {2})', message_args); + ? __('Importing {0} of {1}, {2}', message_args) + : __('Updating {0} of {1}, {2}', message_args); } if (data.skipping) { - message = __('Skipping ({1} of {2})', [data.current, data.total]); + message = __('Skipping {0} of {1}, {2}', [data.current, data.total, eta_message]); } frm.dashboard.show_progress(__('Import Progress'), percent, message); - frm.page.set_indicator(eta_message, 'orange'); + frm.page.set_indicator(__('In Progress'), 'orange'); // hide progress when complete if (data.current === data.total) { @@ -59,14 +68,19 @@ frappe.ui.form.on('Data Import Beta', { frm.trigger('show_import_log'); frm.trigger('show_import_warnings'); frm.trigger('toggle_submit_after_import'); + frm.trigger('show_import_status'); - if (frm.doc.import_log && frm.doc.import_log !== '[]') { - frm.disable_save(); + if (frm.doc.status === 'Partial Success') { + frm.add_custom_button(__('Export Errored Rows'), + () => frm.trigger('export_errored_rows')); } - if (frm.doc.status === 'Success') { - frm.events.show_success_message(frm); - } else { + if (frm.doc.status.includes('Success')) { + frm.add_custom_button(__('Go to {0} List', [frm.doc.reference_doctype]), + () => frappe.set_route('List', frm.doc.reference_doctype)); + } + + if (frm.doc.status !== 'Success') { if (!frm.is_new() && frm.doc.import_file) { let label = frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); frm.page.set_primary_action(label, () => frm.events.start_import(frm)); @@ -74,30 +88,40 @@ frappe.ui.form.on('Data Import Beta', { frm.page.set_primary_action(__('Save'), () => frm.save()); } } - frm.page.set_indicator( - __(frm.doc.status), - frm.doc.status === 'Success' ? 'green' : 'grey' - ); }, - - show_success_message(frm) { + show_import_status(frm) { let import_log = JSON.parse(frm.doc.import_log || '[]'); let successful_records = import_log.filter(log => log.success); - let link = ` - ${__('{0} List', [frm.doc.reference_doctype])} - `; - let message_args = [successful_records.length, link]; + let failed_records = import_log.filter(log => !log.success); + if (successful_records.length === 0) return; + let message; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records. Go to {1}', message_args) - : __('Successfully imported {0} record. Go to {1}', message_args); + if (failed_records.length === 0) { + let message_args = [successful_records.length]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records.length > 1 + ? __('Successfully imported {0} records.', message_args) + : __('Successfully imported {0} record.', message_args); + } else { + message = + successful_records.length > 1 + ? __('Successfully updated {0} records.', message_args) + : __('Successfully updated {0} record.', message_args); + } } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records. Go to {1}', message_args) - : __('Successfully updated {0} record. Go to {1}', message_args); + let message_args = [successful_records.length, import_log.length]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records.length > 1 + ? __('Successfully imported {0} records out of {1}.', message_args) + : __('Successfully imported {0} record out of {1}.', message_args); + } else { + message = + successful_records.length > 1 + ? __('Successfully updated {0} records out of {1}.', message_args) + : __('Successfully updated {0} record out of {1}.', message_args); + } } frm.dashboard.set_headline(message); }, @@ -196,21 +220,17 @@ frappe.ui.form.on('Data Import Beta', { frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => frm.trigger('import_file')); }, - - export_errored_rows() { - open_url_post('/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_errored_template', { - data_import_name: frm.doc.name - }); - }, - - show_warnings() { - frm.scroll_to_field('import_warnings'); - } } }); }); }, + export_errored_rows(frm) { + open_url_post('/api/method/frappe.core.doctype.data_import_beta.data_import_beta.download_errored_template', { + data_import_name: frm.doc.name + }); + }, + show_import_warnings(frm, preview_data) { let warnings = JSON.parse(frm.doc.template_warnings || '[]'); warnings = warnings.concat(preview_data.warnings || []); @@ -299,13 +319,13 @@ frappe.ui.form.on('Data Import Beta', { .map(JSON.parse) .map(m => { let title = m.title ? `${m.title}` : ''; - let message = m.message ? `

    ${m.message}

    ` : ''; + let message = m.message ? `
    ${m.message}
    ` : ''; return title + message; }) .join(''); let id = frappe.dom.get_unique_id(); html = `${messages} -
    @@ -314,9 +334,16 @@ frappe.ui.form.on('Data Import Beta', {
    `; } + let indicator_color = log.success ? 'green' : 'red'; + let title = log.success ? __('Success') : __('Failure'); return ` ${log.row_indexes.join(', ')} - ${html} + +
    ${title}
    + + + ${html} + `; }) .join(''); @@ -324,8 +351,9 @@ frappe.ui.form.on('Data Import Beta', { frm.get_field('import_log_preview').$wrapper.html(` - - + + + ${rows}
    ${__('Row Number')}${__('Message')}${__('Row Number')}${__('Status')}${__('Message')}
    diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index e169ee06ac..705013d9c5 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -34,7 +34,9 @@ class DataImportBeta(Document): def start_import(self): if frappe.utils.scheduler.is_scheduler_inactive(): - frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive")) + frappe.throw( + _("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive") + ) enqueued_jobs = [d.get("job_name") for d in get_info()] @@ -46,37 +48,15 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, - now=True + now=True, ) - def get_importer(self): - return Importer(self.reference_doctype, data_import=self) - - def create_missing_link_values(self, missing_link_values): - docs = [] - for d in missing_link_values: - d = frappe._dict(d) - if not d.has_one_mandatory_field: - continue - - doctype = d.doctype - values = d.missing_values - meta = frappe.get_meta(doctype) - # find the autoname field - if meta.autoname and meta.autoname.startswith("field:"): - autoname_field = meta.autoname[len("field:") :] - else: - autoname_field = "name" - - for value in values: - new_doc = frappe.new_doc(doctype) - new_doc.set(autoname_field, value) - docs.append(new_doc.insert()) - return docs - def export_errored_rows(self): return self.get_importer().export_errored_rows() + def get_importer(self): + return Importer(self.reference_doctype, data_import=self) + def start_import(data_import): """This method runs in background job""" diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index fd512172db..89efb2c69e 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -27,8 +27,6 @@ frappe.data_import.ImportPreview = class ImportPreview { } refresh() { - this.header_row = this.preview_data.header_row; - this.fields = this.preview_data.fields; this.data = this.preview_data.data; this.make_wrapper(); this.prepare_columns(); @@ -57,62 +55,52 @@ frappe.data_import.ImportPreview = class ImportPreview { } prepare_columns() { - this.columns = this.fields.map((df, i) => { + this.columns = this.preview_data.columns.map((col, i) => { + let df = col.df; let column_width = 120; - let header_row_index = i - 1; - if (df.skip_import) { - let is_sr = df.label === 'Sr. No'; + if (col.header_title === 'Sr. No') { + return { + id: 'srno', + name: 'Sr. No', + content: 'Sr. No', + editable: false, + focusable: false, + align: 'left', + width: 60 + } + } + + if (col.skip_import) { let show_warnings_button = ``; - if (!df.parent) { + if (!col.df) { // increase column width for unidentified columns column_width += 50 } - let column_title = is_sr - ? df.label - : ` - ${df.header_title || `${__('Untitled Column')}`} - ${!df.parent ? show_warnings_button : ''} - `; + let column_title = ` + ${col.header_title || `${__('Untitled Column')}`} + ${!col.df ? show_warnings_button : ''} + `; return { id: frappe.utils.get_random(6), - name: df.label, + name: col.header_title || df.label, content: column_title, skip_import: true, editable: false, focusable: false, align: 'left', - header_row_index, - width: is_sr ? 60 : column_width, - format: (value, row, column, data) => { - let html = `
    ${value}
    `; - if (is_sr && this.is_row_imported(row)) { - html = ` -
    ${SVG_ICONS['checkbox-circle-line'] + - html}
    - `; - } - return html; - } + width: column_width, + format: value => `
    ${value}
    ` }; } - let column_title = df.label; - if (this.doctype !== df.parent) { - column_title = `${df.label} (${df.parent})`; - } - let meta = frappe.get_meta(this.doctype); - if (meta.autoname === `field:${df.fieldname}`) { - column_title = `ID (${df.label})`; - } return { id: df.fieldname, - name: column_title, - content: `${df.header_title || df.label}`, + name: col.header_title, + content: `${col.header_title || df.label}`, df: df, editable: false, align: 'left', - header_row_index, width: column_width }; }); @@ -215,11 +203,11 @@ frappe.data_import.ImportPreview = class ImportPreview { } export_errored_rows() { - this.events.export_errored_rows(); + this.frm.trigger('export_errored_rows'); } show_warnings() { - this.events.show_warnings(); + this.frm.scroll_to_field('import_warnings'); } show_column_warning(_, $target) { @@ -234,11 +222,12 @@ frappe.data_import.ImportPreview = class ImportPreview { doctype: this.doctype }); let changed = []; - let fields = this.fields.map((df, i) => { - if (df.label === 'Sr. No') return []; + let fields = this.preview_data.columns.map((col, i) => { + let df = col.df; + if (col.header_title === 'Sr. No') return []; let fieldname; - if (df.skip_import) { + if (!df) { fieldname = null; } else { fieldname = df.parent === this.doctype @@ -249,7 +238,7 @@ frappe.data_import.ImportPreview = class ImportPreview { { label: '', fieldtype: 'Data', - default: df.header_title, + default: col.header_title, fieldname: `Column ${i}`, read_only: 1 }, diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index 3e9dacbd05..153bbadc07 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -92,11 +92,14 @@ frappe.ui.form.Dashboard = Class.extend({ show_progress: function(title, percent, message) { this._progress_map = this._progress_map || {}; - if (!this._progress_map[title]) { - const progress_chart = this.add_progress(title, percent, message); + let progress_chart = this._progress_map[title]; + // create a new progress chart if it doesnt exist + // or the previous one got detached from the DOM + if (!progress_chart || progress_chart.parent().length == 0) { + progress_chart = this.add_progress(title, percent, message); this._progress_map[title] = progress_chart; } - let progress_chart = this._progress_map[title]; + if (!$.isArray(percent)) { percent = this.format_percent(title, percent); } From 9f9a1ecba388bf973d5a605083e6021aa2ff0aa2 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 19:20:41 +0530 Subject: [PATCH 73/83] fix: Run in background if not in developer_mode or not in_test --- frappe/core/doctype/data_import_beta/data_import_beta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 705013d9c5..0edbdaa766 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -48,7 +48,7 @@ class DataImportBeta(Document): event="data_import", job_name=self.name, data_import=self.name, - now=True, + now=frappe.conf.developer_mode or frappe.flags.in_test, ) def export_errored_rows(self): From c917dda03ae18053842fbedad3595c627cba36d3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 19:29:54 +0530 Subject: [PATCH 74/83] fix: Skip invalid link values --- frappe/core/doctype/data_import/importer_new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index ca56dce16a..9612cc1ffb 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -686,7 +686,7 @@ class Importer: for fieldname, value in doc.items(): meta = frappe.get_meta(doctype) df = meta.get_field(fieldname) - if df.fieldtype == "Link": + if df.fieldtype == "Link" and value not in INVALID_VALUES: link_values.append([df.options, value]) elif df.fieldtype in table_fields: for row in value: @@ -696,7 +696,7 @@ class Importer: for link_doctype, link_value in link_values: d = self.missing_link_values.get(link_doctype) - if d.one_mandatory and link_value in d.missing_values: + if d and d.one_mandatory and link_value in d.missing_values: meta = frappe.get_meta(link_doctype) # find the autoname field if meta.autoname and meta.autoname.startswith("field:"): From 63c4991ef938f9df384424102ff294664bf65234 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 19:30:06 +0530 Subject: [PATCH 75/83] fix: Refresh import preview on form change --- frappe/core/doctype/data_import_beta/data_import_beta.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index f2ff1eb11b..ca8b472df5 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -198,7 +198,7 @@ frappe.ui.form.on('Data Import Beta', { show_import_preview(frm, preview_data) { let import_log = JSON.parse(frm.doc.import_log || '[]'); - if (frm.import_preview) { + if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) { frm.import_preview.preview_data = preview_data; frm.import_preview.import_log = import_log; frm.import_preview.refresh(); From 928ce50206b184ee6e63967f9bf7981c2c6c0156 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 29 Sep 2019 22:38:48 +0530 Subject: [PATCH 76/83] style: fix errors reported by codacy --- .../core/doctype/data_import/exporter_new.py | 2 +- .../core/doctype/data_import/importer_new.py | 33 ++++++++++--------- .../data_import_beta/data_import_beta.py | 2 +- .../data_import_beta/data_import_beta_list.js | 2 +- .../js/frappe/data_import/data_exporter.js | 2 +- .../js/frappe/data_import/import_preview.js | 13 ++------ .../public/js/frappe/form/footer/timeline.js | 12 +++---- 7 files changed, 29 insertions(+), 37 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index fec813610d..ca451f6704 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -3,7 +3,6 @@ # MIT License. See license.txt import frappe -from frappe import _ from frappe.model import display_fieldtypes, no_value_fields, table_fields from frappe.utils.csvutils import build_csv_response from frappe.utils.xlsxutils import build_xlsx_response @@ -209,6 +208,7 @@ class Exporter: return data + # pylint: disable=R0201 def remove_empty_rows(self, data): return [row for row in data if any(v not in INVALID_VALUES for v in row)] diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 9612cc1ffb..5ed89bddc8 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -3,26 +3,23 @@ # MIT License. See license.txt import io -import csv import json import timeit import frappe from datetime import datetime from frappe import _ -from frappe.core.doctype.docfield.docfield import DocField -from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT +from frappe.utils import cint, flt from frappe.utils.csvutils import read_csv_content from frappe.utils.xlsxutils import ( read_xlsx_file_from_attached_file, read_xls_file_from_attached_file, ) -from frappe.exceptions import ValidationError, MandatoryError -from frappe.model import display_fieldtypes, no_value_fields, table_fields +from frappe.model import no_value_fields, table_fields INVALID_VALUES = ["", None] MAX_ROWS_IN_PREVIEW = 10 - +# pylint: disable=R0201 class Importer: def __init__(self, doctype, data_import=None, file_path=None, content=None): self.doctype = doctype @@ -355,9 +352,10 @@ class Importer: if column_index == -1: self._guessed_date_formats[fieldname] = None - column_values = map(lambda x: x[column_index], self.data[:PARSE_ROW_COUNT]) - column_values = filter(lambda x: bool(x), column_values) - date_formats = list(map(lambda x: guess_date_format(x), column_values)) + date_values = [ + row[column_index] for row in self.data[:PARSE_ROW_COUNT] if row[column_index] + ] + date_formats = [guess_date_format(d) for d in date_values] if not date_formats: return max_occurred_date_format = max(set(date_formats), key=date_formats.count) @@ -461,7 +459,7 @@ class Importer: # commit after every successful import frappe.db.commit() - except Exception as e: + except Exception: import_log.append( frappe._dict( success=False, @@ -505,7 +503,6 @@ class Importer: Returns the doc, rows, and data without the rows. """ doc = {} - mandatory_fields = [] doctypes = set([col.df.parent for col in self.columns if col.df and col.df.parent]) # first row is included by default @@ -572,10 +569,10 @@ class Importer: # new_doc returns a dict with default values set doc = frappe.new_doc(doctype, as_dict=True) # remove standard fields and __islocal - for key in frappe.model.default_fields + ('__islocal',): + for key in frappe.model.default_fields + ("__islocal",): doc.pop(key, None) - for index, (df, value) in enumerate(zip(docfields, values)): + for df, value in zip(docfields, values): if value in INVALID_VALUES: value = None @@ -609,7 +606,7 @@ class Importer: ) parsed_docs = {} - for row_index, row in enumerate(rows): + for row in rows: for doctype in doctypes: if doctype == self.doctype and parsed_docs.get(doctype): # if parent doc is already parsed from the first row @@ -839,7 +836,9 @@ def guess_date_format(date_string): for f in DATE_FORMATS: try: - parsed_date = datetime.strptime(_date, f) + # if date is parsed without any exception + # capture the date format + datetime.strptime(_date, f) date_format = f break except ValueError: @@ -848,7 +847,9 @@ def guess_date_format(date_string): if _time: for f in TIME_FORMATS: try: - parsed_time = datetime.strptime(_time, f) + # if time is parsed without any exception + # capture the time format + datetime.strptime(_time, f) time_format = f break except ValueError: diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 0edbdaa766..bbe89bdef8 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -23,7 +23,7 @@ class DataImportBeta(Document): if self.import_file: # validate template - i = self.get_importer() + self.get_importer() def get_preview_from_template(self): if not self.import_file: diff --git a/frappe/core/doctype/data_import_beta/data_import_beta_list.js b/frappe/core/doctype/data_import_beta/data_import_beta_list.js index b7674b432d..d36a8024ad 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta_list.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta_list.js @@ -4,7 +4,7 @@ frappe.listview_settings['Data Import Beta'] = { "Pending": "orange", "Partial Success": "orange", "Success": "green", - } + }; return [__(doc.status), colors[doc.status], "status,=," + doc.status]; }, formatters: { diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 39055aed7c..66c9929c59 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -130,7 +130,7 @@ frappe.data_import.DataExporter = class DataExporter { this.filter_group = new frappe.ui.FilterGroup({ parent: this.dialog.get_field('filter_area').$wrapper, doctype: this.doctype, - on_change: e => { + on_change: () => { this.update_record_count_message(); } }); diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 89efb2c69e..f068ce857c 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -3,15 +3,6 @@ import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); -const SVG_ICONS = { - 'checkbox-circle-line': ` - - - - - ` -}; - frappe.data_import.ImportPreview = class ImportPreview { constructor({ wrapper, doctype, preview_data, frm, import_log, events = {} }) { this.wrapper = wrapper; @@ -67,7 +58,7 @@ frappe.data_import.ImportPreview = class ImportPreview { focusable: false, align: 'left', width: 60 - } + }; } if (col.skip_import) { @@ -75,7 +66,7 @@ frappe.data_import.ImportPreview = class ImportPreview { `; if (!col.df) { // increase column width for unidentified columns - column_width += 50 + column_width += 50; } let column_title = ` ${col.header_title || `${__('Untitled Column')}`} diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js index 96481136b4..24f60c7e2c 100644 --- a/frappe/public/js/frappe/form/footer/timeline.js +++ b/frappe/public/js/frappe/form/footer/timeline.js @@ -579,11 +579,11 @@ frappe.ui.form.Timeline = class Timeline { ); // value changed in parent - if(data.changed && data.changed.length) { + if (data.changed && data.changed.length) { var parts = []; data.changed.every(function(p) { - if(p[0]==='docstatus') { - if(p[2]==1) { + if (p[0]==='docstatus') { + if (p[2]==1) { let message = data.data_import ? __('submitted this document {0}', [data_import_link]) : __('submitted this document'); @@ -623,7 +623,7 @@ frappe.ui.form.Timeline = class Timeline { } // value changed in table field - if(data.row_changed && data.row_changed.length) { + if (data.row_changed && data.row_changed.length) { var parts = [], count = 0; data.row_changed.every(function(row) { row[3].every(function(p) { @@ -652,9 +652,9 @@ frappe.ui.form.Timeline = class Timeline { if(parts.length) { let message; if (data.data_import) { - message = __("changed values for {0} {1}", [parts.join(', '), data_import_link]) + message = __("changed values for {0} {1}", [parts.join(', '), data_import_link]); } else { - message = __("changed values for {0}", [parts.join(', ')]) + message = __("changed values for {0}", [parts.join(', ')]); } out.push(me.get_version_comment(version, message)); } From a38aca52c0df8854619729191201f5d7c7867b12 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 12:45:54 +0530 Subject: [PATCH 77/83] fix: Freeze Start Import button on click --- frappe/core/doctype/data_import_beta/data_import_beta.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index ca8b472df5..2eb0bdc498 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -127,11 +127,15 @@ frappe.ui.form.on('Data Import Beta', { }, start_import(frm) { - frm.save().then(() => frm.call('start_import')); + frm.call({ + doc: frm.doc, + method: 'start_import', + btn: frm.page.btn_primary + }); }, download_template(frm) { - if (frm.data_exporter) { + if (frm.data_exporter && frm.data_exporter.doctype === frm.doc.reference_doctype) { frm.data_exporter.dialog.show(); set_export_records(); } else { From d87f60b1773cd79c952b20ac9cd9359c8c62f4ef Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 13:36:31 +0530 Subject: [PATCH 78/83] fix: Child row parsing logic if the row is blank, it's a child row doc if the row has same values as parent row, it's a child row doc if any of those conditions dont match, it's the next doc --- .../core/doctype/data_import/importer_new.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 5ed89bddc8..0e18888195 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -512,19 +512,28 @@ class Importer: # if there are child doctypes, find the subsequent rows if len(doctypes) > 1: # subsequent rows either dont have any parent value set - # or have the same value as the parent + # or have the same value as the parent row # we include a row if either of conditions match - parent_column_index = self.get_first_parent_column_index() - parent_value = first_row[parent_column_index] + parent_column_indexes = [ + col.index + for col in self.columns + if not col.skip_import and col.df and col.df.parent == self.doctype + ] + parent_row_values = [first_row[i] for i in parent_column_indexes] + data_without_first_row = data[1:] - for d in data_without_first_row: - value = d[parent_column_index] - # if value is blank then it's a child row - # if value is same as parent value it's a child row - # if value is different than the parent value, it's the next doc - if value not in INVALID_VALUES and value != parent_value: - break - rows.append(d) + for row in data_without_first_row: + row_values = [row[i] for i in parent_column_indexes] + # if the row is blank, it's a child row doc + if all([v in INVALID_VALUES for v in row_values]): + rows.append(row) + continue + # if the row has same values as parent row, it's a child row doc + if row_values == parent_row_values: + rows.append(row) + continue + # if any of those conditions dont match, it's the next doc + break def get_column_indexes(doctype): return [ From 660a1d43d2a97c64efd802ee24f7487a76321cc0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 13:38:41 +0530 Subject: [PATCH 79/83] fix: Commonify autoname_field code --- .../core/doctype/data_import/importer_new.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 0e18888195..8f45e72924 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -265,15 +265,12 @@ class Importer: # if autoname is based on field # add an entry for "ID (Autoname Field)" - autoname = self.meta.autoname - if autoname and autoname.startswith("field:"): - fieldname = autoname[len("field:") :] - autoname_field = self.meta.get_field(fieldname) - if autoname_field: - out["ID ({})".format(autoname_field.label)] = autoname_field - # ID field should also map to the autoname field - out["ID"] = autoname_field - out["name"] = autoname_field + autoname_field = self.get_autoname_field(self.doctype) + if autoname_field: + out["ID ({})".format(autoname_field.label)] = autoname_field + # ID field should also map to the autoname field + out["ID"] = autoname_field + out["name"] = autoname_field return out @@ -692,6 +689,8 @@ class Importer: for fieldname, value in doc.items(): meta = frappe.get_meta(doctype) df = meta.get_field(fieldname) + if not df: + continue if df.fieldtype == "Link" and value not in INVALID_VALUES: link_values.append([df.options, value]) elif df.fieldtype in table_fields: @@ -705,12 +704,10 @@ class Importer: if d and d.one_mandatory and link_value in d.missing_values: meta = frappe.get_meta(link_doctype) # find the autoname field - if meta.autoname and meta.autoname.startswith("field:"): - autoname_field = meta.autoname[len("field:") :] - else: - autoname_field = "name" + autoname_field = self.get_autoname_field(link_doctype) + name_field = autoname_field.fieldname if autoname_field else "name" new_doc = frappe.new_doc(link_doctype) - new_doc.set(autoname_field, link_value) + new_doc.set(name_field, link_value) new_doc.insert() d.missing_values.remove(link_value) @@ -773,12 +770,9 @@ class Importer: ) def get_id_fieldname(self): - autoname = self.meta.autoname - if autoname and autoname.startswith("field:"): - fieldname = autoname[len("field:") :] - autoname_field = self.meta.get_field(fieldname) - if autoname_field: - return autoname_field.fieldname + autoname_field = self.get_autoname_field(self.doctype) + if autoname_field: + return autoname_field.fieldname return "name" def get_eta(self, current, total, processing_time): @@ -797,6 +791,12 @@ class Importer: mandatory_fields_count += 1 return mandatory_fields_count == 1 + def get_autoname_field(self, doctype): + meta = frappe.get_meta(doctype) + if meta.autoname and meta.autoname.startswith("field:"): + fieldname = meta.autoname[len("field:") :] + return meta.get_field(fieldname) + DATE_FORMATS = [ r"%d-%m-%Y", From b382be67d53305638fd483e9df2ae52bd5a9d83b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 13:39:43 +0530 Subject: [PATCH 80/83] feat: data-import command Use the new Data Import from the command line ``` bench --site sitename data-import --doctype Item --file /path/to/csvfile.csv ``` --- frappe/commands/utils.py | 28 ++++ .../core/doctype/data_import/importer_new.py | 139 +++++++++++++----- 2 files changed, 133 insertions(+), 34 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index b0f70132c1..9a408430e7 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -293,6 +293,33 @@ def import_csv(context, path, only_insert=False, submit_after_import=False, igno frappe.destroy() + +@click.command('data-import') +@click.option('--file', 'file_path', type=click.Path(), required=True, help="Path to import file (.csv, .xlsx)") +@click.option('--doctype', type=str, required=True) +@click.option('--type', 'import_type', type=click.Choice(['Insert', 'Update'], case_sensitive=False), default='Insert', help="Insert New Records or Update Existing Records") +@click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it') +@click.option('--mute-emails', default=True, is_flag=True, help='Mute emails during import') +@pass_context +def data_import(context, file_path, doctype, import_type=None, submit_after_import=False, mute_emails=True): + "Import documents in bulk from CSV or XLSX using data import" + from frappe.core.doctype.data_import.importer_new import Importer + site = get_site(context) + + frappe.init(site=site) + frappe.connect() + + data_import = frappe.new_doc('Data Import Beta') + data_import.submit_after_import = submit_after_import + data_import.mute_emails = mute_emails + data_import.import_type = 'Insert New Records' if import_type.lower() == 'insert' else 'Update Existing Records' + + i = Importer(doctype=doctype, file_path=file_path, data_import=data_import, console=True) + i.import_data() + + frappe.destroy() + + @click.command('bulk-rename') @click.argument('doctype') @click.argument('path') @@ -715,6 +742,7 @@ commands = [ export_json, get_version, import_csv, + data_import, import_doc, make_app, mysql, diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 8f45e72924..8828d9b620 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -3,12 +3,13 @@ # MIT License. See license.txt import io +import os import json import timeit import frappe from datetime import datetime from frappe import _ -from frappe.utils import cint, flt +from frappe.utils import cint, flt, update_progress_bar from frappe.utils.csvutils import read_csv_content from frappe.utils.xlsxutils import ( read_xlsx_file_from_attached_file, @@ -21,9 +22,12 @@ MAX_ROWS_IN_PREVIEW = 10 # pylint: disable=R0201 class Importer: - def __init__(self, doctype, data_import=None, file_path=None, content=None): + def __init__( + self, doctype, data_import=None, file_path=None, content=None, console=False + ): self.doctype = doctype self.template_options = frappe._dict({"remap_column": {}}) + self.console = console if data_import: self.data_import = data_import @@ -47,14 +51,15 @@ class Importer: self.parse_data_from_template() def prepare_content(self, file_path, content): - if self.data_import: + if self.data_import and self.data_import.import_file: file_doc = frappe.get_doc("File", {"file_url": self.data_import.import_file}) content = file_doc.get_content() extension = file_doc.file_name.split(".")[1] if file_path: - self.read_file(file_path) - elif content: + content, extension = self.read_file(file_path) + + if content: self.read_content(content, extension) self.validate_template_content() @@ -67,10 +72,7 @@ class Importer: with io.open(file_path, mode="rb") as f: file_content = f.read() - if extn == "csv": - data = read_csv_content(file_content) - self.header_row = data[0] - self.data = data[1:] + return file_content, extn def read_content(self, content, extension): if extension == "csv": @@ -365,7 +367,8 @@ class Importer: frappe.cache().hdel("lang", frappe.session.user) frappe.set_user_lang(frappe.session.user) - self.data_import.db_set("template_warnings", "") + if not self.console: + self.data_import.db_set("template_warnings", "") # set flags frappe.flags.in_import = True @@ -380,10 +383,13 @@ class Importer: # dont import if there are non-ignorable warnings warnings = [w for w in self.warnings if w.get("type") != "info"] if warnings: - self.data_import.db_set("template_warnings", json.dumps(warnings)) - frappe.publish_realtime( - "data_import_refresh", {"data_import": self.data_import.name} - ) + if self.console: + self.print_grouped_warnings(warnings) + else: + self.data_import.db_set("template_warnings", json.dumps(warnings)) + frappe.publish_realtime( + "data_import_refresh", {"data_import": self.data_import.name} + ) return # setup import log @@ -403,8 +409,6 @@ class Importer: imported_rows += log.row_indexes # start import - print("Importing {0} rows...".format(len(self.rows))) - total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 @@ -431,7 +435,6 @@ class Importer: continue try: - print("Importing", doc) start = timeit.default_timer() doc = self.process_doc(doc) processing_time = timeit.default_timer() - start @@ -450,6 +453,12 @@ class Importer: "eta": eta, }, ) + if self.console: + update_progress_bar( + "Importing {0} records".format(total_payload_count), + current_index, + total_payload_count, + ) import_log.append( frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) ) @@ -478,8 +487,11 @@ class Importer: else: status = "Success" - self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) + if self.console: + self.print_import_log(import_log) + else: + self.data_import.db_set("status", status) + self.data_import.db_set("import_log", json.dumps(import_log)) frappe.flags.in_import = False frappe.flags.mute_emails = False @@ -499,7 +511,6 @@ class Importer: Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. Returns the doc, rows, and data without the rows. """ - doc = {} doctypes = set([col.df.parent for col in self.columns if col.df and col.df.parent]) # first row is included by default @@ -590,9 +601,14 @@ class Importer: return doc def check_mandatory_fields(doctype, doc, row_number): + # check if mandatory fields are set (except table fields) meta = frappe.get_meta(doctype) fields = [ - df for df in meta.fields if df.reqd and doc.get(df.fieldname) in INVALID_VALUES + df + for df in meta.fields + if df.fieldtype not in table_fields + and df.reqd + and doc.get(df.fieldname) in INVALID_VALUES ] if not fields: @@ -633,9 +649,11 @@ class Importer: parsed_docs[doctype] = parsed_docs.get(doctype, []) parsed_docs[doctype].append(doc) + # build the doc with children + doc = {} for doctype, docs in parsed_docs.items(): if doctype == self.doctype: - doc = docs[0] + doc.update(docs[0]) else: table_dfs = self.meta.get( "fields", {"options": doctype, "fieldtype": ["in", table_fields]} @@ -644,19 +662,31 @@ class Importer: table_field = table_dfs[0] doc[table_field.fieldname] = docs - return doc, rows, data[len(rows) :] + # check if there is atleast one row for mandatory table fields + mandatory_table_fields = [ + df + for df in self.meta.fields + if df.fieldtype in table_fields and df.reqd and len(doc.get(df.fieldname, [])) == 0 + ] + if len(mandatory_table_fields) == 1: + self.warnings.append( + { + "row": first_row[0], + "message": _("There should be atleast one row for {0} table").format( + mandatory_table_fields[0].label + ), + } + ) + elif mandatory_table_fields: + fields_string = ", ".join([df.label for df in mandatory_table_fields]) + self.warnings.append( + { + "row": first_row[0], + "message": _("There should be atleast one row for the following tables: {0}").format(fields_string), + } + ) - def get_first_parent_column_index(self): - """ - Returns the first column's index which must be one of the parent columns - """ - # find a parent column - parent_column_index = -1 - for col in self.columns: - if not col.skip_import and col.df and col.df.parent == self.doctype: - parent_column_index = col.index - break - return parent_column_index + return doc, rows, data[len(rows) :] def process_doc(self, doc): import_type = self.data_import.import_type @@ -797,6 +827,47 @@ class Importer: fieldname = meta.autoname[len("field:") :] return meta.get_field(fieldname) + def print_grouped_warnings(self, warnings): + warnings_by_row = {} + other_warnings = [] + for w in warnings: + if w.get("row"): + warnings_by_row.setdefault(w.get("row"), []).append(w) + else: + other_warnings.append(w) + + for row_number, warnings in warnings_by_row.items(): + print("Row {0}".format(row_number)) + for w in warnings: + print(w.get("message")) + + for w in other_warnings: + print(w.get("message")) + + def print_import_log(self, import_log): + failed_records = [l for l in import_log if not l.success] + successful_records = [l for l in import_log if l.success] + + if successful_records: + print( + "Successfully imported {1} records out of {1}".format( + len(successful_records), len(import_log) + ) + ) + + if failed_records: + print("Failed to import {0} records".format(len(failed_records))) + file_name = '{0}_import_on_{1}.txt'.format(self.doctype, frappe.utils.now()) + print('Check {0} for errors'.format(os.path.join('sites', file_name))) + text = "" + for w in failed_records: + text += "Row Indexes: {0}\n".format(str(w.get('row_indexes', []))) + text += "Messages:\n{0}\n".format('\n'.join(w.get('messages', []))) + text += "Traceback:\n{0}\n\n".format(w.get('exception')) + + with open(file_name, 'w') as f: + f.write(text) + DATE_FORMATS = [ r"%d-%m-%Y", From 4277da02ebdc8344035c8567cb8b22b79d6b1499 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 19:17:27 +0530 Subject: [PATCH 81/83] test: Make tests pass --- .../core/doctype/data_import/importer_new.py | 6 +++ .../doctype/data_import/test_importer_new.py | 49 ++++++++++++++----- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 8828d9b620..f5ca47abc8 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -51,6 +51,7 @@ class Importer: self.parse_data_from_template() def prepare_content(self, file_path, content): + extension = None if self.data_import and self.data_import.import_file: file_doc = frappe.get_doc("File", {"file_url": self.data_import.import_file}) content = file_doc.get_content() @@ -59,6 +60,9 @@ class Importer: if file_path: content, extension = self.read_file(file_path) + if not extension: + extension = "csv" + if content: self.read_content(content, extension) @@ -497,6 +501,8 @@ class Importer: frappe.flags.mute_emails = False frappe.publish_realtime("data_import_refresh", {"data_import": self.data_import.name}) + return import_log + def get_payloads_for_import(self): payloads = [] # make a copy diff --git a/frappe/core/doctype/data_import/test_importer_new.py b/frappe/core/doctype/data_import/test_importer_new.py index 396cac577a..d6349daa55 100644 --- a/frappe/core/doctype/data_import/test_importer_new.py +++ b/frappe/core/doctype/data_import/test_importer_new.py @@ -29,25 +29,50 @@ est phasellus sit amet,5/20/2019,52,True,invalid value class TestImporter(unittest.TestCase): def test_should_skip_empty_rows(self): - i = Importer('Web Page', content=content_empty_rows) - i.import_data() - self.assertEqual(len(i.skipped_rows), 1) + i = self.get_importer('Web Page', content=content_empty_rows) + payloads = i.get_payloads_for_import() + row_to_be_imported = [] + for p in payloads: + row_to_be_imported += [row[0] for row in p.rows] + self.assertEqual(len(row_to_be_imported), 2) def test_should_throw_if_mandatory_is_missing(self): - i = Importer('Web Page', content=content_mandatory_missing) - self.assertRaises(frappe.MandatoryError, i.import_data) + i = self.get_importer('Web Page', content=content_mandatory_missing) + i.import_data() + warning = i.warnings[0] + self.assertTrue('Title is a mandatory field' in warning['message']) def test_should_convert_value_based_on_fieldtype(self): - i = Importer('Web Page', content=content_convert_value) - doc = i.parse_data_for_import(i.data[0], 0) + i = self.get_importer('Web Page', content=content_convert_value) + payloads = i.get_payloads_for_import() + doc = payloads[0].doc - self.assertEqual(type(doc.show_title), int) - self.assertEqual(type(doc.idx), int) - self.assertEqual(type(doc.start_date), datetime.datetime) + self.assertEqual(type(doc['show_title']), int) + self.assertEqual(type(doc['idx']), int) + self.assertEqual(type(doc['start_date']), datetime.datetime) def test_should_ignore_invalid_columns(self): - i = Importer('Web Page', content=content_invalid_column) - doc = i.parse_data_for_import(i.data[0], 0) + i = self.get_importer('Web Page', content=content_invalid_column) + payloads = i.get_payloads_for_import() + doc = payloads[0].doc self.assertTrue('invalid_column' not in doc) self.assertTrue('title' in doc) + + def test_should_import_valid_template(self): + title = 'est phasellus sit amet {0}'.format(frappe.utils.random_string(8)) + content_valid_content = '''title,start_date,idx,show_title +{0},5/20/2019,52,1'''.format(title) + i = self.get_importer('Web Page', content=content_valid_content) + import_log = i.import_data() + log = import_log[0] + self.assertTrue(log.success) + doc = frappe.get_doc('Web Page', { 'title': title }) + self.assertEqual(frappe.utils.get_datetime_str(doc.start_date), + frappe.utils.get_datetime_str('2019-05-20')) + + def get_importer(self, doctype, content): + data_import = frappe.new_doc('Data Import Beta') + data_import.import_type = 'Insert New Records' + i = Importer(doctype, content=content, data_import=data_import) + return i From 15c48079df90bb21da5f82ee6a9dd9a2143978d0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 19:32:21 +0530 Subject: [PATCH 82/83] tests: Make exporter tests pass --- frappe/core/doctype/data_import/exporter_new.py | 5 +++-- frappe/core/doctype/data_import/test_exporter_new.py | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index ca451f6704..21f5fc4bd2 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -95,11 +95,12 @@ class Exporter: if self.export_fields == "Mandatory": fields = [df for df in fields if df.reqd] + if self.export_fields == "All": + fields = list(fields) + elif isinstance(self.export_fields, dict): whitelist = self.export_fields.get(doctype, []) fields = [df for df in fields if df.fieldname in whitelist] - else: - fields = [df for df in fields if df.reqd or df.in_list_view or df.bold] name_field = frappe._dict( { diff --git a/frappe/core/doctype/data_import/test_exporter_new.py b/frappe/core/doctype/data_import/test_exporter_new.py index 32cd7a6a46..dee5af543c 100644 --- a/frappe/core/doctype/data_import/test_exporter_new.py +++ b/frappe/core/doctype/data_import/test_exporter_new.py @@ -13,7 +13,7 @@ class TestExporter(unittest.TestCase): e = Exporter('Web Page', export_fields='Mandatory') csv_array = e.get_csv_array() header_row = csv_array[0] - self.assertEqual(header_row, ['ID (name)', 'Title (title)']) + self.assertEqual(header_row, ['ID', 'Title']) def test_exports_all_fields(self): @@ -24,11 +24,13 @@ class TestExporter(unittest.TestCase): def test_exports_selected_fields(self): - export_fields = ['title', 'route', 'published'] + export_fields = { + 'Web Page': ['title', 'route', 'published'] + } e = Exporter('Web Page', export_fields=export_fields) csv_array = e.get_csv_array() header = csv_array[0] - self.assertEqual(header, ['Title (title)', 'Route (route)', 'Published (published)']) + self.assertEqual(header, ['ID', 'Title', 'Route', 'Published']) def test_exports_data(self): From d3b3d0c38d770bf930956b50426af03c267954ea Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 30 Sep 2019 20:08:28 +0530 Subject: [PATCH 83/83] style: Fix codacy errors --- frappe/core/doctype/data_import/importer_new.py | 3 +-- frappe/public/js/frappe/data_import/data_exporter.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index f5ca47abc8..6fccbc89ef 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -738,7 +738,6 @@ class Importer: for link_doctype, link_value in link_values: d = self.missing_link_values.get(link_doctype) if d and d.one_mandatory and link_value in d.missing_values: - meta = frappe.get_meta(link_doctype) # find the autoname field autoname_field = self.get_autoname_field(link_doctype) name_field = autoname_field.fieldname if autoname_field else "name" @@ -856,7 +855,7 @@ class Importer: if successful_records: print( - "Successfully imported {1} records out of {1}".format( + "Successfully imported {0} records out of {1}".format( len(successful_records), len(import_log) ) ) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 66c9929c59..d0bf794df6 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -212,7 +212,7 @@ frappe.data_import.DataExporter = class DataExporter { count_method[export_records]().then(value => { let message = ''; - value = parseInt(value); + value = parseInt(value, 10); if (value === 0) { message = __('No records will be exported'); } else if (value === 1) {