diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index 0f12cef52d..df91543049 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -6,9 +6,18 @@ import frappe from frappe import _ from frappe.model import display_fieldtypes, no_value_fields, table_fields from frappe.utils.csvutils import build_csv_response +from .importer_new import INVALID_VALUES + class Exporter: - def __init__(self, doctype, export_fields=None, export_data=False, export_filters=None, file_type='CSV'): + def __init__( + self, + doctype, + export_fields=None, + export_data=False, + export_filters=None, + file_type="CSV", + ): """ Exports records of a DocType for use with Importer :param doctype: Document Type to export @@ -20,6 +29,7 @@ class Exporter: self.doctype = doctype self.meta = frappe.get_meta(doctype) self.export_fields = export_fields + self.export_filters = export_filters # this will contain the csv content self.csv_array = [] @@ -30,40 +40,32 @@ class Exporter: self.add_header() if export_data: - self.data = self.get_data_to_export(export_filters) + self.data = self.get_data_to_export() else: self.data = [] self.add_data() - def get_all_exportable_fields(self): return self.get_exportable_parent_fields() + self.get_exportable_children_fields() - def get_exportable_parent_fields(self): - name_field = frappe._dict({ - 'fieldtype': 'Data', - 'fieldname': 'name', - 'label': 'ID', - 'reqd': 1, - 'parent': self.doctype - }) - parent_fields = self.get_exportable_fields(self.doctype) # if autoname is based on field # then merge ID and the field column title as "ID (Autoname Field)" autoname = self.meta.autoname - if autoname and autoname.startswith('field:'): - fieldname = autoname[len('field:'):] + if autoname and autoname.startswith("field:"): + fieldname = autoname[len("field:") :] autoname_field = self.meta.get_field(fieldname) if autoname_field: - name_field.label = 'ID ({})'.format(autoname_field.label) + name_field = parent_fields[0] + name_field.label = "ID ({})".format(autoname_field.label) # remove the autoname field as it is a duplicate of ID field - parent_fields = [df for df in parent_fields if df.fieldname != autoname_field.fieldname] - - return [name_field] + parent_fields + parent_fields = [ + df for df in parent_fields if df.fieldname != autoname_field.fieldname + ] + return parent_fields def get_exportable_children_fields(self): children = [df.options for df in self.meta.fields if df.fieldtype in table_fields] @@ -73,8 +75,9 @@ class Exporter: return children_fields - def get_exportable_fields(self, doctype): + fields = [] + def is_exportable(df): return ( df.fieldtype not in display_fieldtypes @@ -87,83 +90,146 @@ class Exporter: # filter out layout fields fields = [df for df in meta.fields if is_exportable(df)] - if self.export_fields == 'All': - return fields + if self.export_fields == "Mandatory": + fields = [df for df in fields if df.reqd] - if self.export_fields == 'Mandatory': - return [df for df in fields if df.reqd] - - if isinstance(self.export_fields, dict): + elif isinstance(self.export_fields, dict): whitelist = self.export_fields.get(doctype, []) - return [df for df in fields if df.fieldname in whitelist] + fields = [df for df in fields if df.fieldname in whitelist] + else: + fields = [df for df in fields if df.reqd or df.in_list_view or df.bold] - return [df for df in fields if df.reqd or df.in_list_view or df.bold] + name_field = frappe._dict( + { + "fieldtype": "Data", + "fieldname": "name", + "label": "ID", + "reqd": 1, + "parent": doctype, + } + ) + if fields: + return [name_field] + fields + else: + return [] - def get_data_to_export(self, filters=None): + def get_data_to_export(self): frappe.permissions.can_export(self.doctype, raise_exception=True) def get_column_name(df): - return '`tab{0}`.`{1}`'.format(df.parent, df.fieldname) + return "`tab{0}`.`{1}`".format(df.parent, df.fieldname) - fieldnames = [get_column_name(df) for df in self.fields] + fields = [get_column_name(df) for df in self.fields] + filters = self.export_filters if self.meta.is_nested_set(): - order_by = '`tab{0}`.`lft` ASC'.format(self.doctype) + order_by = "`tab{0}`.`lft` ASC".format(self.doctype) else: - order_by = '`tab{0}`.`creation` DESC'.format(self.doctype) + order_by = "`tab{0}`.`creation` DESC".format(self.doctype) - data = frappe.db.get_list(self.doctype, + data = frappe.db.get_list( + self.doctype, filters=filters, - fields=fieldnames, + fields=fields, limit_page_length=None, order_by=order_by, as_list=1, ) - return self.remove_duplicate_parent_values(data) + data = self.remove_duplicate_values(data) + data = self.remove_row_gaps(data) + data = self.remove_empty_rows(data) - def remove_duplicate_parent_values(self, data): + return data + + def remove_duplicate_values(self, data): out = [] - parent_fields = self.get_exportable_parent_fields() - parent_fields_count = len(parent_fields) + doctypes = set([df.parent for df in self.fields]) + + def name_exists_in_column_before_row(name, column_index, row_index): + column_values = [row[column_index] for i, row in enumerate(data) if i < row_index] + return name in column_values + + for i, row in enumerate(data): + # first row is fine + if i == 0: + out.append(row) + continue - current_name = None - # first column is always name, can be used to find duplicate rows - for row in data: row = list(row) - name = row[0] + for doctype in doctypes: + name_index = self.get_name_column_index(doctype) + name = row[name_index] + column_indexes = self.get_column_indexes(doctype) - if name != current_name: - current_name = name - out.append(row) + if name_exists_in_column_before_row(name, name_index, i): + # remove the values from the row + row = [None if i in column_indexes else d for i, d in enumerate(row)] - elif name == current_name: - # remove the parent values from the row - row = row[parent_fields_count:] - # replace them with None values and add back the row contents - row = [None] * parent_fields_count + row - out.append(row) + out.append(row) return out + def remove_row_gaps(self, data): + doctypes = set([df.parent for df in self.fields if df.parent != self.doctype]) + + def get_nearest_empty_row_index(col_index, row_index): + col_values = [row[col_index] for row in data] + i = row_index - 1 + while not col_values[i]: + i = i - 1 + out = i + 1 + if row_index != out: + return out + + for i, row in enumerate(data): + # if this is the row that contains parent values then skip + if row[0]: + continue + + for doctype in doctypes: + name_index = self.get_name_column_index(doctype) + name = row[name_index] + column_indexes = self.get_column_indexes(doctype) + + if not name: + continue + + row_index = get_nearest_empty_row_index(name_index, i) + if row_index: + for col_index in column_indexes: + data[row_index][col_index] = row[col_index] + row[col_index] = None + + return data + + def remove_empty_rows(self, data): + return [row for row in data if any(v not in INVALID_VALUES for v in row)] + + def get_name_column_index(self, doctype): + for i, df in enumerate(self.fields): + if df.parent == doctype and df.fieldname == 'name': + return i + return -1 + + def get_column_indexes(self, doctype): + return [i for i, df in enumerate(self.fields) if df.parent == doctype] def add_header(self): def get_label(df): if df.parent == self.doctype: return df.label else: - return '{0} ({1})'.format(df.label, df.parent) + return "{0} ({1})".format(df.label, df.parent) header = [get_label(df) for df in self.fields] self.csv_array.append(header) - def add_data(self): self.csv_array += self.data - def get_csv_array(self): return self.csv_array diff --git a/frappe/public/js/frappe/data_import/column_picker_fields.js b/frappe/public/js/frappe/data_import/column_picker_fields.js index 47aab004cd..36cbf3c413 100644 --- a/frappe/public/js/frappe/data_import/column_picker_fields.js +++ b/frappe/public/js/frappe/data_import/column_picker_fields.js @@ -4,10 +4,7 @@ export default class ColumnPickerFields extends frappe.views.ReportView { get_fields_as_options() { let column_map = this.get_columns_for_picker(); let doctypes = [this.doctype].concat( - ...frappe.meta - .get_table_fields(this.doctype) - .filter(df => !df.hidden) - .map(df => df.options) + ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) ); // flatten array return [].concat( diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 83e276bd1a..e5c234abe4 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -3,6 +3,7 @@ frappe.provide('frappe.data_import'); frappe.data_import.DataExporter = class DataExporter { constructor(doctype) { + frappe.data_exporter = this; this.doctype = doctype; frappe.model.with_doctype(doctype, () => { this.make_dialog(); @@ -11,10 +12,7 @@ frappe.data_import.DataExporter = class DataExporter { make_dialog() { let doctypes = [this.doctype].concat( - ...frappe.meta - .get_table_fields(this.doctype) - .filter(df => !df.hidden) - .map(df => df.options) + ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) ); this.dialog = new frappe.ui.Dialog({ diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 0a2a1d6072..6503f1c7ac 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -772,8 +772,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { get_columns_for_picker() { let out = {}; - const standard_fields_filter = df => - !in_list(frappe.model.no_value_type, df.fieldtype) && !df.report_hide; + const standard_fields_filter = df => !in_list(frappe.model.no_value_type, df.fieldtype); let doctype_fields = frappe.meta.get_docfields(this.doctype).filter(standard_fields_filter); @@ -786,8 +785,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { out[this.doctype] = doctype_fields; - const table_fields = frappe.meta.get_table_fields(this.doctype) - .filter(df => !df.hidden); + const table_fields = frappe.meta.get_table_fields(this.doctype); table_fields.forEach(df => { const cdt = df.options;