fix: Column Mapping
- Remove double headers - Indicate match with indicators - Column Mapping Dialog - Handle untitled columns
This commit is contained in:
Faris Ansari
parent
e09b45ed81
commit
1bfd7d3731
4 changed files with 143 additions and 118 deletions
|
|
@ -9,6 +9,7 @@ import timeit
|
|||
import frappe
|
||||
from datetime import datetime
|
||||
from frappe import _
|
||||
from frappe.core.doctype.docfield.docfield import DocField
|
||||
from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT
|
||||
from frappe.utils.csvutils import read_csv_content
|
||||
from frappe.utils.xlsxutils import (
|
||||
|
|
@ -119,6 +120,23 @@ class Importer:
|
|||
|
||||
def get_data_for_import_preview(self):
|
||||
out = self.get_parsed_data_from_template()
|
||||
|
||||
# prepare fields
|
||||
fields = []
|
||||
for df in out.fields:
|
||||
header_title = df.header_title
|
||||
skip_import = df.skip_import
|
||||
if isinstance(df, DocField):
|
||||
field = df.as_dict()
|
||||
else:
|
||||
field = df
|
||||
field.update({
|
||||
'header_title': header_title,
|
||||
'skip_import': skip_import
|
||||
})
|
||||
fields.append(field)
|
||||
out.fields = fields
|
||||
|
||||
if len(out.data) > MAX_ROWS_IN_PREVIEW:
|
||||
out.data = []
|
||||
out.max_rows_exceeded = True
|
||||
|
|
@ -146,30 +164,37 @@ class Importer:
|
|||
|
||||
df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching()
|
||||
|
||||
for i, value in enumerate(self.header_row):
|
||||
for i, header_title in enumerate(self.header_row):
|
||||
header_row_index = str(i)
|
||||
if remap_column.get(header_row_index):
|
||||
column_name = value
|
||||
value = remap_column.get(header_row_index)
|
||||
fieldname = remap_column.get(header_row_index)
|
||||
df = df_by_labels_and_fieldnames.get(fieldname)
|
||||
warnings.append(
|
||||
_("Column {0}: Mapping column {1} to field {2}").format(
|
||||
i, frappe.bold(column_name), frappe.bold(value)
|
||||
i, frappe.bold(header_title or '<i>Untitled Column</i>'), frappe.bold(df.label)
|
||||
)
|
||||
)
|
||||
else:
|
||||
df = df_by_labels_and_fieldnames.get(header_title)
|
||||
|
||||
field = df_by_labels_and_fieldnames.get(value)
|
||||
if not field or i in skip_import:
|
||||
field = frappe._dict({"label": value, "skip_import": True})
|
||||
if value and i not in skip_import:
|
||||
warnings.append(
|
||||
_("Column {0}: Cannot match column {1} with any field").format(
|
||||
i, frappe.bold(value)
|
||||
)
|
||||
if not df:
|
||||
field = frappe._dict(header_title=header_title, skip_import=True)
|
||||
else:
|
||||
field = df
|
||||
field.header_title = header_title
|
||||
field.skip_import = False
|
||||
|
||||
if i in skip_import:
|
||||
field.skip_import = True
|
||||
warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(header_title)))
|
||||
elif header_title and not df:
|
||||
warnings.append(
|
||||
_("Column {0}: Cannot match column {1} with any field").format(
|
||||
i, frappe.bold(header_title)
|
||||
)
|
||||
elif i in skip_import:
|
||||
warnings.append(_("Column {0}: Skipping column {1}").format(i, frappe.bold(value)))
|
||||
else:
|
||||
warnings.append(_("Column {0}: Skipping untitled column").format(i))
|
||||
)
|
||||
elif not header_title and not df:
|
||||
warnings.append(_("Column {0}: Skipping untitled column").format(i))
|
||||
fields.append(field)
|
||||
|
||||
return fields, warnings
|
||||
|
|
@ -442,7 +467,9 @@ class Importer:
|
|||
|
||||
# set status
|
||||
failures = [l for l in import_log if l.get("success") == False]
|
||||
if len(failures) > 0:
|
||||
if len(failures) == total_payload_count:
|
||||
status = "Pending"
|
||||
elif len(failures) > 0:
|
||||
status = "Partial Success"
|
||||
else:
|
||||
status = "Success"
|
||||
|
|
|
|||
|
|
@ -59,10 +59,13 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
frm.trigger('show_import_log');
|
||||
frm.trigger('toggle_submit_after_import');
|
||||
|
||||
if (frm.doc.status === 'Success') {
|
||||
if (frm.doc.import_log && frm.doc.import_log !== '[]') {
|
||||
// set form as readonly
|
||||
frm.fields.forEach(f => f.df.read_only = 1);
|
||||
frm.disable_save();
|
||||
}
|
||||
|
||||
if (frm.doc.status === 'Success') {
|
||||
frm.events.show_success_message(frm);
|
||||
} else {
|
||||
if (!frm.is_new() && frm.doc.import_file) {
|
||||
|
|
@ -144,8 +147,6 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
.call({
|
||||
doc: frm.doc,
|
||||
method: 'get_preview_from_template',
|
||||
freeze: true,
|
||||
freeze_message: __('Preparing Preview...'),
|
||||
error_handlers: {
|
||||
TimestampMismatchError() {
|
||||
// ignore this error
|
||||
|
|
@ -179,23 +180,19 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
import_log,
|
||||
warnings,
|
||||
events: {
|
||||
remap_column(header_row_index, fieldname) {
|
||||
remap_column(changed_map) {
|
||||
let template_options = JSON.parse(frm.doc.template_options || '{}');
|
||||
template_options.remap_column = template_options.remap_column || {};
|
||||
template_options.remap_column[header_row_index] = fieldname;
|
||||
Object.assign(template_options.remap_column, changed_map);
|
||||
|
||||
// if the column is remapped, remove it from skip_import
|
||||
if (
|
||||
template_options.skip_import &&
|
||||
template_options.skip_import.includes(header_row_index)
|
||||
) {
|
||||
if (template_options.skip_import) {
|
||||
template_options.skip_import = template_options.skip_import.filter(
|
||||
d => d !== header_row_index
|
||||
d => !Object.keys(template_options.remap_column).includes(cstr(d))
|
||||
);
|
||||
}
|
||||
frm.set_value('template_options', JSON.stringify(template_options));
|
||||
frm.save().then(() => {
|
||||
frm.trigger('import_file');
|
||||
});
|
||||
frm.save().then(() => frm.trigger('import_file'));
|
||||
},
|
||||
|
||||
skip_import(header_row_index) {
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
import ColumnManager from 'frappe-datatable/src/columnmanager';
|
||||
|
||||
export default function(header_row) {
|
||||
return class CustomColumnManager extends ColumnManager {
|
||||
getHeaderHTML(columns) {
|
||||
let html = super.getHeaderHTML(columns);
|
||||
|
||||
let header_row_columns = [
|
||||
{
|
||||
id: '_checkbox',
|
||||
colIndex: 0,
|
||||
format: () => ''
|
||||
}
|
||||
// {
|
||||
// id: 'Sr. No',
|
||||
// colIndex: 1,
|
||||
// format: () => ''
|
||||
// }
|
||||
].concat(
|
||||
...header_row.map((col, i) => {
|
||||
return {
|
||||
id: col,
|
||||
name: col,
|
||||
align: 'left',
|
||||
dropdown: false,
|
||||
content: col,
|
||||
colIndex: i + 1
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
let header_row_html = this.rowmanager.getRowHTML(header_row_columns, {
|
||||
rowIndex: 'header-row'
|
||||
});
|
||||
return header_row_html + html;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import DataTable from 'frappe-datatable';
|
||||
import get_custom_column_manager from './custom_column_manager';
|
||||
import ColumnPickerFields from './column_picker_fields';
|
||||
|
||||
frappe.provide('frappe.data_import');
|
||||
|
|
@ -47,7 +46,11 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
<div>
|
||||
<div class="warnings text-muted"></div>
|
||||
<div class="table-preview"></div>
|
||||
<div class="table-actions margin-top"></div>
|
||||
<div class="table-actions margin-top">
|
||||
<button class="btn btn-sm btn-default" data-action="show_column_mapper">
|
||||
${__('Map Columns')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
`);
|
||||
frappe.utils.bind_actions_with_class(this.wrapper, this);
|
||||
|
|
@ -61,18 +64,23 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
this.columns = this.fields.map((df, i) => {
|
||||
let header_row_index = i - 1;
|
||||
if (df.skip_import) {
|
||||
let is_sr = df.label === 'Sr. No';
|
||||
let column_title = is_sr
|
||||
? df.label
|
||||
: `<span class="indicator red">${df.header_title || `<i>${__('Untitled Column')}</i>`}</span>`;
|
||||
return {
|
||||
id: frappe.utils.get_random(6),
|
||||
name: df.label,
|
||||
content: column_title,
|
||||
skip_import: true,
|
||||
editable: false,
|
||||
focusable: false,
|
||||
align: 'left',
|
||||
header_row_index,
|
||||
width: df.label === 'Sr. No' ? 60 : column_width,
|
||||
width: is_sr ? 60 : column_width,
|
||||
format: (value, row, column, data) => {
|
||||
let html = `<div class="text-muted">${value}</div>`;
|
||||
if (df.label === 'Sr. No' && this.is_row_imported(row)) {
|
||||
if (is_sr && this.is_row_imported(row)) {
|
||||
html = `
|
||||
<div class="flex justify-between">${SVG_ICONS['checkbox-circle-line'] +
|
||||
html}</div>
|
||||
|
|
@ -94,6 +102,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
return {
|
||||
id: df.fieldname,
|
||||
name: column_title,
|
||||
content: `<span class="indicator green">${df.header_title || df.label}</span>`,
|
||||
df: df,
|
||||
editable: true,
|
||||
align: 'left',
|
||||
|
|
@ -146,20 +155,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
serialNoColumn: false,
|
||||
checkboxColumn: false,
|
||||
pasteFromClipboard: true,
|
||||
noDataMessage: no_data_message,
|
||||
headerDropdown: [
|
||||
{
|
||||
label: __('Remap Column'),
|
||||
action: col => this.remap_column(col)
|
||||
},
|
||||
{
|
||||
label: __('Skip Import'),
|
||||
action: col => this.skip_import(col)
|
||||
}
|
||||
],
|
||||
overrideComponents: {
|
||||
ColumnManager: get_custom_column_manager(this.header_row)
|
||||
}
|
||||
noDataMessage: no_data_message
|
||||
});
|
||||
|
||||
if (this.data.length === 0) {
|
||||
|
|
@ -168,7 +164,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
});
|
||||
}
|
||||
|
||||
this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', {
|
||||
this.datatable.style.setStyle('.dt-dropdown', {
|
||||
display: 'none'
|
||||
});
|
||||
}
|
||||
|
|
@ -180,35 +176,6 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
}
|
||||
|
||||
setup_styles() {
|
||||
let columns = this.datatable.getColumns();
|
||||
columns.forEach(col => {
|
||||
let class_name = [
|
||||
`.dt-header .dt-cell--col-${col.colIndex}`,
|
||||
`.dt-header .dt-cell--col-${col.colIndex} .dt-dropdown__toggle`
|
||||
].join(',');
|
||||
|
||||
if (!col.skip_import && col.df) {
|
||||
this.datatable.style.setStyle(class_name, {
|
||||
backgroundColor: frappe.ui.color.get_color_shade(
|
||||
'green',
|
||||
'extra-light'
|
||||
),
|
||||
color: frappe.ui.color.get_color_shade('green', 'dark')
|
||||
});
|
||||
}
|
||||
if (col.skip_import && col.name !== 'Sr. No') {
|
||||
this.datatable.style.setStyle(class_name, {
|
||||
backgroundColor: frappe.ui.color.get_color_shade(
|
||||
'orange',
|
||||
'extra-light'
|
||||
),
|
||||
color: frappe.ui.color.get_color_shade('orange', 'dark')
|
||||
});
|
||||
this.datatable.style.setStyle(`.dt-cell--col-${col.colIndex}`, {
|
||||
backgroundColor: frappe.ui.color.get_color_shade('white', 'light')
|
||||
});
|
||||
}
|
||||
});
|
||||
// import success checkbox
|
||||
this.datatable.style.setStyle(`svg.import-success`, {
|
||||
width: '16px',
|
||||
|
|
@ -232,8 +199,8 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
let failures = this.import_log.filter(log => !log.success);
|
||||
if (failures.length > 0) {
|
||||
this.wrapper.find('.table-actions').append(
|
||||
`<button class="btn btn-xs btn-default" data-action="export_errored_rows">
|
||||
${__('Export rows which are not imported')}
|
||||
`<button class="btn btn-sm btn-default" data-action="export_errored_rows">
|
||||
${__('Export Errored Rows')}
|
||||
</button>
|
||||
`);
|
||||
}
|
||||
|
|
@ -243,6 +210,78 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
this.events.export_errored_rows();
|
||||
}
|
||||
|
||||
show_column_mapper() {
|
||||
let column_picker_fields = new ColumnPickerFields({
|
||||
doctype: this.doctype
|
||||
});
|
||||
let changed = [];
|
||||
let fields = this.fields.map((df, i) => {
|
||||
if (df.label === 'Sr. No') return [];
|
||||
|
||||
let fieldname;
|
||||
if (df.skip_import) {
|
||||
fieldname = null;
|
||||
} else {
|
||||
fieldname = df.parent === this.doctype
|
||||
? df.fieldname
|
||||
: `${df.parent}:${df.fieldname}`;
|
||||
}
|
||||
return [
|
||||
{
|
||||
label: __('Column {0}', [i]),
|
||||
fieldtype: 'Data',
|
||||
default: df.header_title,
|
||||
fieldname: `Column ${i}`,
|
||||
read_only: 1
|
||||
},
|
||||
{
|
||||
fieldtype: 'Button',
|
||||
label: 'Skip Column',
|
||||
fieldname: 'skip_' + i,
|
||||
click: () => {
|
||||
let header_row_index = i - 1;
|
||||
this.events.skip_import(header_row_index);
|
||||
}
|
||||
},
|
||||
{
|
||||
fieldtype: 'Column Break'
|
||||
},
|
||||
{
|
||||
fieldtype: 'Autocomplete',
|
||||
fieldname: i,
|
||||
label: __('Select field'),
|
||||
max_items: Infinity,
|
||||
options: column_picker_fields.get_fields_as_options(),
|
||||
default: fieldname,
|
||||
change() {
|
||||
changed.push(i);
|
||||
}
|
||||
},
|
||||
{
|
||||
fieldtype: 'Section Break'
|
||||
}
|
||||
];
|
||||
});
|
||||
// flatten the array
|
||||
fields = fields.reduce((acc, curr) => [...acc, ...curr]);
|
||||
let dialog = new frappe.ui.Dialog({
|
||||
title: __('Column Mapper'),
|
||||
fields,
|
||||
primary_action: (values) => {
|
||||
let changed_map = {};
|
||||
changed.map(i => {
|
||||
let header_row_index = i - 1;
|
||||
changed_map[header_row_index] = values[i];
|
||||
});
|
||||
if (changed.length > 0) {
|
||||
this.events.remap_column(changed_map);
|
||||
}
|
||||
dialog.hide();
|
||||
}
|
||||
});
|
||||
dialog.show();
|
||||
}
|
||||
|
||||
remap_column(col) {
|
||||
let column_picker_fields = new ColumnPickerFields({
|
||||
doctype: this.doctype
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue