vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

fix: Parse template and prepare preview data

Browse source 
- Match column header with label or fieldname
- Remove empty rows and columns
- Merge ID and autoname field
This commit is contained in:
Faris Ansari 2019-08-13 00:57:52 +05:30
parent cf796cf29b
commit bae336154f
7 changed files with 309 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
frappe/core/doctype/data_import/exporter_new.py
View file

@ -48,7 +48,21 @@ class Exporter:
'reqd': 1,
'parent': self.doctype
})
return [name_field] + self.get_exportable_fields(self.doctype)
parent_fields = self.get_exportable_fields(self.doctype)
# if autoname is based on field
# then merge ID and the field column title as "ID (Autoname Field)"
autoname = self.meta.autoname
if autoname and autoname.startswith('field:'):
fieldname = autoname[len('field:'):]
autoname_field = self.meta.get_field(fieldname)
if autoname_field:
name_field.label = 'ID ({})'.format(autoname_field.label)
# remove the autoname field as it is a duplicate of ID field
parent_fields = [df for df in parent_fields if df.fieldname != autoname_field.fieldname]
return [name_field] + parent_fields
def get_exportable_children_fields(self):

157
frappe/core/doctype/data_import/importer_new.py
View file

@ -10,6 +10,7 @@ from frappe import _
from frappe.utils import cint, flt, DATE_FORMAT, DATETIME_FORMAT
from frappe.utils.csvutils import read_csv_content
from frappe.exceptions import ValidationError, MandatoryError
from frappe.model import display_fieldtypes, no_value_fields, table_fields
# set user lang
# set flags: frappe.flags.in_import = True
@ -18,6 +19,8 @@ from frappe.exceptions import ValidationError, MandatoryError
# check empty row
# validate naming
INVALID_VALUES = ['', None]
class Importer:
def __init__(self, doctype, file_path=None, content=None, options=None):
@ -33,6 +36,7 @@ class Importer:
elif content:
self.read_content(content)
self.remove_empty_rows_and_columns()
def read_file(self, file_path):
extn = file_path.split('.')[1]
@ -53,8 +57,159 @@ class Importer:
self.data = data[1:]
def remove_empty_rows_and_columns(self):
self.row_index_map = []
removed_rows = []
removed_columns = []
# remove empty rows
data = []
for i, row in enumerate(self.data):
if all(v in INVALID_VALUES for v in row):
# empty row
removed_rows.append(i)
else:
data.append(row)
self.row_index_map.append(i)
# remove empty columns
# a column with a header and no data is a valid column
# a column with no header and no data will be removed
header_row = []
for i, column in enumerate(self.header_row):
column_values = [row[i] for row in data]
values = [column] + column_values
if all(v in INVALID_VALUES for v in values):
# empty column
removed_columns.append(i)
else:
header_row.append(column)
data_without_empty_columns = []
# remove empty columns from data
for i, row in enumerate(data):
new_row = [v for j, v in enumerate(row) if j not in removed_columns]
data_without_empty_columns.append(new_row)
self.data = data_without_empty_columns
self.header_row = header_row
def get_data_for_import_preview(self):
fields, fields_warnings = self.parse_fields_from_header_row()
formats, formats_warnings = self.parse_formats_from_first_10_rows()
fields, data = self.add_serial_no_column(fields, self.data)
warnings = fields_warnings + formats_warnings
return dict(
fields=fields,
data=data,
warnings=warnings
)
def parse_fields_from_header_row(self):
fields = []
warnings = []
df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching()
for i, value in enumerate(self.header_row):
field = df_by_labels_and_fieldnames.get(value)
if not field:
field = {
'label': value,
'skip_import': True
}
if value:
warnings.append(_('Column {0}: Cannot match column {1} with any field').format(i, frappe.bold(value)))
else:
warnings.append(_('Column {0}: Skipping untitled column').format(i))
fields.append(field)
return fields, warnings
def build_fields_dict_for_column_matching(self):
"""
Build a dict with various keys to match with column headers and value as docfield
The keys can be label or fieldname
{
'Customer': df1,
'customer': df1,
'Due Date': df2,
'due_date': df2,
'Sales Invoice Item / Item Code': df3
}
"""
out = {
'ID': frappe._dict({
'fieldtype': 'Data',
'fieldname': 'name',
'label': 'ID',
'reqd': 1,
'parent': self.doctype
})
}
doctypes = [self.doctype] + [df.options for df in self.meta.get_table_fields()]
for doctype in doctypes:
meta = frappe.get_meta(doctype)
for df in meta.fields:
if df.fieldtype not in no_value_fields:
# label as key
label = df.label if self.doctype == doctype else '{0} / {1}'.format(df.parent, df.label)
out[label] = df
# fieldname as key
if self.doctype == doctype:
out[df.fieldname] = df
# if autoname is based on field
# add an entry for "ID (Autoname Field)"
autoname = self.meta.autoname
if autoname and autoname.startswith('field:'):
fieldname = autoname[len('field:'):]
autoname_field = self.meta.get_field(fieldname)
if autoname_field:
out['ID ({})'.format(autoname_field.label)] = autoname_field
# ID field should also map to the autoname field
out['ID'] = autoname_field
return out
def parse_formats_from_first_10_rows(self):
"""
Returns a list of column descriptors for columns that might need parsing.
For e.g if it is a Date column return the Date format
[
[['Data']],
[['Date', '%m/%d/%y']],
[['Currency', '#,###.##']],
...
]
"""
formats = []
return formats, []
def add_serial_no_column(self, fields, data):
fields_with_serial_no = [
{
'label': _('Sr. No'),
'skip_import': True
}
] + fields
data_with_serial_no = []
for i, row in enumerate(data):
data_with_serial_no.append([self.row_index_map[i] + 1] + row)
return fields_with_serial_no, data_with_serial_no
def parse_data_for_import(self, row, index):
INVALID_VALUES = ['', None]
if all(v in INVALID_VALUES for v in row):
# empty row

5
frappe/core/doctype/data_import_beta/data_import_beta.js
View file

@ -24,12 +24,13 @@ frappe.ui.form.on('Data Import Beta', {
.appendTo(frm.get_field('import_preview').$wrapper);
frm.call('get_preview_from_template').then(r => {
let csv_array = r.message;
let preview_data = r.message;
frappe.require('/assets/js/data_import_tools.min.js', () => {
new frappe.data_import.ImportPreview(
frm.get_field('import_preview').$wrapper,
csv_array
frm.doc.reference_doctype,
preview_data
);
});
});

6
frappe/core/doctype/data_import_beta/data_import_beta.py
View file

@ -5,7 +5,7 @@
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.utils.csvutils import read_csv_content
from frappe.core.doctype.data_import.importer_new import Importer
from frappe.core.doctype.data_import.exporter_new import Exporter
class DataImportBeta(Document):
@ -16,9 +16,9 @@ class DataImportBeta(Document):
f = frappe.get_doc('File', { 'file_url': self.import_file })
file_content = f.get_content()
csv_content = read_csv_content(file_content)
return csv_content[:100]
i = Importer(self.reference_doctype, content=file_content)
return i.get_data_for_import_preview()
@frappe.whitelist()

64
frappe/public/js/frappe/data_import/data_exporter.js
View file

@ -13,10 +13,6 @@ frappe.data_import.DataExporter = class DataExporter {
}
make_dialog() {
let column_map = new ColumnPickerFields({
doctype: this.doctype
}).get_columns_for_picker();
let doctypes = [this.doctype].concat(
...frappe.meta
.get_table_fields(this.doctype)
@ -78,15 +74,7 @@ frappe.data_import.DataExporter = class DataExporter {
on_change: () => {
this.update_primary_action();
},
options: column_map[doctype].map(df => ({
label: __(df.label),
value: df.fieldname,
danger: df.reqd,
checked: df.reqd,
description: `${df.fieldname} ${
df.reqd ? __('(Mandatory)') : ''
}`
}))
options: this.get_multicheck_options(doctype)
};
})
],
@ -161,12 +149,8 @@ frappe.data_import.DataExporter = class DataExporter {
${__('Unselect All')}
</button>
</div>
`).on('click', '[data-action]', e => {
let $target = $(e.currentTarget);
let action = $target.data('action');
let method = this[action];
method ? this[action]() : null;
});
`);
frappe.utils.bind_actions_with_class($select_all_buttons, this);
this.dialog
.get_field('select_all_buttons')
.$wrapper.html($select_all_buttons);
@ -239,11 +223,7 @@ frappe.data_import.DataExporter = class DataExporter {
} else {
message = __('{0} records will be exported', [value]);
}
this.dialog.set_df_property(
'export_records',
'description',
message
);
this.dialog.set_df_property('export_records', 'description', message);
this.update_primary_action(value);
});
@ -277,4 +257,40 @@ frappe.data_import.DataExporter = class DataExporter {
});
}, {});
}
get_multicheck_options(doctype) {
if (!this.column_map) {
this.column_map = new ColumnPickerFields({
doctype: this.doctype
}).get_columns_for_picker();
}
let autoname_field = null;
let meta = frappe.get_meta(doctype);
if (meta.autoname && meta.autoname.startsWith('field:')) {
let fieldname = meta.autoname.slice('field:'.length);
autoname_field = frappe.meta.get_field(doctype, fieldname);
}
return this.column_map[doctype]
.filter(df => {
if (autoname_field && df.fieldname === autoname_field.fieldname) {
return false;
}
return true;
})
.map(df => {
let label = __(df.label);
if (autoname_field && df.fieldname === 'name') {
label = label + ` (${__(autoname_field.label)})`;
}
return {
label,
value: df.fieldname,
danger: df.reqd,
checked: df.reqd,
description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}`
};
});
}
};

103
frappe/public/js/frappe/data_import/import_preview.js
View file

@ -3,17 +3,69 @@ import DataTable from 'frappe-datatable';
frappe.provide('frappe.data_import');
frappe.data_import.ImportPreview = class ImportPreview {
constructor(wrapper, csv_array) {
constructor(wrapper, doctype, preview_data) {
frappe.import_preview = this;
this.wrapper = wrapper;
this.csv_array = csv_array;
this.doctype = doctype;
this.preview_fields = preview_data.fields;
this.preview_data = preview_data.data;
this.preview_warnings = preview_data.warnings;
this.prepare_csv_array();
this.make_wrapper();
this.prepare_columns();
this.prepare_data();
this.render_warnings();
this.render_datatable();
}
prepare_csv_array() {
this.csv_array = this.csv_array.map(row => {
make_wrapper() {
this.wrapper.html(`
<div>
<div class="warnings"></div>
<div class="table-preview"></div>
<div class="table-actions margin-top">
<button class="btn btn-xs btn-default" data-action="add_row">
${__('Add Row')}
</button>
</div>
</div>
`);
frappe.utils.bind_actions_with_class(this.wrapper, this);
this.$warnings = this.wrapper.find('.warnings');
this.$table_preview = this.wrapper.find('.table-preview');
}
prepare_columns() {
this.columns = this.preview_fields.map(df => {
if (df.skip_import) {
return {
id: frappe.utils.get_random(6),
name: df.label,
skip_import: true,
editable: false,
focusable: false,
format: (value, row, column, data) => {
return `<div class="text-muted">${value}</div>`;
}
};
}
let column_title = df.label;
if (this.doctype !== df.parent) {
column_title = `${df.label} (${df.parent})`;
}
return {
id: df.fieldname,
name: column_title,
df: df,
editable: true
};
});
}
prepare_data() {
this.preview_data = this.preview_data.map(row => {
return row.map(cell => {
if (cell == null) {
return '';
@ -23,19 +75,26 @@ frappe.data_import.ImportPreview = class ImportPreview {
});
}
render_warnings() {
let warning_html = this.preview_warnings
.map(warning => {
return `<div style="line-height: 2">${warning}</div>`;
})
.join('');
let html = `<div class="border text-muted padding rounded margin-bottom">${warning_html}</div>`;
this.$warnings.html(html);
}
render_datatable() {
let columns = this.csv_array[0].map(col => {
return {
id: col,
name: col
};
});
let data = this.csv_array.slice(1);
this.datatable = new DataTable(this.wrapper.get(0), {
data,
columns,
this.datatable = new DataTable(this.$table_preview.get(0), {
data: this.preview_data,
columns: this.columns,
layout: 'fixed',
cellHeight: 35,
serialNoColumn: false,
checkboxColumn: true,
pasteFromClipboard: true,
headerDropdown: [
{
label: __('Change column mapping'),
@ -48,11 +107,13 @@ frappe.data_import.ImportPreview = class ImportPreview {
]
});
this.datatable.style.setStyle(
'.dt-dropdown__list-item:nth-child(-n+4)',
{
display: 'none'
}
);
this.datatable.style.setStyle('.dt-dropdown__list-item:nth-child(-n+4)', {
display: 'none'
});
}
add_row() {
this.preview_data.push([]);
this.datatable.refresh(this.preview_data);
}
};

10
frappe/public/js/frappe/utils/utils.js
View file

@ -725,6 +725,16 @@ Object.assign(frappe.utils, {
},
is_rtl() {
return ["ar", "he", "fa"].includes(frappe.boot.lang);
},
bind_actions_with_class($el, class_instance) {
$($el).on('click', '[data-action]', e => {
let $target = $(e.currentTarget);
let action = $target.data('action');
let method = class_instance[action];
method ? class_instance[action]() : null;
});
return $el;
}
});