fix: Refactor Data Import
- Break Importer into classes ImportFile, Row, Column, Header - Show warnings section before import preview
This commit is contained in:
Faris Ansari
parent
806c2ac0b6
commit
429ff2ef87
10 changed files with 1078 additions and 939 deletions
|
|
@ -24,7 +24,7 @@ user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
|
|||
"has_role:Page", "has_role:Report")
|
||||
|
||||
doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified",
|
||||
"linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map')
|
||||
"linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map', 'data_import_column_header_map')
|
||||
|
||||
|
||||
def clear_user_cache(user=None):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
.warnings .warning {
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
frm.set_query('reference_doctype', () => {
|
||||
return {
|
||||
filters: {
|
||||
allow_import: 1
|
||||
name: ['in', frappe.boot.user.can_import]
|
||||
}
|
||||
};
|
||||
});
|
||||
|
|
@ -236,7 +236,7 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
frm
|
||||
.call({
|
||||
method: 'get_preview_from_template',
|
||||
args: { data_import: frm.doc.name },
|
||||
args: { data_import: frm.doc.name, import_file: frm.doc.import_file },
|
||||
error_handlers: {
|
||||
TimestampMismatchError() {
|
||||
// ignore this error
|
||||
|
|
@ -331,8 +331,8 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
})
|
||||
.join('');
|
||||
return `
|
||||
<div class="alert border" data-row="${row_number}">
|
||||
<div class="uppercase">${__('Row {0}', [row_number])}</div>
|
||||
<div class="warning" data-row="${row_number}">
|
||||
<h5 class="text-uppercase">${__('Row {0}', [row_number])}</h5>
|
||||
<div class="body"><ul>${message}</ul></div>
|
||||
</div>
|
||||
`;
|
||||
|
|
@ -346,8 +346,8 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
header = __('Column {0}', [warning.col]);
|
||||
}
|
||||
return `
|
||||
<div class="alert border" data-col="${warning.col}">
|
||||
<div class="uppercase">${header}</div>
|
||||
<div class="warning" data-col="${warning.col}">
|
||||
<h5 class="text-uppercase">${header}</h5>
|
||||
<div class="body">${warning.message}</div>
|
||||
</div>
|
||||
`;
|
||||
|
|
@ -355,7 +355,7 @@ frappe.ui.form.on('Data Import Beta', {
|
|||
.join('');
|
||||
frm.get_field('import_warnings').$wrapper.html(`
|
||||
<div class="row">
|
||||
<div class="col-sm-6 warnings text-muted">${html}</div>
|
||||
<div class="col-sm-10 warnings">${html}</div>
|
||||
</div>
|
||||
`);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -16,11 +16,11 @@
|
|||
"submit_after_import",
|
||||
"mute_emails",
|
||||
"template_options",
|
||||
"section_import_preview",
|
||||
"import_preview",
|
||||
"import_warnings_section",
|
||||
"template_warnings",
|
||||
"import_warnings",
|
||||
"section_import_preview",
|
||||
"import_preview",
|
||||
"import_log_section",
|
||||
"import_log",
|
||||
"show_failed_logs",
|
||||
|
|
@ -34,7 +34,9 @@
|
|||
"label": "Document Type",
|
||||
"options": "DocType",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_type",
|
||||
|
|
@ -43,28 +45,38 @@
|
|||
"label": "Import Type",
|
||||
"options": "\nInsert New Records\nUpdate Existing Records",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal",
|
||||
"fieldname": "import_file",
|
||||
"fieldtype": "Attach",
|
||||
"in_list_view": 1,
|
||||
"label": "Import File"
|
||||
"label": "Import File",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Preview"
|
||||
"label": "Import Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "section_import_preview",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Preview"
|
||||
"label": "Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "column_break_5",
|
||||
"fieldtype": "Column Break"
|
||||
"fieldtype": "Column Break",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "template_options",
|
||||
|
|
@ -72,23 +84,31 @@
|
|||
"hidden": 1,
|
||||
"label": "Template Options",
|
||||
"options": "JSON",
|
||||
"read_only": 1
|
||||
"read_only": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log",
|
||||
"fieldtype": "Code",
|
||||
"label": "Import Log",
|
||||
"options": "JSON"
|
||||
"options": "JSON",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Import Log"
|
||||
"label": "Import Log",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Log Preview"
|
||||
"label": "Import Log Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "Pending",
|
||||
|
|
@ -97,56 +117,72 @@
|
|||
"hidden": 1,
|
||||
"label": "Status",
|
||||
"options": "Pending\nSuccess\nPartial Success\nError",
|
||||
"read_only": 1
|
||||
"read_only": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "template_warnings",
|
||||
"fieldtype": "Code",
|
||||
"hidden": 1,
|
||||
"label": "Template Warnings",
|
||||
"options": "JSON"
|
||||
"options": "JSON",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "submit_after_import",
|
||||
"fieldtype": "Check",
|
||||
"label": "Submit After Import",
|
||||
"set_only_once": 1
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Warnings"
|
||||
"label": "Warnings",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Warnings"
|
||||
"label": "Import Warnings",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "reference_doctype",
|
||||
"fieldname": "download_template",
|
||||
"fieldtype": "Button",
|
||||
"label": "Download Template"
|
||||
"label": "Download Template",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "1",
|
||||
"fieldname": "mute_emails",
|
||||
"fieldtype": "Check",
|
||||
"label": "Don't Send Emails",
|
||||
"set_only_once": 1
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "show_failed_logs",
|
||||
"fieldtype": "Check",
|
||||
"label": "Show Failed Logs"
|
||||
"label": "Show Failed Logs",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
}
|
||||
],
|
||||
"hide_toolbar": 1,
|
||||
"links": [],
|
||||
"modified": "2020-02-17 15:35:04.386098",
|
||||
"modified_by": "faris@erpnext.com",
|
||||
"modified": "2020-05-28 22:11:38.266208",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Core",
|
||||
"name": "Data Import Beta",
|
||||
"owner": "Administrator",
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@
|
|||
from __future__ import unicode_literals
|
||||
import frappe
|
||||
from frappe.model.document import Document
|
||||
from frappe.core.doctype.data_import.importer_new import Importer
|
||||
from frappe.core.doctype.data_import.exporter_new import Exporter
|
||||
|
||||
from frappe.core.doctype.data_import_beta.importer import Importer
|
||||
from frappe.core.doctype.data_import_beta.exporter import Exporter
|
||||
from frappe.core.page.background_jobs.background_jobs import get_info
|
||||
from frappe.utils.background_jobs import enqueue
|
||||
from frappe import _
|
||||
|
|
@ -25,7 +26,10 @@ class DataImportBeta(Document):
|
|||
# validate template
|
||||
self.get_importer()
|
||||
|
||||
def get_preview_from_template(self):
|
||||
def get_preview_from_template(self, import_file=None):
|
||||
if import_file:
|
||||
self.import_file = import_file
|
||||
|
||||
if not self.import_file:
|
||||
return
|
||||
|
||||
|
|
@ -62,8 +66,8 @@ class DataImportBeta(Document):
|
|||
|
||||
|
||||
@frappe.whitelist()
|
||||
def get_preview_from_template(data_import):
|
||||
return frappe.get_doc("Data Import Beta", data_import).get_preview_from_template()
|
||||
def get_preview_from_template(data_import, import_file):
|
||||
return frappe.get_doc("Data Import Beta", data_import).get_preview_from_template(import_file)
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
|
|
@ -81,8 +85,8 @@ def start_import(data_import):
|
|||
frappe.db.rollback()
|
||||
data_import.db_set("status", "Error")
|
||||
frappe.log_error(title=data_import.name)
|
||||
frappe.db.commit()
|
||||
frappe.publish_realtime("data_import_refresh", {"data_import": data_import.name})
|
||||
|
||||
frappe.publish_realtime("data_import_refresh", {"data_import": data_import.name})
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -202,16 +202,16 @@ frappe.data_import.DataExporter = class DataExporter {
|
|||
}
|
||||
|
||||
select_mandatory() {
|
||||
let mandatory_table_doctypes = frappe.meta
|
||||
let mandatory_table_fields = frappe.meta
|
||||
.get_table_fields(this.doctype)
|
||||
.filter(df => df.reqd)
|
||||
.map(df => df.options);
|
||||
mandatory_table_doctypes.push(this.doctype);
|
||||
.map(df => df.fieldname);
|
||||
mandatory_table_fields.push(this.doctype);
|
||||
|
||||
let multicheck_fields = this.dialog.fields
|
||||
.filter(df => df.fieldtype === 'MultiCheck')
|
||||
.map(df => df.fieldname)
|
||||
.filter(doctype => mandatory_table_doctypes.includes(doctype));
|
||||
.filter(doctype => mandatory_table_fields.includes(doctype));
|
||||
|
||||
let checkboxes = [].concat(
|
||||
...multicheck_fields.map(fieldname => {
|
||||
|
|
@ -333,16 +333,24 @@ frappe.data_import.DataExporter = class DataExporter {
|
|||
}
|
||||
};
|
||||
|
||||
function get_columns_for_picker(doctype) {
|
||||
export function get_columns_for_picker(doctype) {
|
||||
let out = {};
|
||||
|
||||
const standard_fields_filter = df =>
|
||||
!in_list(frappe.model.no_value_type, df.fieldtype);
|
||||
const exportable_fields = df => {
|
||||
let keep = true;
|
||||
if (frappe.model.no_value_type.includes(df.fieldtype)) {
|
||||
keep = false;
|
||||
}
|
||||
if (['lft', 'rgt'].includes(df.fieldname)) {
|
||||
keep = false;
|
||||
}
|
||||
return keep;
|
||||
};
|
||||
|
||||
// parent
|
||||
let doctype_fields = frappe.meta
|
||||
.get_docfields(doctype)
|
||||
.filter(standard_fields_filter);
|
||||
.filter(exportable_fields);
|
||||
|
||||
out[doctype] = [
|
||||
{
|
||||
|
|
@ -359,7 +367,7 @@ function get_columns_for_picker(doctype) {
|
|||
const cdt = df.options;
|
||||
const child_table_fields = frappe.meta
|
||||
.get_docfields(cdt)
|
||||
.filter(standard_fields_filter);
|
||||
.filter(exportable_fields);
|
||||
|
||||
out[df.fieldname] = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import DataTable from 'frappe-datatable';
|
||||
import ColumnPickerFields from './column_picker_fields';
|
||||
import { get_columns_for_picker } from './data_exporter';
|
||||
|
||||
frappe.provide('frappe.data_import');
|
||||
|
||||
|
|
@ -236,9 +236,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
|
|||
}
|
||||
|
||||
show_column_mapper() {
|
||||
let column_picker_fields = new ColumnPickerFields({
|
||||
doctype: this.doctype
|
||||
});
|
||||
let column_picker_fields = get_columns_for_picker(this.doctype);
|
||||
let changed = [];
|
||||
let fields = this.preview_data.columns.map((col, i) => {
|
||||
let df = col.df;
|
||||
|
|
|
|||
|
|
@ -249,6 +249,7 @@
|
|||
}
|
||||
|
||||
.progress-message {
|
||||
font-feature-settings: "tnum" 1;
|
||||
margin-top: 0px;
|
||||
}
|
||||
}
|
||||
|
|
@ -1011,7 +1012,7 @@ body[data-route^="Form/Communication"] textarea[data-fieldname="subject"] {
|
|||
|
||||
.map-columns .form-section {
|
||||
padding: 0 7px 7px;
|
||||
border-bottom: none;
|
||||
border-top: none;
|
||||
|
||||
.clearfix {
|
||||
display: none;
|
||||
|
|
@ -1021,3 +1022,7 @@ body[data-route^="Form/Communication"] textarea[data-fieldname="subject"] {
|
|||
.map-columns .form-section:first-child {
|
||||
padding-top: 7px;
|
||||
}
|
||||
|
||||
.table-preview {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1185,3 +1185,75 @@ def is_subset(list_a, list_b):
|
|||
|
||||
def generate_hash(*args, **kwargs):
|
||||
return frappe.generate_hash(*args, **kwargs)
|
||||
|
||||
|
||||
|
||||
def guess_date_format(date_string):
|
||||
DATE_FORMATS = [
|
||||
r"%d-%m-%Y",
|
||||
r"%m-%d-%Y",
|
||||
r"%Y-%m-%d",
|
||||
r"%d-%m-%y",
|
||||
r"%m-%d-%y",
|
||||
r"%y-%m-%d",
|
||||
r"%d/%m/%Y",
|
||||
r"%m/%d/%Y",
|
||||
r"%Y/%m/%d",
|
||||
r"%d/%m/%y",
|
||||
r"%m/%d/%y",
|
||||
r"%y/%m/%d",
|
||||
r"%d.%m.%Y",
|
||||
r"%m.%d.%Y",
|
||||
r"%Y.%m.%d",
|
||||
r"%d.%m.%y",
|
||||
r"%m.%d.%y",
|
||||
r"%y.%m.%d",
|
||||
]
|
||||
|
||||
TIME_FORMATS = [
|
||||
r"%H:%M:%S.%f",
|
||||
r"%H:%M:%S",
|
||||
r"%H:%M",
|
||||
r"%I:%M:%S.%f %p",
|
||||
r"%I:%M:%S %p",
|
||||
r"%I:%M %p",
|
||||
]
|
||||
|
||||
date_string = date_string.strip()
|
||||
|
||||
_date = None
|
||||
_time = None
|
||||
|
||||
if " " in date_string:
|
||||
_date, _time = date_string.split(" ", 1)
|
||||
else:
|
||||
_date = date_string
|
||||
|
||||
date_format = None
|
||||
time_format = None
|
||||
|
||||
for f in DATE_FORMATS:
|
||||
try:
|
||||
# if date is parsed without any exception
|
||||
# capture the date format
|
||||
datetime.datetime.strptime(_date, f)
|
||||
date_format = f
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if _time:
|
||||
for f in TIME_FORMATS:
|
||||
try:
|
||||
# if time is parsed without any exception
|
||||
# capture the time format
|
||||
datetime.datetime.strptime(_time, f)
|
||||
time_format = f
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
full_format = date_format
|
||||
if time_format:
|
||||
full_format += " " + time_format
|
||||
return full_format
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue