vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

New data import (#4601)

Browse source 
* created/moved the files

* added the model for the downloading the data

* add the file with the error data

* changes added other changes and fix codacy

* changes in the config and utils files

* fixed the test cases

* minor changes in the data keys dict

* changed the test file location

* fixed the tests

* set the route in the list view and show only erors

* minor fixes in the childtable import and log tables rendering

* Refactor Download dialog to use MultiCheck
This commit is contained in:
Manas Solanki 2017-12-15 12:17:24 +05:30 • committed by Nabin Hait
parent a4ced6e4f2
commit 520bfc2ae4
32 changed files with 1270 additions and 538 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
frappe/__init__.py
View file

@ -976,9 +976,9 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
ps.insert()
def import_doc(path, ignore_links=False, ignore_insert=False, insert=False):
"""Import a file using Data Import Tool."""
from frappe.core.page.data_import_tool import data_import_tool
data_import_tool.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
"""Import a file using Data Import."""
from frappe.core.doctype.data_import import data_import
data_import.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
def copy_doc(doc, ignore_no_copy=True):
""" No_copy fields also get copied."""
@ -1366,7 +1366,7 @@ def logger(module=None, with_more_info=True):
def log_error(message=None, title=None):
'''Log error to Error Log'''
get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
method=title)).insert(ignore_permissions=True)
def get_desk_link(doctype, name):

16
frappe/commands/utils.py
View file

@ -162,12 +162,12 @@ def export_doc(context, doctype, docname):
@pass_context
def export_json(context, doctype, path, name=None):
"Export doclist as json to the given path, use '-' as name for Singles."
from frappe.core.page.data_import_tool import data_import_tool
from frappe.core.doctype.data_import import data_import
for site in context.sites:
try:
frappe.init(site=site)
frappe.connect()
data_import_tool.export_json(doctype, path, name=name)
data_import.export_json(doctype, path, name=name)
finally:
frappe.destroy()
@ -177,12 +177,12 @@ def export_json(context, doctype, path, name=None):
@pass_context
def export_csv(context, doctype, path):
"Export data import template with data for DocType"
from frappe.core.page.data_import_tool import data_import_tool
from frappe.core.doctype.data_import import data_import
for site in context.sites:
try:
frappe.init(site=site)
frappe.connect()
data_import_tool.export_csv(doctype, path)
data_import.export_csv(doctype, path)
finally:
frappe.destroy()
@ -204,7 +204,7 @@ def export_fixtures(context):
@pass_context
def import_doc(context, path, force=False):
"Import (insert/update) doclist. If the argument is a directory, all files ending with .json are imported"
from frappe.core.page.data_import_tool import data_import_tool
from frappe.core.doctype.data_import import data_import
if not os.path.exists(path):
path = os.path.join('..', path)
@ -216,7 +216,7 @@ def import_doc(context, path, force=False):
try:
frappe.init(site=site)
frappe.connect()
data_import_tool.import_doc(path, overwrite=context.force)
data_import.import_doc(path, overwrite=context.force)
finally:
frappe.destroy()
@ -229,8 +229,8 @@ def import_doc(context, path, force=False):
@pass_context
def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
"Import CSV using data import tool"
from frappe.core.page.data_import_tool import importer
"Import CSV using data import"
from frappe.core.doctype.data_import import importer
from frappe.utils.csvutils import read_csv_content
site = get_site(context)

2
frappe/config/desktop.py
View file

@ -70,5 +70,5 @@ def get_data():
"icon": "octicon octicon-book",
"color": '#FFAEDB',
"hidden": 1,
},
}
]

8
frappe/config/setup.py
View file

@ -93,11 +93,11 @@ def get_data():
"icon": "fa fa-th",
"items": [
{
"type": "page",
"name": "data-import-tool",
"type": "doctype",
"name": "Data Import",
"label": _("Import / Export Data"),
"icon": "fa fa-upload",
"description": _("Import / Export Data from .csv files.")
"icon": "octicon octicon-cloud-upload",
"description": _("Import / Export Data from CSV and Excel files.")
},
{
"type": "doctype",

0
frappe/core/page/data_import_tool/README.md → frappe/core/doctype/data_import/README.md
View file

0
frappe/core/doctype/data_import/__init__.py Normal file
View file

260
frappe/core/doctype/data_import/data_import.js Normal file
View file

@ -0,0 +1,260 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Import', {
onload: function(frm) {
frm.set_query("reference_doctype", function() {
return {
"filters": {
"issingle": 0,
"istable": 0,
"name": ['in', frappe.boot.user.can_import]
}
};
});
frappe.realtime.on("data_import_progress", function(data) {
if (data.data_import === frm.doc.name) {
if (data.reload && data.reload === true) {
frm.reload_doc();
}
if (data.progress) {
let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar");
if (progress_bar) {
$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
$(progress_bar).css("width", data.progress+"%");
}
}
}
});
},
refresh: function(frm) {
frm.disable_save();
frm.dashboard.clear_headline();
if (frm.doc.reference_doctype && !frm.doc.import_file) {
frm.dashboard.add_comment(__('Please attach a file to import'));
} else {
if (frm.doc.import_status) {
frm.dashboard.add_comment(frm.doc.import_status);
if (frm.doc.import_status==="In Progress") {
frm.dashboard.add_progress("Data Import Progress", "0");
frm.set_read_only(true);
}
}
}
if (frm.doc.reference_doctype) {
frappe.model.with_doctype(frm.doc.reference_doctype);
}
frm.add_custom_button(__("Help"), function() {
frappe.help.show_video("6wiriRKPhmg");
});
if(frm.doc.reference_doctype && frm.doc.docstatus === 0) {
frm.add_custom_button(__("Download template"), function() {
frappe.data_import.download_dialog(frm).show();
});
}
if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows && frm.doc.docstatus === 0) {
frm.page.set_primary_action(__("Start Import"), function() {
frappe.call({
method: "frappe.core.doctype.data_import.data_import.import_data",
args: {
data_import: frm.doc.name
}
});
}).addClass('btn btn-primary');
}
if (frm.doc.log_details) {
frm.events.create_log_table(frm);
} else {
$(frm.fields_dict.import_log.wrapper).empty();
}
},
reference_doctype: function(frm) {
if (frm.doc.reference_doctype) {
frm.save();
}
},
// import_file: function(frm) {
// frm.save();
// },
overwrite: function(frm) {
if (frm.doc.overwrite === 0) {
frm.doc.only_update = 0;
}
frm.save();
},
only_update: function(frm) {
frm.save();
},
submit_after_import: function(frm) {
frm.save();
},
skip_errors: function(frm) {
frm.save();
},
ignore_encoding_errors: function(frm) {
frm.save();
},
no_email: function(frm) {
frm.save();
},
show_only_errors: function(frm) {
frm.events.create_log_table(frm);
},
create_log_table: function(frm) {
let msg = JSON.parse(frm.doc.log_details);
var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
$(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors,
import_status: frm.doc.import_status})).appendTo($log_wrapper);
}
});
frappe.provide('frappe.data_import');
frappe.data_import.download_dialog = function(frm) {
var dialog;
const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);
const get_doctypes = parentdt => {
return [parentdt].concat(
frappe.meta.get_table_fields(parentdt).map(df => df.options)
);
};
const get_doctype_checkbox_fields = () => {
return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
.map(df => dialog.fields_dict[df.fieldname]);
}
const doctype_fields = get_fields(frm.doc.reference_doctype)
.map(df => ({
label: df.label,
value: df.fieldname,
checked: 1
}));
let fields = [
{
"label": __("Select Columns"),
"fieldname": "select_columns",
"fieldtype": "Select",
"options": "All\nMandatory\nManually",
"reqd": 1,
"onchange": function() {
const fields = get_doctype_checkbox_fields();
fields.map(f => f.toggle(this.value === 'Manually'));
}
},
{
"label": __("File Type"),
"fieldname": "file_type",
"fieldtype": "Select",
"options": "Excel\nCSV",
"default": "Excel"
},
{
"label": __("Download with Data"),
"fieldname": "with_data",
"fieldtype": "Check"
},
{
"label": frm.doc.reference_doctype,
"fieldname": "doctype_fields",
"fieldtype": "MultiCheck",
"options": doctype_fields,
"columns": 2,
"hidden": 1
}
];
const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype)
.map(df => {
return {
"label": df.options,
"fieldname": df.fieldname + '_fields',
"fieldtype": "MultiCheck",
"options": frappe.meta.get_docfields(df.options)
.filter(filter_fields)
.map(df => ({
label: df.label,
value: df.fieldname,
checked: 1
})),
"columns": 2,
"hidden": 1
};
});
fields = fields.concat(child_table_fields);
dialog = new frappe.ui.Dialog({
title: __('Download Template'),
fields: fields,
primary_action: function(values) {
var data = values;
if (frm.doc.reference_doctype) {
var export_params = () => {
let columns = {};
if (values.select_columns === 'All') {
columns = get_doctypes(frm.doc.reference_doctype).reduce((columns, doctype) => {
columns[doctype] = get_fields(doctype).map(df => df.fieldname);
return columns;
}, {});
} else if (values.select_columns === 'Mandatory') {
// only reqd child tables
const doctypes = [frm.doc.reference_doctype].concat(
frappe.meta.get_table_fields(frm.doc.reference_doctype)
.filter(df => df.reqd).map(df => df.options)
);
columns = doctypes.reduce((columns, doctype) => {
columns[doctype] = get_fields(doctype).filter(df => df.reqd).map(df => df.fieldname);
return columns;
}, {});
} else if (values.select_columns === 'Manually') {
columns = get_doctype_checkbox_fields().reduce((columns, field) => {
const options = field.get_checked_options();
columns[field.df.label] = options;
return columns;
}, {});
}
return {
doctype: frm.doc.reference_doctype,
parent_doctype: frm.doc.reference_doctype,
select_columns: JSON.stringify(columns),
with_data: data.with_data ? 'Yes' : 'No',
all_doctypes: 'Yes',
from_data_import: 'Yes',
excel_format: data.file_type === 'Excel' ? 'Yes' : 'No'
};
};
let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template';
open_url_post(get_template_url, export_params());
} else {
frappe.msgprint(__("Please select the Document Type."));
}
dialog.hide();
},
primary_action_label: __('Download')
});
return dialog;
};

661
frappe/core/doctype/data_import/data_import.json Normal file
View file

@ -0,0 +1,661 @@
{
"allow_copy": 1,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"autoname": "",
"beta": 0,
"creation": "2016-12-09 14:27:32.720061",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "Document",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "reference_doctype",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 1,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Document Type",
"length": 0,
"no_copy": 0,
"options": "DocType",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"collapsible_depends_on": "",
"columns": 0,
"depends_on": "eval:(!doc.__islocal)",
"fieldname": "section_break_4",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "import_file",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Attach file for Import",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_4",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "eval: doc.import_status == \"Partially Successful\"",
"description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.",
"fieldname": "error_file",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Generated File",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"collapsible_depends_on": "",
"columns": 0,
"depends_on": "eval:(!doc.__islocal)",
"fieldname": "section_break_6",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"depends_on": "",
"description": "If you are updating/overwriting already created records.",
"fieldname": "overwrite",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Update records",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"depends_on": "overwrite",
"description": "If you don't want to create any new records while updating the older records.",
"fieldname": "only_update",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Don't create new records",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.\n",
"fieldname": "skip_errors",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Skip rows with errors",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"depends_on": "",
"fieldname": "submit_after_import",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Submit after importing",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "0",
"depends_on": "",
"fieldname": "ignore_encoding_errors",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Ignore encoding errors",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "1",
"depends_on": "",
"fieldname": "no_email",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Do not send Emails",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 1,
"collapsible_depends_on": "eval: doc.import_status == \"Failed\"",
"columns": 0,
"depends_on": "import_status",
"fieldname": "import_detail",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Import Log",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "import_status",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Import Status",
"length": 0,
"no_copy": 0,
"options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 1,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "1",
"fieldname": "show_only_errors",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Show only errors",
"length": 0,
"no_copy": 1,
"permlevel": 0,
"precision": "",
"print_hide": 1,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 1,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "",
"depends_on": "import_status",
"fieldname": "import_log",
"fieldtype": "HTML",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Import Log",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 1,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "log_details",
"fieldtype": "Code",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Log Details",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "amended_from",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Amended From",
"length": 0,
"no_copy": 1,
"options": "Data Import",
"permlevel": 0,
"print_hide": 1,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "total_rows",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Total Rows",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 1,
"issingle": 0,
"istable": 0,
"max_attachments": 1,
"modified": "2017-12-14 16:27:37.683505",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 0,
"read": 1,
"report": 0,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 1,
"write": 1
}
],
"quick_entry": 0,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"title_field": "",
"track_changes": 1,
"track_seen": 1
}

101
frappe/core/page/data_import_tool/data_import_tool.py → frappe/core/doctype/data_import/data_import.py
View file

@ -1,44 +1,67 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, print_function
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe, os
from frappe import _
import frappe.modules.import_file
from frappe.model.document import Document
from frappe.utils.data import format_datetime
from frappe.core.doctype.data_import.importer import upload
from frappe.utils.background_jobs import enqueue
class DataImport(Document):
def autoname(self):
self.name = "Import on "+ format_datetime(self.creation)
def validate(self):
if not self.import_file:
self.db_set("total_rows", 0)
if self.import_status == "In Progress":
frappe.throw(_("Can't save the form as data import is in progress."))
# validate the template just after the upload
# if there is total_rows in the doc, it means that the template is already validated and error free
if self.import_file and not self.total_rows:
upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
@frappe.whitelist()
def get_data_keys():
return frappe._dict({
"data_separator": _('Start entering data below this line'),
"main_table": _("Table") + ":",
"parent_table": _("Parent Table") + ":",
"columns": _("Column Name") + ":",
"doctype": _("DocType") + ":"
})
def import_data(data_import):
frappe.db.set_value("Data Import", data_import, "import_status", "In Progress")
frappe.publish_realtime("data_import_progress", {"progress": "0",
"data_import": data_import, "reload": True}, user=frappe.session.user)
enqueue(upload, queue='default', timeout=6000, event='data_import',
data_import_doc=data_import, from_data_import="Yes", validate_template=False)
@frappe.whitelist()
def get_doctypes():
return frappe.get_user()._get("can_import")
@frappe.whitelist()
def get_doctype_options():
doctype = frappe.form_dict['doctype']
return [doctype] + [d.options for d in frappe.get_meta(doctype).get_table_fields()]
def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
insert=False, submit=False, pre_process=None):
if os.path.isdir(path):
files = [os.path.join(path, f) for f in os.listdir(path)]
else:
files = [path]
for f in files:
if f.endswith(".json"):
frappe.flags.mute_emails = True
frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
frappe.flags.mute_emails = False
frappe.db.commit()
elif f.endswith(".csv"):
import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
frappe.db.commit()
def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
from frappe.utils.csvutils import read_csv_content
from frappe.core.page.data_import_tool.importer import upload
print("Importing " + path)
with open(path, "r") as infile:
upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
submit_after_import=submit, pre_process=pre_process)
submit_after_import=submit, pre_process=pre_process)
def export_csv(doctype, path):
from frappe.core.page.data_import_tool.exporter import get_template
with open(path, "wb") as csvfile:
get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
csvfile.write(frappe.response.result.encode("utf-8"))
def export_json(doctype, path, filters=None, or_filters=None, name=None):
def post_process(out):
@ -71,6 +94,14 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None):
with open(path, "w") as outfile:
outfile.write(frappe.as_json(out))
def export_csv(doctype, path):
from frappe.core.doctype.data_import.exporter import get_template
with open(path, "wb") as csvfile:
get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
csvfile.write(frappe.response.result.encode("utf-8"))
@frappe.whitelist()
def export_fixture(doctype, app):
if frappe.session.user != "Administrator":
@ -80,21 +111,3 @@ def export_fixture(doctype, app):
os.mkdir(frappe.get_app_path(app, "fixtures"))
export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"))
def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
insert=False, submit=False, pre_process=None):
if os.path.isdir(path):
files = [os.path.join(path, f) for f in os.listdir(path)]
else:
files = [path]
for f in files:
if f.endswith(".json"):
frappe.flags.mute_emails = True
frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
frappe.flags.mute_emails = False
frappe.db.commit()
elif f.endswith(".csv"):
import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
frappe.db.commit()

16
frappe/core/doctype/data_import/data_import_list.js Normal file
View file

@ -0,0 +1,16 @@
frappe.listview_settings['Data Import'] = {
add_fields: ["import_status"],
get_indicator: function(doc) {
if (doc.import_status=="Successful") {
return [__("Data imported"), "blue", "import_status,=,Successful"];
} else if(doc.import_status == "Partially Successful") {
return [__("Data partially imported"), "blue", "import_status,=,Partially Successful"];
} else if(doc.import_status == "In Process") {
return [__("Data import in progress"), "orange", "import_status,=,In Process"];
} else if(doc.import_status == "Failed") {
return [__("Data import failed"), "red", "import_status,=,Failed"];
} else {
return [__("Data import pending"), "green", "import_status,=,"];
}
}
};

24
frappe/core/doctype/data_import/export_template.html Normal file
View file

@ -0,0 +1,24 @@
<div style="margin: 15px 0px;">
<h4>{{ __("Select Columns Manually") }}</h4>
{% for doctype in doctype_list %}
<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
<div class="row">
{% for f in doctype.fields %}
{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %}
{% doctype.reqd||(f.reqd=0);%}
<div class="col-sm-6">
<div class="checkbox" style="margin: 5px 0px;">
<label>
<input type="checkbox" class="select-column-check"
data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
data-doctype="{{ doctype.name }}" checked>
<small>{{ __(f.label) }}</small>
</label>
</div>
</div>
{% endif %}
{% endfor %}
</div>
{% endfor %}
</div>

2
frappe/core/page/data_import_tool/exporter.py → frappe/core/doctype/data_import/exporter.py
View file

@ -9,7 +9,7 @@ import frappe.permissions
import re, csv, os
from frappe.utils.csvutils import UnicodeWriter
from frappe.utils import cstr, formatdate, format_datetime
from frappe.core.page.data_import_tool.data_import_tool import get_data_keys
from frappe.core.doctype.data_import.importer import get_data_keys
from six import string_types
reflags = {

205
frappe/core/page/data_import_tool/importer.py → frappe/core/doctype/data_import/importer.py
View file

@ -15,35 +15,54 @@ from frappe.utils.csvutils import getlink
from frappe.utils.dateutils import parse_date
from frappe.utils.file_manager import save_url
from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url
from frappe.core.page.data_import_tool.data_import_tool import get_data_keys
from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_url_to_form
from six import text_type, string_types
@frappe.whitelist()
def get_data_keys():
return frappe._dict({
"data_separator": _('Start entering data below this line'),
"main_table": _("Table") + ":",
"parent_table": _("Parent Table") + ":",
"columns": _("Column Name") + ":",
"doctype": _("DocType") + ":"
})
@frappe.whitelist()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
skip_errors = True):
skip_errors = True, data_import_doc=None, validate_template=False):
"""upload data"""
if data_import_doc and isinstance(data_import_doc, string_types):
data_import_doc = frappe.get_doc("Data Import", data_import_doc)
if data_import_doc and from_data_import == "Yes":
no_email = data_import_doc.no_email
ignore_encoding_errors = data_import_doc.ignore_encoding_errors
update_only = data_import_doc.only_update
submit_after_import = data_import_doc.submit_after_import
overwrite = data_import_doc.overwrite
skip_errors = data_import_doc.skip_errors
else:
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.in_import = True
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.mute_emails = no_email
def get_data_keys_definition():
@ -55,9 +74,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
def check_data_length():
max_rows = 5000
if not data:
frappe.throw(_("No data found"))
elif not via_console and len(data) > max_rows:
frappe.throw(_("Only allowed {0} rows in one import").format(max_rows))
frappe.throw(_("No data found in the file. Please reattach the new file with data."))
def get_start_row():
for i, row in enumerate(rows):
@ -111,6 +128,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
if doctypes:
doc = {}
for idx in range(start_idx, len(rows)):
last_error_row_idx = idx # pylint: disable=W0612
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
@ -119,7 +137,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
if not fieldname:
if not fieldname or not rows[idx][column_idx]:
continue
d[fieldname] = rows[idx][column_idx]
@ -176,10 +194,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
return doc
def main_doc_empty(row):
return not (row and filter(None, row))
return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))
def validate_naming(doc):
autoname = frappe.get_meta(doc['doctype']).autoname
autoname = frappe.get_meta(doctype).autoname
if ".#" in autoname or "hash" in autoname:
autoname = ""
@ -238,19 +256,18 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
# header
filename, file_extension = ['','']
if not rows:
from frappe.utils.file_manager import get_file_doc
file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1)
filename, file_extension = os.path.splitext(file_doc.file_name)
from frappe.utils.file_manager import get_file # get_file_doc
fname, fcontent = get_file(data_import_doc.import_file)
filename, file_extension = os.path.splitext(fname)
if file_extension == '.xlsx' and from_data_import == 'Yes':
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(file_id=file_doc.name)
rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)
elif file_extension == '.csv':
from frappe.utils.file_manager import get_file
from frappe.utils.csvutils import read_csv_content
fname, fcontent = get_file(file_doc.name)
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
@ -266,6 +283,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
column_idx_to_fieldtype = {}
attachments = []
if skip_errors:
last_error_row_idx = None
data_rows_with_error = header
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
@ -280,14 +301,19 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# allow limit rows to be uploaded
# Throw expception in case of the empty data file
check_data_length()
make_column_map()
total = len(data)
if validate_template:
if total:
data_import_doc.total_rows = total
return True
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
@ -296,13 +322,12 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
if overwrite:
delete_child_rows(data, doctype)
ret = []
def log(msg):
import_log = []
def log(**kwargs):
if via_console:
print(msg.encode('utf-8'))
print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
else:
ret.append(msg)
import_log.append(kwargs)
def as_link(doctype, name):
if via_console:
@ -310,8 +335,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
else:
return getlink(doctype, name)
error = False
total = len(data)
# publish realtime task update
def publish_progress(achieved, reload=False):
if data_import_doc:
frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
error_flag = rollback_flag = False
for i, row in enumerate(data):
# bypass empty rows
if main_doc_empty(row):
@ -320,9 +351,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
row_idx = i + start_row
doc = None
# publish task_update
frappe.publish_realtime("data_import_progress", {"progress": [i, total]},
user=frappe.session.user)
publish_progress(i)
try:
doc = get_doc(row_idx)
@ -330,12 +359,11 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
if pre_process:
pre_process(doc)
original = None
if parentfield:
parent = frappe.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
log('Inserted row for %s at #%s' % (as_link(parenttype,
doc.parent),text_type(doc.idx)))
else:
if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
@ -345,7 +373,6 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name)))
doc = original
else:
if not update_only:
@ -353,29 +380,50 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
else:
log('Ignored row (#%d) %s' % (row_idx + 1, row[1]))
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
# log errors
if parentfield:
log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
elif submit_after_import:
log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
elif original:
log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
elif not update_only:
log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
else:
log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
"message": "Document updation ignored", "indicator": "orange"})
except Exception as e:
if not skip_errors:
error = True
if doc:
frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
log('Error for row (#%d) %s : %s' % (row_idx + 1,
len(row)>1 and row[1] or "", err_msg))
frappe.errprint(frappe.get_traceback())
error_flag = True
err_msg = frappe.local.message_log and "\n".join([json.loads(msg).get('message') for msg in frappe.local.message_log]) or cstr(e)
error_trace = frappe.get_traceback()
if error_trace:
error_log_doc = frappe.log_error(error_trace)
error_link = get_url_to_form("Error Log", error_log_doc.name)
else:
error_link = None
log(**{"row": row_idx + 1, "title":'Error for row %s' % (len(row)>1 and row[1] or ""), "message": err_msg,
"indicator": "red", "link":error_link})
# data with error to create a new file
if skip_errors:
data_rows_with_error += data[row_idx:last_error_row_idx]
else:
rollback_flag = True
finally:
frappe.local.message_log = []
if error:
if rollback_flag:
frappe.db.rollback()
else:
frappe.db.commit()
@ -383,7 +431,40 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
frappe.flags.mute_emails = False
frappe.flags.in_import = False
return {"messages": ret, "error": error}
log_message = {"messages": import_log, "error": error_flag}
if data_import_doc:
data_import_doc.log_details = json.dumps(log_message)
import_status = None
if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
import_status = "Partially Successful"
# write the file with the faulty row
from frappe.utils.file_manager import save_file
file_name = 'error_' + filename + file_extension
if file_extension == '.xlsx':
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
file_data = xlsx_file.getvalue()
else:
from frappe.utils.csvutils import to_csv
file_data = to_csv(data_rows_with_error)
error_data_file = save_file(file_name, file_data, "Data Import",
data_import_doc.name, "Home/Attachments")
data_import_doc.error_file = error_data_file.file_url
elif error_flag:
import_status = "Failed"
else:
import_status = "Successful"
data_import_doc.import_status = import_status
data_import_doc.save()
if data_import_doc.import_status in ["Successful", "Partially Successful"]:
data_import_doc.submit()
frappe.db.commit()
publish_progress(100, True)
else:
return log_message
def get_parent_field(doctype, parenttype):
parentfield = None

38
frappe/core/doctype/data_import/log_details.html Normal file
View file

@ -0,0 +1,38 @@
<div>
<div class="table-responsive">
<table class="table table-bordered table-hover log-details-table">
<tr>
<th style="width:10%"> {{ __("Row No") }} </th>
<th style="width:40%"> {{ __("Row Status") }} </th>
<th style="width:50%"> {{ __("Message") }} </th>
</tr>
{% for row in data %}
{% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %}
<tr>
<td>
<span>{{ row.row }} </span>
</td>
<td>
<span class="indicator {{ row.indicator }}"> {{ row.title }} </span>
</td>
<td>
{% if (import_status != "Failed" || (row.indicator == "red")) { %}
<span> {{ row.message }} </span>
{% if row.link %}
<span style="width: 10%; float:right;">
<a class="btn-open no-decoration" title="Open Link" href="{{ row.link }}">
<i class="octicon octicon-arrow-right"></i>
</a>
</span>
{% endif %}
{% } else { %}
<span> {{ __("Document can't saved.") }} </span>
{% } %}
</td>
</tr>
{% endif %}
{% endfor %}
</table>
</div>
</div>

9
frappe/core/doctype/data_import/test_data_import.py Normal file
View file

@ -0,0 +1,9 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
class TestDataImport(unittest.TestCase):
pass

4
frappe/core/page/data_import_tool/__init__.py
View file

@ -1,4 +0,0 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

118
frappe/core/page/data_import_tool/data_import_main.html
View file

@ -1,118 +0,0 @@
<div class="data-import-tool">
<div class="data-import-selector">
<h3>{%= __("Export Template") %}</h3>
<p class="text-muted">{%= __("To import or update records, you must first download the template for importing.") %}</p>
<h6>{%= __("Select Type of Document to Download") %}</h6>
<div>
<select class="form-control doctype" style="width: 200px" placeholder="{%= __("Select Type") %}">
<option value=""></option>
{% for (var i=0, l= frappe.boot.user.can_import.length; i < l; i++) {
var doctype = frappe.boot.user.can_import[i]; %}
<option value="{%= doctype %}">{%= __(doctype) %}</option>
{% } %}
</select>
<br>
</div>
</div>
<div class="export-import-section hide" style="max-width: 700px;">
<h4>{{ __("1. Select Columns") }}</h4>
<p>
<a class="btn btn-default btn-xs btn-select-all" style="margin-right: 7px;">
{%= __("Select All") %}</a>
<a class="btn btn-default btn-xs btn-select-mandatory" style="margin-right: 7px;">
{%= __("Select Mandatory") %}</a>
<a class="btn btn-default btn-xs btn-unselect-all">
{%= __("Unselect All") %}</a>
</p>
<div class="select-columns">
</div>
<br>
<h4>{{ __("2. Download") }}</h4>
<div class="row">
<div class="col-sm-4">
<p><a class="btn btn-primary btn-xs btn-download-template">
{%= __("Download Blank Template") %}</a></p>
</div>
<div class="col-sm-8">
<h6 class="text-muted">{%= __("Recommended for inserting new records.") %}</h6>
</div>
</div>
<div class="row">
<div class="col-sm-4">
<p><a class="btn btn-primary btn-xs btn-download-data">
{%= __("Download with Data") %}</a></p>
</div>
<div class="col-sm-8">
<h6 class="text-muted">{%= __("Recommended bulk editing records via import, or understanding the import format.") %}</h6>
</div>
</div>
<div class="row">
<div class="col-sm-4">
<div class="checkbox" style="margin: 5px 0px;">
<label>
<input type="checkbox" class="excel-check" data-fieldname="excel_check" checked>
<small>{%= __("Download in Excel File Format") %}</small>
</label>
</div>
</div>
</div>
</div>
<div>
<hr style="margin-top: 50px;">
<h3>{%= __("Import") %}</h3>
<p class="text-muted">{%= __("Update the template and save in downloaded format before attaching.") %}</p>
<div class="row">
<div class="col-md-6">
<br>
<h4>{{ __("1. Select File") }}</h4>
<div class="upload-area"></div>
<br>
<h4>{{ __("2. Upload") }}</h4>
<div class="checkbox">
<label>
<input type="checkbox" name="always_insert">
{%= __("Do not update, but insert new records.") %}
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="update_only">
{%= __("Update only, do not insert new records.") %}
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="submit_after_import">
{%= __("Submit after importing.") %}
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="ignore_encoding_errors">
{%= __("Ignore encoding errors.") %}
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="skip_errors">
{%= __("Skip rows with errors.") %}
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="no_email" checked>
{%= __("Do not send emails.") %}
</label>
</div>
<p>
<button class="btn btn-sm btn-primary btn-import">Import</button>
</p>
</div>
<div class="import-log hide col-md-6">
<h3>Import Log</h3>
<div class="import-log-messages"></div>
</div>
</div>
</div>
</div>

7
frappe/core/page/data_import_tool/data_import_tool.css
View file

@ -1,7 +0,0 @@
.data-import-tool {
padding: 15px;
}
.data-import-tool hr {
margin: 10px -15px;
}

233
frappe/core/page/data_import_tool/data_import_tool.js
View file

@ -1,233 +0,0 @@
frappe.DataImportTool = Class.extend({
init: function(parent) {
this.page = frappe.ui.make_app_page({
parent: parent,
title: __("Data Import Tool"),
single_column: true
});
this.page.add_inner_button(__("Help"), function() {
frappe.help.show_video("6wiriRKPhmg");
});
this.make();
this.make_upload();
},
set_route_options: function() {
var doctype = null;
if(frappe.get_route()[1]) {
doctype = frappe.get_route()[1];
} else if(frappe.route_options && frappe.route_options.doctype) {
doctype = frappe.route_options.doctype;
}
if(in_list(frappe.boot.user.can_import, doctype)) {
this.select.val(doctype).change();
}
frappe.route_options = null;
},
make: function() {
var me = this;
frappe.boot.user.can_import = frappe.boot.user.can_import.sort();
$(frappe.render_template("data_import_main", this)).appendTo(this.page.main);
this.select = this.page.main.find("select.doctype");
this.select_columns = this.page.main.find('.select-columns');
this.select.on("change", function() {
me.doctype = $(this).val();
frappe.model.with_doctype(me.doctype, function() {
me.page.main.find(".export-import-section").toggleClass(!!me.doctype);
if(me.doctype) {
// render select columns
var parent_doctype = frappe.get_doc('DocType', me.doctype);
parent_doctype["reqd"] = true;
var doctype_list = [parent_doctype];
frappe.meta.get_table_fields(me.doctype).forEach(function(df) {
var d = frappe.get_doc('DocType', df.options);
d["reqd"]=df.reqd;
doctype_list.push(d);
});
$(frappe.render_template("data_import_tool_columns", {doctype_list: doctype_list}))
.appendTo(me.select_columns.empty());
}
});
});
this.page.main.find('.btn-select-all').on('click', function() {
me.select_columns.find('.select-column-check').prop('checked', true);
});
this.page.main.find('.btn-unselect-all').on('click', function() {
me.select_columns.find('.select-column-check').prop('checked', false);
});
this.page.main.find('.btn-select-mandatory').on('click', function() {
me.select_columns.find('.select-column-check').prop('checked', false);
me.select_columns.find('.select-column-check[data-reqd="1"]').prop('checked', true);
});
var get_template_url = '/api/method/frappe.core.page.data_import_tool.exporter.get_template';
this.page.main.find(".btn-download-template").on('click', function() {
open_url_post(get_template_url, me.get_export_params(false));
});
this.page.main.find(".btn-download-data").on('click', function() {
open_url_post(get_template_url, me.get_export_params(true));
});
},
get_export_params: function(with_data) {
var doctype = this.select.val();
var columns = {};
this.select_columns.find('.select-column-check:checked').each(function() {
var _doctype = $(this).attr('data-doctype');
var _fieldname = $(this).attr('data-fieldname');
if(!columns[_doctype]) {
columns[_doctype] = [];
}
columns[_doctype].push(_fieldname);
});
return {
doctype: doctype,
parent_doctype: doctype,
select_columns: JSON.stringify(columns),
with_data: with_data ? 'Yes' : 'No',
all_doctypes: 'Yes',
from_data_import: 'Yes',
excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No'
}
},
make_upload: function() {
var me = this;
frappe.upload.make({
no_socketio: true,
parent: this.page.main.find(".upload-area"),
btn: this.page.main.find(".btn-import"),
get_params: function() {
return {
submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
update_only: me.page.main.find('[name="update_only"]').prop("checked"),
no_email: me.page.main.find('[name="no_email"]').prop("checked"),
from_data_import: 'Yes'
}
},
args: {
method: 'frappe.core.page.data_import_tool.importer.upload',
},
allow_multiple: 0,
onerror: function(r) {
me.onerror(r);
},
queued: function() {
// async, show queued
msg_dialog.clear();
frappe.msgprint(__("Import Request Queued. This may take a few moments, please be patient."));
},
running: function() {
// update async status as running
msg_dialog.clear();
frappe.msgprint(__("Importing..."));
me.write_messages([__("Importing")]);
me.has_progress = false;
},
progress: function(data) {
// show callback if async
if(data.progress) {
frappe.hide_msgprint(true);
me.has_progress = true;
frappe.show_progress(__("Importing"), data.progress[0],
data.progress[1]);
}
},
callback: function(attachment, r) {
if(r.message.error || r.message.messages.length==0) {
me.onerror(r);
} else {
if(me.has_progress) {
frappe.show_progress(__("Importing"), 1, 1);
setTimeout(frappe.hide_progress, 1000);
}
r.messages = ["<h5 style='color:green'>" + __("Import Successful!") + "</h5>"].
concat(r.message.messages)
me.write_messages(r.messages);
}
},
is_private: true
});
frappe.realtime.on("data_import_progress", function(data) {
if(data.progress) {
frappe.hide_msgprint(true);
me.has_progress = true;
frappe.show_progress(__("Importing"), data.progress[0],
data.progress[1]);
}
})
},
write_messages: function(data) {
this.page.main.find(".import-log").removeClass("hide");
var parent = this.page.main.find(".import-log-messages").empty();
// TODO render using template!
for (var i=0, l=data.length; i<l; i++) {
var v = data[i];
var $p = $('<p></p>').html(frappe.markdown(v)).appendTo(parent);
if(v.substr(0,5)=='Error') {
$p.css('color', 'red');
} else if(v.substr(0,8)=='Inserted') {
$p.css('color', 'green');
} else if(v.substr(0,7)=='Updated') {
$p.css('color', 'green');
} else if(v.substr(0,5)=='Valid') {
$p.css('color', '#777');
} else if(v.substr(0,7)=='Ignored') {
$p.css('color', '#777');
}
}
},
onerror: function(r) {
if(r.message) {
// bad design: moves r.messages to r.message.messages
r.messages = $.map(r.message.messages, function(v) {
var msg = v.replace("Inserted", "Valid")
.replace("Updated", "Valid").split("<");
if (msg.length > 1) {
v = msg[0] + (msg[1].split(">").slice(-1)[0]);
} else {
v = msg[0];
}
return v;
});
r.messages = ["<h4 style='color:red'>" + __("Import Failed") + "</h4>"]
.concat(r.messages);
r.messages.push("Please correct the format of the file and import again.");
frappe.show_progress(__("Importing"), 1, 1);
this.write_messages(r.messages);
}
}
});
frappe.pages['data-import-tool'].on_page_load = function(wrapper) {
frappe.data_import_tool = new frappe.DataImportTool(wrapper);
}
frappe.pages['data-import-tool'].on_page_show = function(wrapper) {
frappe.data_import_tool && frappe.data_import_tool.set_route_options();
}

19
frappe/core/page/data_import_tool/data_import_tool.json
View file

@ -1,19 +0,0 @@
{
"content": null,
"creation": "2012-06-14 15:07:25",
"docstatus": 0,
"doctype": "Page",
"icon": "fa fa-upload",
"idx": 1,
"modified": "2016-05-11 03:37:53.385693",
"modified_by": "Administrator",
"module": "Core",
"name": "data-import-tool",
"owner": "Administrator",
"page_name": "data-import-tool",
"roles": [],
"script": null,
"standard": "Yes",
"style": null,
"title": "Data Import Tool"
}

22
frappe/core/page/data_import_tool/data_import_tool_columns.html
View file

@ -1,22 +0,0 @@
<div style="margin: 15px 0px;">
{% for doctype in doctype_list %}
<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
<div class="row">
{% for f in doctype.fields %}
{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %}
{% doctype.reqd||(f.reqd=0);%}
<div class="col-sm-4">
<div class="checkbox" style="margin: 5px 0px;">
<label>
<input type="checkbox" class="select-column-check"
data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
data-doctype="{{ doctype.name }}" checked>
<small>{{ __(f.label) }}</small>
</label>
</div>
</div>
{% endif %}
{% endfor %}
</div>
{% endfor %}
</div>

1
frappe/patches.txt
View file

@ -200,3 +200,4 @@ frappe.patches.v9_1.add_sms_sender_name_as_parameters
frappe.patches.v9_1.resave_domain_settings
frappe.patches.v9_1.revert_domain_settings
frappe.patches.v9_1.move_feed_to_activity_log
execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True)

6
frappe/public/css/controls.css
View file

@ -1,5 +1,11 @@
.unit-checkbox {
color: #36414c;
margin-top: 5px;
margin-bottom: 5px;
}
.unit-checkbox + .checkbox {
margin-top: 5px;
margin-bottom: 5px;
}
.frappe-control .select-all {
margin-right: 5px;

7
frappe/public/js/frappe/form/controls/multicheck.js
View file

@ -5,12 +5,12 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
make() {
this._super();
// this.$label = $(`<label class="control-label">${this.df.label}</label>`).appendTo(this.wrapper);
this.$label = $(`<label class="control-label">${this.df.label || ''}</label>`).appendTo(this.wrapper);
this.$load_state = $('<div class="load-state text-muted small">' + __("Loading") + '...</div>');
this.$select_buttons = this.get_select_buttons().appendTo(this.wrapper);
this.$load_state.appendTo(this.wrapper);
this.$checkbox_area = $('<div class="checkbox-options"></div>').appendTo(this.wrapper);
this.$checkbox_area = $('<div class="checkbox-options row"></div>').appendTo(this.wrapper);
this.set_options();
this.bind_checkboxes();
},
@ -120,8 +120,9 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
},
get_checkbox_element(option) {
const column_size = 12 / (this.df.columns || 1);
return $(`
<div class="checkbox unit-checkbox">
<div class="checkbox unit-checkbox col-sm-${column_size}">
<label>
<input type="checkbox" data-unit="${option.value}">
</input>

4
frappe/public/js/frappe/list/list_view.js
View file

@ -601,8 +601,8 @@ frappe.views.ListView = frappe.ui.BaseList.extend({
if (frappe.model.can_import(this.doctype)) {
this.page.add_menu_item(__('Import'), function () {
frappe.set_route('data-import-tool', {
doctype: me.doctype
frappe.set_route('List', 'Data Import', {
reference_doctype: me.doctype
});
}, true);
}

3
frappe/public/js/frappe/model/model.js
View file

@ -67,6 +67,9 @@ $.extend(frappe.model, {
},
is_value_type: function(fieldtype) {
if (typeof fieldtype == 'object') {
fieldtype = fieldtype.fieldtype;
}
// not in no-value type
return frappe.model.no_value_type.indexOf(fieldtype)===-1;
},

9
frappe/public/js/legacy/client_script_helpers.js
View file

@ -359,7 +359,7 @@ _f.Frm.prototype.get_field = function(field) {
};
_f.Frm.prototype.set_read_only = function() {
_f.Frm.prototype.set_read_only = function(refresh_fields = false) {
var perm = [];
var docperms = frappe.perm.get_perm(this.doc.doctype);
for (var i=0, l=docperms.length; i<l; i++) {
@ -367,6 +367,13 @@ _f.Frm.prototype.set_read_only = function() {
perm[p.permlevel || 0] = {read:1, print:1, cancel:1};
}
this.perm = perm;
if (refresh_fields) {
this.fields.map(f => {
f.perm = this.perm;
f.refresh();
});
}
}
_f.Frm.prototype.trigger = function(event) {

9
frappe/public/less/controls.less
View file

@ -1,5 +1,12 @@
.unit-checkbox{
.unit-checkbox {
color: #36414c;
margin-top: 5px;
margin-bottom: 5px;
& + .checkbox {
margin-top: 5px;
margin-bottom: 5px;
}
}
.frappe-control .select-all {

4
frappe/tests/test_data_import.py
View file

@ -3,8 +3,8 @@
from __future__ import unicode_literals
import frappe, unittest
from frappe.core.page.data_import_tool import exporter
from frappe.core.page.data_import_tool import importer
from frappe.core.doctype.data_import import exporter
from frappe.core.doctype.data_import import importer
from frappe.utils.csvutils import read_csv_content
class TestDataImport(unittest.TestCase):

2
frappe/core/page/data_import_tool/test_exporter_fixtures.py → frappe/tests/test_exporter_fixtures.py
View file

@ -4,7 +4,7 @@ from __future__ import unicode_literals
import frappe
import frappe.defaults
from frappe.core.page.data_import_tool.data_import_tool import export_csv
from frappe.core.doctype.data_import.data_import import export_csv
import unittest
import os

8
frappe/translate.py
View file

@ -98,6 +98,8 @@ def get_dict(fortype, name=None):
asset_key = fortype + ":" + (name or "-")
translation_assets = cache.hget("translation_assets", frappe.local.lang, shared=True) or {}
print(translation_assets)
if not asset_key in translation_assets:
if fortype=="doctype":
messages = get_messages_from_doctype(name)
@ -121,9 +123,13 @@ def get_dict(fortype, name=None):
message_dict = make_dict_from_messages(messages)
message_dict.update(get_dict_from_hooks(fortype, name))
print(message_dict)
# remove untranslated
message_dict = {k:v for k, v in iteritems(message_dict) if k!=v}
print(message_dict)
translation_assets[asset_key] = message_dict
cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True)
@ -163,6 +169,8 @@ def make_dict_from_messages(messages, full_dict=None):
for m in messages:
if m[1] in full_dict:
out[m[1]] = full_dict[m[1]]
print(out)
return out
def get_lang_js(fortype, name):

2
frappe/utils/fixtures.py
View file

@ -4,7 +4,7 @@
from __future__ import unicode_literals, print_function
import frappe, os
from frappe.core.page.data_import_tool.data_import_tool import import_doc, export_json
from frappe.core.doctype.data_import.data_import import import_doc, export_json
def sync_fixtures(app=None):
"""Import, overwrite fixtures from `[app]/fixtures`"""