vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Data export tool (#5213)

Browse source 
* Data export tool
New doctype to export data
Child table export option
Ability to add filter

* Filter view refactor
Separated filter css

* Moved exporter script from data import doctype to data export doctype

* some input validation

* removed unwanted file and some style fixes

* removed console log

* hide sidebar

* renamed export method

* added data export link in setup module page

* minor fix

* refactor exporter.py

* data export ui tweaks

* codacy and bug fix

* silly code fixes

* [minor] indentation fix
This commit is contained in:
Suraj Shetty 2018-03-22 11:30:48 +05:30 • committed by Rushabh Mehta
parent 29baefefe3
commit 436d7a0d4e
17 changed files with 872 additions and 1110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
frappe/config/setup.py
View file

@ -95,9 +95,16 @@ def get_data():
{
"type": "doctype",
"name": "Data Import",
"label": _("Import / Export Data"),
"label": _("Import Data"),
"icon": "octicon octicon-cloud-upload",
"description": _("Import / Export Data from CSV and Excel files.")
"description": _("Import Data from CSV / Excel files.")
},
{
"type": "doctype",
"name": "Data Export",
"label": _("Export Data"),
"icon": "octicon octicon-cloud-upload",
"description": _("Export Data in CSV / Excel format.")
},
{
"type": "doctype",

0
frappe/core/doctype/data_export/__init__.py Normal file
View file

138
frappe/core/doctype/data_export/data_export.js Normal file
View file

@ -0,0 +1,138 @@
// Copyright (c) 2018, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Export', {
refresh: frm => {
frm.disable_save();
frm.page.set_primary_action('Export', () => {
can_export(frm) ? export_data(frm) : null;
});
$(frm.footer.wrapper).toggle(false);
},
onload: (frm) => {
frm.set_query("reference_doctype", () => {
return {
"filters": {
"issingle": 0,
"istable": 0,
"name": ['in', frappe.boot.user.can_export]
}
};
});
},
reference_doctype: frm => {
const doctype = frm.doc.reference_doctype;
if (doctype) {
frappe.model.with_doctype(doctype, () => set_field_options(frm));
} else {
reset_filter_and_field(frm);
}
}
});
const can_export = frm => {
const doctype = frm.doc.reference_doctype;
const parent_multicheck_options = frm.fields_multicheck[doctype] ?
frm.fields_multicheck[doctype].get_checked_options() : [];
let is_valid_form = false;
if (!doctype) {
frappe.msgprint(__('Please select the Document Type.'));
} else if (!parent_multicheck_options.length) {
frappe.msgprint(__('Atleast one field of Parent Document Type is mandatory'));
} else {
is_valid_form = true;
}
return is_valid_form;
};
const export_data = frm => {
let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data';
var export_params = () => {
let columns = {};
Object.keys(frm.fields_multicheck).forEach(dt => {
const options = frm.fields_multicheck[dt].get_checked_options();
columns[dt] = options;
});
return {
doctype: frm.doc.reference_doctype,
select_columns: JSON.stringify(columns),
filters: frm.filter_list.get_filters().map(filter => filter.slice(1, 4)),
file_type: frm.doc.file_type,
template: true,
with_data: true
};
};
open_url_post(get_template_url, export_params());
};
const reset_filter_and_field = (frm) => {
const parent_wrapper = frm.fields_dict.fields_multicheck.$wrapper;
const filter_wrapper = frm.fields_dict.filter_list.$wrapper;
parent_wrapper.empty();
filter_wrapper.empty();
frm.filter_list = [];
frm.fields_multicheck = {};
};
const set_field_options = (frm) => {
const parent_wrapper = frm.fields_dict.fields_multicheck.$wrapper;
const filter_wrapper = frm.fields_dict.filter_list.$wrapper;
const doctype = frm.doc.reference_doctype;
const related_doctypes = get_doctypes(doctype);
parent_wrapper.empty();
filter_wrapper.empty();
frm.filter_list = new frappe.ui.FilterGroup({
parent: filter_wrapper,
doctype: doctype,
on_change: () => { },
});
frm.fields_multicheck = {};
related_doctypes.forEach(dt => {
frm.fields_multicheck[dt] = add_doctype_field_multicheck_control(dt, parent_wrapper);
});
frm.refresh();
};
const get_doctypes = parentdt => {
return [parentdt].concat(
frappe.meta.get_table_fields(parentdt).map(df => df.options)
);
};
const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => {
const fields = get_fields(doctype);
const options = fields
.map(df => {
return {
label: df.label + (df.reqd ? ' (M)' : ''),
value: df.fieldname,
checked: 1
};
});
const multicheck_control = frappe.ui.form.make_control({
parent: parent_wrapper,
df: {
"label": doctype,
"fieldname": doctype + '_fields',
"fieldtype": "MultiCheck",
"options": options,
"select_all": options.length > 5,
"columns": 3,
"hidden": 1,
},
render_input: true
});
multicheck_control.refresh_input();
return multicheck_control;
};
const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);

250
frappe/core/doctype/data_export/data_export.json Normal file
View file

@ -0,0 +1,250 @@
{
"allow_copy": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"beta": 0,
"creation": "2018-03-07 10:09:49.794764",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "reference_doctype",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Select Doctype",
"length": 0,
"no_copy": 0,
"options": "DocType",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_2",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "CSV",
"fieldname": "file_type",
"fieldtype": "Select",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "File Type",
"length": 0,
"no_copy": 0,
"options": "Excel\nCSV",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "reference_doctype",
"fieldname": "section_break",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "filter_list",
"fieldtype": "HTML",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Filter List",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "fields_multicheck",
"fieldtype": "HTML",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Fields Multicheck",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 1,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 1,
"istable": 0,
"max_attachments": 0,
"modified": "2018-03-21 13:23:05.623052",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Export",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"apply_user_permissions": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 0,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 1,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 0,
"track_seen": 0
}

9
frappe/core/doctype/data_export/data_export.py Normal file
View file

@ -0,0 +1,9 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from frappe.model.document import Document
class DataExport(Document):
pass

340
frappe/core/doctype/data_export/exporter.py Normal file
View file

@ -0,0 +1,340 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from frappe import _
import frappe.permissions
import re, csv, os
from frappe.utils.csvutils import UnicodeWriter
from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint
from frappe.core.doctype.data_import.importer import get_data_keys
from six import string_types
reflags = {
"I":re.I,
"L":re.L,
"M":re.M,
"U":re.U,
"S":re.S,
"X":re.X,
"D": re.DEBUG
}
@frappe.whitelist()
def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False,
select_columns=None, file_type='CSV', template=False, filters=None):
exporter = DataExporter(doctype=doctype, parent_doctype=parent_doctype, all_doctypes=all_doctypes, with_data=with_data,
select_columns=select_columns, file_type=file_type, template=template, filters=filters)
exporter.build_response()
class DataExporter():
def __init__(self, doctype=None, parent_doctype=None, all_doctypes=True, with_data=False,
select_columns=None, file_type='CSV', template=False, filters=None):
self.doctype = doctype
self.parent_doctype = parent_doctype
self.all_doctypes = all_doctypes
self.with_data = cint(with_data)
self.select_columns = select_columns
self.file_type = file_type
self.template = template
self.filters = filters
self.data_keys = get_data_keys()
self.prepare_args()
def prepare_args(self):
if self.select_columns:
self.select_columns = parse_json(self.select_columns)
if self.filters:
self.filters = parse_json(self.filters)
self.docs_to_export = {}
if self.doctype:
if isinstance(self.doctype, string_types):
self.doctype = [self.doctype]
if len(self.doctype) > 1:
self.docs_to_export = self.doctype[1]
self.doctype = self.doctype[0]
if not self.parent_doctype:
self.parent_doctype = self.doctype
self.column_start_end = {}
if self.all_doctypes:
self.child_doctypes = []
for df in frappe.get_meta(self.doctype).get_table_fields():
self.child_doctypes.append(dict(doctype=df.options, parentfield=df.fieldname))
def build_response(self):
self.writer = UnicodeWriter()
self.name_field = 'parent' if self.parent_doctype != self.doctype else 'name'
if self.template:
self.add_main_header()
self.writer.writerow([''])
self.tablerow = [self.data_keys.doctype, ""]
self.labelrow = [_("Column Labels:"), "ID"]
self.fieldrow = [self.data_keys.columns, self.name_field]
self.mandatoryrow = [_("Mandatory:"), _("Yes")]
self.typerow = [_('Type:'), 'Data (text)']
self.inforow = [_('Info:'), '']
self.columns = [self.name_field]
self.build_field_columns(self.doctype)
if self.all_doctypes:
for d in self.child_doctypes:
self.append_empty_field_column()
if (self.select_columns and self.select_columns.get(d['doctype'], None)) or not self.select_columns:
# if atleast one column is selected for this doctype
self.build_field_columns(d['doctype'], d['parentfield'])
self.add_field_headings()
self.add_data()
if self.with_data and not self.data:
frappe.respond_as_web_page(_('No Data'), _('There is no data to be exported'), indicator_color='orange')
return
if self.file_type == 'Excel':
return self.build_response_as_excel()
else:
# write out response as a type csv
frappe.response['result'] = cstr(self.writer.getvalue())
frappe.response['type'] = 'csv'
frappe.response['doctype'] = self.doctype
def add_main_header(self):
self.writer.writerow([_('Data Import Template')])
self.writer.writerow([self.data_keys.main_table, self.doctype])
if self.parent_doctype != self.doctype:
self.writer.writerow([self.data_keys.parent_table, self.parent_doctype])
else:
self.writer.writerow([''])
self.writer.writerow([''])
self.writer.writerow([_('Notes:')])
self.writer.writerow([_('Please do not change the template headings.')])
self.writer.writerow([_('First data column must be blank.')])
self.writer.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')])
self.writer.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')])
self.writer.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')])
self.writer.writerow([_('For updating, you can update only selective columns.')])
self.writer.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')])
if self.name_field == "parent":
self.writer.writerow([_('"Parent" signifies the parent table in which this row must be added')])
self.writer.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')])
def build_field_columns(self, dt, parentfield=None):
meta = frappe.get_meta(dt)
# build list of valid docfields
tablecolumns = []
for f in frappe.db.sql('desc `tab%s`' % dt):
field = meta.get_field(f[0])
if field and ((self.select_columns and f[0] in self.select_columns[dt]) or not self.select_columns):
tablecolumns.append(field)
tablecolumns.sort(key = lambda a: int(a.idx))
_column_start_end = frappe._dict(start=0)
if dt==self.doctype:
_column_start_end = frappe._dict(start=0)
else:
_column_start_end = frappe._dict(start=len(self.columns))
self.append_field_column(frappe._dict({
"fieldname": "name",
"parent": dt,
"label": "ID",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": _("Leave blank for new records")
}), True)
for docfield in tablecolumns:
self.append_field_column(docfield, True)
# all non mandatory fields
for docfield in tablecolumns:
self.append_field_column(docfield, False)
# if there is one column, add a blank column (?)
if len(self.columns)-_column_start_end.start == 1:
self.append_empty_field_column()
# append DocType name
self.tablerow[_column_start_end.start + 1] = dt
if parentfield:
self.tablerow[_column_start_end.start + 2] = parentfield
_column_start_end.end = len(self.columns) + 1
self.column_start_end[(dt, parentfield)] = _column_start_end
def append_field_column(self, docfield, for_mandatory):
if not docfield:
return
if for_mandatory and not docfield.reqd:
return
if not for_mandatory and docfield.reqd:
return
if docfield.fieldname in ('parenttype', 'trash_reason'):
return
if docfield.hidden:
return
if self.select_columns and docfield.fieldname not in self.select_columns.get(docfield.parent, []):
return
self.tablerow.append("")
self.fieldrow.append(docfield.fieldname)
self.labelrow.append(_(docfield.label))
self.mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
self.typerow.append(docfield.fieldtype)
self.inforow.append(self.getinforow(docfield))
self.columns.append(docfield.fieldname)
def append_empty_field_column(self):
self.tablerow.append("~")
self.fieldrow.append("~")
self.labelrow.append("")
self.mandatoryrow.append("")
self.typerow.append("")
self.inforow.append("")
self.columns.append("")
@staticmethod
def getinforow(docfield):
"""make info comment for options, links etc."""
if docfield.fieldtype == 'Select':
if not docfield.options:
return ''
else:
return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n')))
elif docfield.fieldtype == 'Link':
return 'Valid %s' % docfield.options
elif docfield.fieldtype == 'Int':
return 'Integer'
elif docfield.fieldtype == "Check":
return "0 or 1"
elif docfield.fieldtype in ["Date", "Datetime"]:
return cstr(frappe.defaults.get_defaults().date_format)
elif hasattr(docfield, "info"):
return docfield.info
else:
return ''
def add_field_headings(self):
self.writer.writerow(self.tablerow)
self.writer.writerow(self.labelrow)
self.writer.writerow(self.fieldrow)
self.writer.writerow(self.mandatoryrow)
self.writer.writerow(self.typerow)
self.writer.writerow(self.inforow)
if self.template:
self.writer.writerow([self.data_keys.data_separator])
def add_data(self):
if self.template and not self.with_data:
return
frappe.permissions.can_export(self.parent_doctype, raise_exception=True)
# sort nested set doctypes by `lft asc`
order_by = None
table_columns = frappe.db.get_table_columns(self.parent_doctype)
if 'lft' in table_columns and 'rgt' in table_columns:
order_by = '`tab{doctype}`.`lft` asc'.format(doctype=self.parent_doctype)
# get permitted data only
self.data = frappe.get_list(self.doctype, fields=["*"], filters=self.filters, limit_page_length=None, order_by=order_by)
for doc in self.data:
op = self.docs_to_export.get("op")
names = self.docs_to_export.get("name")
if names and op:
if op == '=' and doc.name not in names:
continue
elif op == '!=' and doc.name in names:
continue
elif names:
try:
sflags = self.docs_to_export.get("flags", "I,U").upper()
flags = 0
for a in re.split('\W+',sflags):
flags = flags | reflags.get(a,0)
c = re.compile(names, flags)
m = c.match(doc.name)
if not m:
continue
except Exception:
if doc.name not in names:
continue
# add main table
rows = []
self.add_data_row(rows, self.doctype, None, doc, 0)
if self.all_doctypes:
# add child tables
for c in self.child_doctypes:
for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}`
where parent=%s and parentfield=%s order by idx""".format(c['doctype']),
(doc.name, c['parentfield']), as_dict=1)):
self.add_data_row(rows, c['doctype'], c['parentfield'], child, ci)
for row in rows:
self.writer.writerow(row)
def add_data_row(self, rows, dt, parentfield, doc, rowidx):
d = doc.copy()
meta = frappe.get_meta(dt)
if self.all_doctypes:
d.name = '"'+ d.name+'"'
if len(rows) < rowidx + 1:
rows.append([""] * (len(self.columns) + 1))
row = rows[rowidx]
_column_start_end = self.column_start_end.get((dt, parentfield))
if _column_start_end:
for i, c in enumerate(self.columns[_column_start_end.start:_column_start_end.end]):
df = meta.get_field(c)
fieldtype = df.fieldtype if df else "Data"
value = d.get(c, "")
if value:
if fieldtype == "Date":
value = formatdate(value)
elif fieldtype == "Datetime":
value = format_datetime(value)
row[_column_start_end.start + i + 1] = value
def build_response_as_excel(self):
filename = frappe.generate_hash("", 10)
with open(filename, 'wb') as f:
f.write(cstr(self.writer.getvalue()).encode("utf-8"))
f = open(filename)
reader = csv.reader(f)
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else 'Data Export')
f.close()
os.remove(filename)
# write out response as a xlsx type
frappe.response['filename'] = self.doctype + '.xlsx'
frappe.response['filecontent'] = xlsx_file.getvalue()
frappe.response['type'] = 'binary'

33
frappe/core/doctype/data_import/data_import.js
View file

@ -22,7 +22,7 @@ frappe.ui.form.on('Data Import', {
let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar");
if (progress_bar) {
$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
$(progress_bar).css("width", data.progress+"%");
$(progress_bar).css("width", data.progress + "%");
}
}
}
@ -38,10 +38,10 @@ frappe.ui.form.on('Data Import', {
if (frm.doc.import_status) {
const listview_settings = frappe.listview_settings['Data Import'];
const indicator = listview_settings.get_indicator(frm.doc);
frm.page.set_indicator(indicator[0], indicator[1]);
if (frm.doc.import_status==="In Progress") {
if (frm.doc.import_status === "In Progress") {
frm.dashboard.add_progress("Data Import Progress", "0");
frm.set_read_only();
frm.refresh_fields();
@ -57,14 +57,14 @@ frappe.ui.form.on('Data Import', {
frappe.help.show_video("6wiriRKPhmg");
});
if(frm.doc.reference_doctype && frm.doc.docstatus === 0) {
if (frm.doc.reference_doctype && frm.doc.docstatus === 0) {
frm.add_custom_button(__("Download template"), function() {
frappe.data_import.download_dialog(frm).show();
});
}
if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows &&
frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status=="Failed")) {
frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) {
frm.page.set_primary_action(__("Start Import"), function() {
frappe.call({
method: "frappe.core.doctype.data_import.data_import.import_data",
@ -88,10 +88,6 @@ frappe.ui.form.on('Data Import', {
}
},
// import_file: function(frm) {
// frm.save();
// },
overwrite: function(frm) {
if (frm.doc.overwrite === 0) {
frm.doc.only_update = 0;
@ -126,8 +122,11 @@ frappe.ui.form.on('Data Import', {
create_log_table: function(frm) {
let msg = JSON.parse(frm.doc.log_details);
var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
$(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors,
import_status: frm.doc.import_status})).appendTo($log_wrapper);
$(frappe.render_template("log_details", {
data: msg.messages,
import_status: frm.doc.import_status,
show_only_errors: frm.doc.show_only_errors,
})).appendTo($log_wrapper);
}
});
@ -146,7 +145,7 @@ frappe.data_import.download_dialog = function(frm) {
const get_doctype_checkbox_fields = () => {
return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
.map(df => dialog.fields_dict[df.fieldname]);
}
};
const doctype_fields = get_fields(frm.doc.reference_doctype)
.map(df => ({
@ -245,13 +244,13 @@ frappe.data_import.download_dialog = function(frm) {
doctype: frm.doc.reference_doctype,
parent_doctype: frm.doc.reference_doctype,
select_columns: JSON.stringify(columns),
with_data: data.with_data ? 'Yes' : 'No',
all_doctypes: 'Yes',
from_data_import: 'Yes',
excel_format: data.file_type === 'Excel' ? 'Yes' : 'No'
with_data: data.with_data,
all_doctypes: true,
file_type: data.file_type,
template: true
};
};
let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template';
let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data';
open_url_post(get_template_url, export_params());
} else {
frappe.msgprint(__("Please select the Document Type."));

4
frappe/core/doctype/data_import/data_import.py
View file

@ -99,9 +99,9 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None, order_b
def export_csv(doctype, path):
from frappe.core.doctype.data_import.exporter import get_template
from frappe.core.doctype.data_export.exporter import export_data
with open(path, "wb") as csvfile:
get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True)
csvfile.write(frappe.response.result.encode("utf-8"))

309
frappe/core/doctype/data_import/exporter.py
View file

@ -1,309 +0,0 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe, json
from frappe import _
import frappe.permissions
import re, csv, os
from frappe.utils.csvutils import UnicodeWriter
from frappe.utils import cstr, formatdate, format_datetime
from frappe.core.doctype.data_import.importer import get_data_keys
from six import string_types
reflags = {
"I":re.I,
"L":re.L,
"M":re.M,
"U":re.U,
"S":re.S,
"X":re.X,
"D": re.DEBUG
}
@frappe.whitelist()
def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None,
from_data_import="No", excel_format="No"):
all_doctypes = all_doctypes=="Yes"
if select_columns:
select_columns = json.loads(select_columns);
docs_to_export = {}
if doctype:
if isinstance(doctype, string_types):
doctype = [doctype];
if len(doctype) > 1:
docs_to_export = doctype[1]
doctype = doctype[0]
if not parent_doctype:
parent_doctype = doctype
column_start_end = {}
if all_doctypes:
child_doctypes = []
for df in frappe.get_meta(doctype).get_table_fields():
child_doctypes.append(dict(doctype=df.options, parentfield=df.fieldname))
def get_data_keys_definition():
return get_data_keys()
def add_main_header():
w.writerow([_('Data Import Template')])
w.writerow([get_data_keys_definition().main_table, doctype])
if parent_doctype != doctype:
w.writerow([get_data_keys_definition().parent_table, parent_doctype])
else:
w.writerow([''])
w.writerow([''])
w.writerow([_('Notes:')])
w.writerow([_('Please do not change the template headings.')])
w.writerow([_('First data column must be blank.')])
w.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')])
w.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')])
w.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')])
w.writerow([_('For updating, you can update only selective columns.')])
w.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')])
if key == "parent":
w.writerow([_('"Parent" signifies the parent table in which this row must be added')])
w.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')])
def build_field_columns(dt, parentfield=None):
meta = frappe.get_meta(dt)
# build list of valid docfields
tablecolumns = []
for f in frappe.db.sql('desc `tab%s`' % dt):
field = meta.get_field(f[0])
if field and ((select_columns and f[0] in select_columns[dt]) or not select_columns):
tablecolumns.append(field)
tablecolumns.sort(key = lambda a: int(a.idx))
_column_start_end = frappe._dict(start=0)
if dt==doctype:
_column_start_end = frappe._dict(start=0)
else:
_column_start_end = frappe._dict(start=len(columns))
append_field_column(frappe._dict({
"fieldname": "name",
"parent": dt,
"label": "ID",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": _("Leave blank for new records")
}), True)
for docfield in tablecolumns:
append_field_column(docfield, True)
# all non mandatory fields
for docfield in tablecolumns:
append_field_column(docfield, False)
# if there is one column, add a blank column (?)
if len(columns)-_column_start_end.start == 1:
append_empty_field_column()
# append DocType name
tablerow[_column_start_end.start + 1] = dt
if parentfield:
tablerow[_column_start_end.start + 2] = parentfield
_column_start_end.end = len(columns) + 1
column_start_end[(dt, parentfield)] = _column_start_end
def append_field_column(docfield, for_mandatory):
if not docfield:
return
if for_mandatory and not docfield.reqd:
return
if not for_mandatory and docfield.reqd:
return
if docfield.fieldname in ('parenttype', 'trash_reason'):
return
if docfield.hidden:
return
if select_columns and docfield.fieldname not in select_columns.get(docfield.parent, []):
return
tablerow.append("")
fieldrow.append(docfield.fieldname)
labelrow.append(_(docfield.label))
mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(docfield.fieldname)
def append_empty_field_column():
tablerow.append("~")
fieldrow.append("~")
labelrow.append("")
mandatoryrow.append("")
typerow.append("")
inforow.append("")
columns.append("")
def getinforow(docfield):
"""make info comment for options, links etc."""
if docfield.fieldtype == 'Select':
if not docfield.options:
return ''
else:
return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n')))
elif docfield.fieldtype == 'Link':
return 'Valid %s' % docfield.options
elif docfield.fieldtype == 'Int':
return 'Integer'
elif docfield.fieldtype == "Check":
return "0 or 1"
elif docfield.fieldtype in ["Date", "Datetime"]:
return cstr(frappe.defaults.get_defaults().date_format)
elif hasattr(docfield, "info"):
return docfield.info
else:
return ''
def add_field_headings():
w.writerow(tablerow)
w.writerow(labelrow)
w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
w.writerow(inforow)
w.writerow([get_data_keys_definition().data_separator])
def add_data():
def add_data_row(row_group, dt, parentfield, doc, rowidx):
d = doc.copy()
meta = frappe.get_meta(dt)
if all_doctypes:
d.name = '"'+ d.name+'"'
if len(row_group) < rowidx + 1:
row_group.append([""] * (len(columns) + 1))
row = row_group[rowidx]
_column_start_end = column_start_end.get((dt, parentfield))
if _column_start_end:
for i, c in enumerate(columns[_column_start_end.start:_column_start_end.end]):
df = meta.get_field(c)
fieldtype = df.fieldtype if df else "Data"
value = d.get(c, "")
if value:
if fieldtype == "Date":
value = formatdate(value)
elif fieldtype == "Datetime":
value = format_datetime(value)
row[_column_start_end.start + i + 1] = value
if with_data=='Yes':
frappe.permissions.can_export(parent_doctype, raise_exception=True)
# sort nested set doctypes by `lft asc`
order_by = None
table_columns = frappe.db.get_table_columns(parent_doctype)
if 'lft' in table_columns and 'rgt' in table_columns:
order_by = '`tab{doctype}`.`lft` asc'.format(doctype=parent_doctype)
# get permitted data only
data = frappe.get_list(doctype, fields=["*"], limit_page_length=None, order_by=order_by)
for doc in data:
op = docs_to_export.get("op")
names = docs_to_export.get("name")
if names and op:
if op == '=' and doc.name not in names:
continue
elif op == '!=' and doc.name in names:
continue
elif names:
try:
sflags = docs_to_export.get("flags", "I,U").upper()
flags = 0
for a in re.split('\W+',sflags):
flags = flags | reflags.get(a,0)
c = re.compile(names, flags)
m = c.match(doc.name)
if not m:
continue
except:
if doc.name not in names:
continue
# add main table
row_group = []
add_data_row(row_group, doctype, None, doc, 0)
if all_doctypes:
# add child tables
for c in child_doctypes:
for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}`
where parent=%s and parentfield=%s order by idx""".format(c['doctype']),
(doc.name, c['parentfield']), as_dict=1)):
add_data_row(row_group, c['doctype'], c['parentfield'], child, ci)
for row in row_group:
w.writerow(row)
w = UnicodeWriter()
key = 'parent' if parent_doctype != doctype else 'name'
add_main_header()
w.writerow([''])
tablerow = [get_data_keys_definition().doctype, ""]
labelrow = [_("Column Labels:"), "ID"]
fieldrow = [get_data_keys_definition().columns, key]
mandatoryrow = [_("Mandatory:"), _("Yes")]
typerow = [_('Type:'), 'Data (text)']
inforow = [_('Info:'), '']
columns = [key]
build_field_columns(doctype)
if all_doctypes:
for d in child_doctypes:
append_empty_field_column()
if (select_columns and select_columns.get(d['doctype'], None)) or not select_columns:
# if atleast one column is selected for this doctype
build_field_columns(d['doctype'], d['parentfield'])
add_field_headings()
add_data()
if from_data_import == "Yes" and excel_format == "Yes":
filename = frappe.generate_hash("", 10)
with open(filename, 'wb') as f:
f.write(cstr(w.getvalue()).encode("utf-8"))
f = open(filename)
reader = csv.reader(f)
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(reader, "Data Import Template")
f.close()
os.remove(filename)
# write out response as a xlsx type
frappe.response['filename'] = doctype + '.xlsx'
frappe.response['filecontent'] = xlsx_file.getvalue()
frappe.response['type'] = 'binary'
else:
# write out response as a type csv
frappe.response['result'] = cstr(w.getvalue())
frappe.response['type'] = 'csv'
frappe.response['doctype'] = doctype

22
frappe/core/doctype/data_import/test_data_import.py
View file

@ -4,24 +4,24 @@
from __future__ import unicode_literals
import frappe, unittest
from frappe.core.doctype.data_import import exporter
from frappe.core.doctype.data_export import exporter
from frappe.core.doctype.data_import import importer
from frappe.utils.csvutils import read_csv_content
class TestDataImport(unittest.TestCase):
def test_export(self):
exporter.get_template("User", all_doctypes="No", with_data="No")
exporter.export_data("User", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
self.assertTrue(content[1][1], "User")
def test_export_with_data(self):
exporter.get_template("User", all_doctypes="No", with_data="Yes")
exporter.export_data("User", all_doctypes=True, template=True, with_data=True)
content = read_csv_content(frappe.response.result)
self.assertTrue(content[1][1], "User")
self.assertTrue("Administrator" in [c[1] for c in content if len(c)>1])
self.assertTrue('"Administrator"' in [c[1] for c in content if len(c)>1])
def test_export_with_all_doctypes(self):
exporter.get_template("User", all_doctypes="Yes", with_data="Yes")
exporter.export_data("User", all_doctypes="Yes", template=True, with_data=True)
content = read_csv_content(frappe.response.result)
self.assertTrue(content[1][1], "User")
self.assertTrue('"Administrator"' in [c[1] for c in content if len(c)>1])
@ -33,14 +33,14 @@ class TestDataImport(unittest.TestCase):
if frappe.db.exists("Blog Category", "test-category"):
frappe.delete_doc("Blog Category", "test-category")
exporter.get_template("Blog Category", all_doctypes="No", with_data="No")
exporter.export_data("Blog Category", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "", "test-category", "Test Cateogry"])
importer.upload(content)
self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "Test Category")
# export with data
exporter.get_template("Blog Category", all_doctypes="No", with_data="Yes")
exporter.export_data("Blog Category", all_doctypes=True, template=True, with_data=True)
content = read_csv_content(frappe.response.result)
# overwrite
@ -55,7 +55,7 @@ class TestDataImport(unittest.TestCase):
frappe.get_doc({"doctype": "User", "email": user_email, "first_name": "Test Import UserRole"}).insert()
exporter.get_template("Has Role", "User", all_doctypes="No", with_data="No")
exporter.export_data("Has Role", "User", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "test_import_userrole@example.com", "Blogger"])
importer.upload(content)
@ -65,7 +65,7 @@ class TestDataImport(unittest.TestCase):
self.assertTrue(user.get("roles")[0].role, "Blogger")
# overwrite
exporter.get_template("Has Role", "User", all_doctypes="No", with_data="No")
exporter.export_data("Has Role", "User", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "test_import_userrole@example.com", "Website Manager"])
importer.upload(content, overwrite=True)
@ -77,7 +77,7 @@ class TestDataImport(unittest.TestCase):
def test_import_with_children(self): #pylint: disable=R0201
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.get_template("Event", all_doctypes="Yes", with_data="No")
exporter.export_data("Event", all_doctypes="Yes", template=True)
content = read_csv_content(frappe.response.result)
content.append([None] * len(content[-2]))
@ -93,7 +93,7 @@ class TestDataImport(unittest.TestCase):
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes")
exporter.export_data("Event", all_doctypes=True, template=True, file_type="Excel")
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent)
content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"])

3
frappe/public/build.json
View file

@ -97,7 +97,8 @@
"public/less/mobile.less",
"public/less/kanban.less",
"public/less/controls.less",
"public/less/chat.less"
"public/less/chat.less",
"public/less/filters.less"
],
"css/frappe-rtl.css": [
"public/css/bootstrap-rtl.css",

1
frappe/public/js/frappe/ui/filters/filter.js
View file

@ -65,6 +65,7 @@ frappe.ui.Filter = class {
this.filter_edit_area.find(".set-filter-and-run").on("click", () => {
this.filter_edit_area.removeClass("new-filter");
this.on_change();
this.update_filter_tag();
});
this.filter_edit_area.find('.condition').change(() => {

678
frappe/public/js/frappe/ui/filters/filters.js
View file

@ -1,678 +0,0 @@
// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
// MIT License. See license.txt
frappe.ui.FilterList = Class.extend({
init: function(opts) {
$.extend(this, opts);
this.filters = [];
this.wrapper = this.parent;
this.stats = [];
this.make();
this.set_events();
},
make: function() {
this.wrapper.find('.show_filters, .filter_area').remove();
this.wrapper.append(`
<div class="show_filters">
<div class="set-filters">
<button
style="margin-right: 10px;"
class="btn btn-default btn-xs new-filter text-muted">
${__("Add Filter")}</button>
</div>
</div>
<div class="filter_area"></div>`);
},
set_events: function() {
var me = this;
// show filters
this.wrapper.find('.new-filter').bind('click', function() {
me.add_filter();
});
this.wrapper.find('.clear-filters').bind('click', function() {
me.clear_filters();
$('.date-range-picker').val('')
me.base_list.run();
$(this).addClass("hide");
});
},
show_filters: function() {
this.wrapper.find('.show_filters').toggle();
if(!this.filters.length) {
this.add_filter(this.doctype, 'name');
this.filters[0].wrapper.find(".filter_field input").focus();
}
},
clear_filters: function() {
$.each(this.filters, function(i, f) { f.remove(true); });
if(this.base_list.page.fields_dict) {
$.each(this.base_list.page.fields_dict, (key, value) => {
value.set_input('');
});
}
this.filters = [];
},
add_filter: function(doctype, fieldname, condition, value, hidden) {
// adds a new filter, returns true if filter has been added
// allow equal to be used as like
let base_filter = this.base_list.page.fields_dict[fieldname];
if (base_filter
&& (base_filter.df.condition==condition
|| (condition==='=' && base_filter.df.condition==='like'))) {
// if filter exists in base_list, then exit
this.base_list.page.fields_dict[fieldname].set_input(value);
return true;
}
if(doctype && fieldname
&& !frappe.meta.has_field(doctype, fieldname)
&& !in_list(frappe.model.std_fields_list, fieldname)) {
frappe.msgprint({
message: __('Filter {0} missing', [fieldname.bold()]),
title: 'Invalid Filter',
indicator: 'red'
});
return false;
}
this.wrapper.find('.show_filters').toggle(true);
var is_new_filter = arguments.length===0;
if (is_new_filter && this.wrapper.find(".is-new-filter:visible").length) {
// only allow 1 new filter at a time!
return false;
}
var filter = this.push_new_filter(doctype, fieldname, condition, value);
if (!filter) return;
if(this.wrapper.find('.clear-filters').hasClass("hide")) {
this.wrapper.find('.clear-filters').removeClass("hide");
}
if (filter && is_new_filter) {
filter.wrapper.addClass("is-new-filter");
} else {
filter.freeze();
}
if (hidden) {
filter.$btn_group.addClass("hide");
}
return true;
},
push_new_filter: function(doctype, fieldname, condition, value) {
if(this.filter_exists(doctype, fieldname, condition, value)) {
return;
}
// if standard filter exists, then clear it.
if(this.base_list.page.fields_dict[fieldname]) {
this.base_list.page.fields_dict[fieldname].set_input('');
}
var filter = new frappe.ui.Filter({
flist: this,
_doctype: doctype,
fieldname: fieldname,
condition: condition,
value: value
});
this.filters.push(filter);
return filter;
},
remove: function(filter) {
// remove `filter` from flist
for (var i in this.filters) {
if (this.filters[i] === filter) {
break;
}
}
if (i!==undefined) {
// remove index
this.filters.splice(i, 1);
}
},
filter_exists: function(doctype, fieldname, condition, value) {
var flag = false;
for(var i in this.filters) {
if(this.filters[i].field) {
var f = this.filters[i].get_value();
if(f[0]==doctype && f[1]==fieldname && f[2]==condition && f[3]==value) {
flag = true;
} else if($.isArray(value) && frappe.utils.arrays_equal(value, f[3])) {
flag = true;
}
}
}
return flag;
},
get_filters: function() {
// get filter values as dict
var values = [];
$.each(this.filters, function(i, filter) {
if(filter.field) {
filter.freeze();
values.push(filter.get_value());
}
});
this.base_list.update_standard_filters(values);
return values;
},
// remove hidden filters
update_filters: function() {
var fl = [];
$.each(this.filters, function(i, f) {
if(f.field) fl.push(f);
})
this.filters = fl;
if(this.filters.length === 0) {
this.wrapper.find('.clear-filters').addClass("hide");
}
},
get_filter: function(fieldname) {
for(var i in this.filters) {
if(this.filters[i].field && this.filters[i].field.df.fieldname==fieldname)
return this.filters[i];
}
},
get_formatted_value: function(field, val){
var value = val;
if(field.df.fieldname==="docstatus") {
value = {0:"Draft", 1:"Submitted", 2:"Cancelled"}[value] || value;
} else if(field.df.original_type==="Check") {
value = {0:"No", 1:"Yes"}[cint(value)];
}
value = frappe.format(value, field.df, {only_value: 1});
return value;
}
});
frappe.ui.Filter = Class.extend({
init: function(opts) {
$.extend(this, opts);
this.doctype = this.flist.doctype;
this.make();
this.make_select();
this.set_events();
},
make: function() {
this.wrapper = $(frappe.render_template("edit_filter", {}))
.appendTo(this.flist.wrapper.find('.filter_area'));
},
make_select: function() {
var me = this;
this.fieldselect = new frappe.ui.FieldSelect({
parent: this.wrapper.find('.fieldname_select_area'),
doctype: this.doctype,
filter_fields: this.filter_fields,
select: function(doctype, fieldname) {
me.set_field(doctype, fieldname);
}
});
if(this.fieldname) {
this.fieldselect.set_value(this._doctype || this.doctype, this.fieldname);
}
},
set_events: function() {
var me = this;
this.wrapper.find("a.remove-filter").on("click", function() {
me.remove();
});
this.wrapper.find(".set-filter-and-run").on("click", function() {
me.wrapper.removeClass("is-new-filter");
me.flist.base_list.run();
me.apply();
});
// add help for "in" codition
me.wrapper.find('.condition').change(function() {
if(!me.field) return;
var condition = $(this).val();
if(in_list(["in", "like", "not in", "not like"], condition)) {
me.set_field(me.field.df.parent, me.field.df.fieldname, 'Data', condition);
if(!me.field.desc_area) {
me.field.desc_area = $('<div class="text-muted small">').appendTo(me.field.wrapper);
}
// set description
me.field.desc_area.html((in_list(["in", "not in"], condition)==="in"
? __("values separated by commas")
: __("use % as wildcard"))+'</div>');
} else {
//if condition selected after refresh
me.set_field(me.field.df.parent, me.field.df.fieldname, null, condition);
}
});
// set the field
if(me.fieldname) {
// pre-sets given (could be via tags!)
return this.set_values(me._doctype, me.fieldname, me.condition, me.value);
} else {
me.set_field(me.doctype, 'name');
}
},
apply: function() {
var f = this.get_value();
this.flist.remove(this);
this.flist.push_new_filter(f[0], f[1], f[2], f[3]);
this.remove();
},
remove: function(dont_run) {
this.wrapper.remove();
this.$btn_group && this.$btn_group.remove();
this.field = null;
this.flist.update_filters();
if(!dont_run) {
this.flist.base_list.refresh(true);
}
},
set_values: function(doctype, fieldname, condition, value) {
// presents given (could be via tags!)
this.set_field(doctype, fieldname);
// change 0,1 to Yes, No for check field type
if(this.field.df.original_type==='Check') {
if(value==0) value = 'No';
else if(value==1) value = 'Yes';
}
if(condition) {
this.wrapper.find('.condition').val(condition).change();
}
if(value!=null) {
return this.field.set_value(value);
}
},
set_field: function(doctype, fieldname, fieldtype, condition) {
var me = this;
// set in fieldname (again)
var cur = me.field ? {
fieldname: me.field.df.fieldname,
fieldtype: me.field.df.fieldtype,
parent: me.field.df.parent,
} : {};
var original_docfield = me.fieldselect.fields_by_name[doctype][fieldname];
if(!original_docfield) {
frappe.msgprint(__("Field {0} is not selectable.", [fieldname]));
return;
}
var df = copy_dict(me.fieldselect.fields_by_name[doctype][fieldname]);
// filter field shouldn't be read only or hidden
df.read_only = 0;
df.hidden = 0;
if(!condition) this.set_default_condition(df, fieldtype);
this.set_fieldtype(df, fieldtype);
// called when condition is changed,
// don't change if all is well
if(me.field && cur.fieldname == fieldname && df.fieldtype == cur.fieldtype &&
df.parent == cur.parent) {
return;
}
// clear field area and make field
me.fieldselect.selected_doctype = doctype;
me.fieldselect.selected_fieldname = fieldname;
// save old text
var old_text = null;
if(me.field) {
old_text = me.field.get_value();
}
var field_area = me.wrapper.find('.filter_field').empty().get(0);
var f = frappe.ui.form.make_control({
df: df,
parent: field_area,
only_input: true,
})
f.refresh();
me.field = f;
if(old_text && me.field.df.fieldtype===cur.fieldtype) {
me.field.set_value(old_text);
}
// run on enter
$(me.field.wrapper).find(':input').keydown(function(ev) {
if(ev.which==13) {
me.flist.base_list.run();
}
})
},
set_fieldtype: function(df, fieldtype) {
// reset
if(df.original_type)
df.fieldtype = df.original_type;
else
df.original_type = df.fieldtype;
df.description = ''; df.reqd = 0;
df.ignore_link_validation = true;
// given
if(fieldtype) {
df.fieldtype = fieldtype;
return;
}
// scrub
if(df.fieldname=="docstatus") {
df.fieldtype="Select",
df.options=[
{value:0, label:__("Draft")},
{value:1, label:__("Submitted")},
{value:2, label:__("Cancelled")}
]
} else if(df.fieldtype=='Check') {
df.fieldtype='Select';
df.options='No\nYes';
} else if(['Text','Small Text','Text Editor','Code','Tag','Comments',
'Dynamic Link','Read Only','Assign'].indexOf(df.fieldtype)!=-1) {
df.fieldtype = 'Data';
} else if(df.fieldtype=='Link' && ['=', '!='].indexOf(this.wrapper.find('.condition').val())==-1) {
df.fieldtype = 'Data';
}
if(df.fieldtype==="Data" && (df.options || "").toLowerCase()==="email") {
df.options = null;
}
if(this.wrapper.find('.condition').val()== "Between" && (df.fieldtype == 'Date' || df.fieldtype == 'Datetime')){
df.fieldtype = 'DateRange';
}
},
set_default_condition: function(df, fieldtype) {
if(!fieldtype) {
// set as "like" for data fields
if (df.fieldtype == 'Data') {
this.wrapper.find('.condition').val('like');
} else if (df.fieldtype == 'Date' || df.fieldtype == 'Datetime'){
this.wrapper.find('.condition').val('Between');
}else{
this.wrapper.find('.condition').val('=');
}
}
},
get_value: function() {
return [this.fieldselect.selected_doctype,
this.field.df.fieldname, this.get_condition(), this.get_selected_value()];
},
get_selected_value: function() {
var val = this.field.get_value();
if(typeof val==='string') {
val = strip(val);
}
if(this.field.df.original_type == 'Check') {
val = (val=='Yes' ? 1 :0);
}
if(this.get_condition().indexOf('like', 'not like')!==-1) {
// automatically append wildcards
if(val) {
if(val.slice(0,1) !== "%") {
val = "%" + val;
}
if(val.slice(-1) !== "%") {
val = val + "%";
}
}
} else if(in_list(["in", "not in"], this.get_condition())) {
if(val) {
val = $.map(val.split(","), function(v) { return strip(v); });
}
} if(val === '%') {
val = "";
}
return val;
},
get_condition: function() {
return this.wrapper.find('.condition').val();
},
freeze: function() {
if(this.$btn_group) {
// already made, just hide the condition setter
this.set_filter_button_text();
this.wrapper.toggle(false);
return;
}
var me = this;
// add a button for new filter if missing
this.$btn_group = $(`<div class="btn-group">
<button class="btn btn-default btn-xs toggle-filter"
title="${ __("Edit Filter") }">
</button>
<button class="btn btn-default btn-xs remove-filter"
title="${ __("Remove Filter") }">
<i class="fa fa-remove text-muted"></i>
</button></div>`)
.insertAfter(this.flist.wrapper.find(".set-filters .new-filter"));
this.set_filter_button_text();
this.$btn_group.find(".remove-filter").on("click", function() {
me.remove();
});
this.$btn_group.find(".toggle-filter").on("click", function() {
$(this).closest('.show_filters').find('.filter_area').show();
me.wrapper.toggle();
})
this.wrapper.toggle(false);
},
set_filter_button_text: function() {
var value = this.get_selected_value();
value = this.flist.get_formatted_value(this.field, value);
// for translations
// __("like"), __("not like"), __("in")
this.$btn_group.find(".toggle-filter")
.html(repl('%(label)s %(condition)s "%(value)s"', {
label: __(this.field.df.label),
condition: __(this.get_condition()),
value: __(value),
}));
}
});
// <select> widget with all fields of a doctype as options
frappe.ui.FieldSelect = Class.extend({
// opts parent, doctype, filter_fields, with_blank, select
init: function(opts) {
var me = this;
$.extend(this, opts);
this.fields_by_name = {};
this.options = [];
this.$select = $('<input class="form-control">')
.appendTo(this.parent)
.on("click", function () { $(this).select(); });
this.select_input = this.$select.get(0);
this.awesomplete = new Awesomplete(this.select_input, {
minChars: 0,
maxItems: 99,
autoFirst: true,
list: me.options,
item: function(item, input) {
return $(repl('<li class="filter-field-select"><p>%(label)s</p></li>', item))
.data("item.autocomplete", item)
.get(0);
}
});
this.$select.on("awesomplete-select", function(e) {
var o = e.originalEvent;
var value = o.text.value;
var item = me.awesomplete.get_item(value);
me.selected_doctype = item.doctype;
me.selected_fieldname = item.fieldname;
if(me.select) me.select(item.doctype, item.fieldname);
});
this.$select.on("awesomplete-selectcomplete", function(e) {
var o = e.originalEvent;
var value = o.text.value;
var item = me.awesomplete.get_item(value);
me.$select.val(item.label);
});
if(this.filter_fields) {
for(var i in this.filter_fields)
this.add_field_option(this.filter_fields[i])
} else {
this.build_options();
}
this.set_value(this.doctype, "name");
window.last_filter = this;
},
get_value: function() {
return this.selected_doctype ? this.selected_doctype + "." + this.selected_fieldname : null;
},
val: function(value) {
if(value===undefined) {
return this.get_value();
} else {
this.set_value(value);
}
},
clear: function() {
this.selected_doctype = null;
this.selected_fieldname = null;
this.$select.val("");
},
set_value: function(doctype, fieldname) {
var me = this;
this.clear();
if(!doctype) return;
// old style
if(doctype.indexOf(".")!==-1) {
var parts = doctype.split(".");
doctype = parts[0];
fieldname = parts[1];
}
$.each(this.options, function(i, v) {
if(v.doctype===doctype && v.fieldname===fieldname) {
me.selected_doctype = doctype;
me.selected_fieldname = fieldname;
me.$select.val(v.label);
return false;
}
});
},
build_options: function() {
var me = this;
me.table_fields = [];
var std_filters = $.map(frappe.model.std_fields, function(d) {
var opts = {parent: me.doctype}
if(d.fieldname=="name") opts.options = me.doctype;
return $.extend(copy_dict(d), opts);
});
// add parenttype column
var doctype_obj = locals['DocType'][me.doctype];
if(doctype_obj && cint(doctype_obj.istable)) {
std_filters = std_filters.concat([{
fieldname: 'parent',
fieldtype: 'Data',
label: 'Parent',
parent: me.doctype,
}]);
}
// blank
if(this.with_blank) {
this.options.push({
label:"",
value:"",
})
}
// main table
var main_table_fields = std_filters.concat(frappe.meta.docfield_list[me.doctype]);
$.each(frappe.utils.sort(main_table_fields, "label", "string"), function(i, df) {
// show fields where user has read access and if report hide flag is not set
if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide)
me.add_field_option(df);
});
// child tables
$.each(me.table_fields, function(i, table_df) {
if(table_df.options) {
var child_table_fields = [].concat(frappe.meta.docfield_list[table_df.options]);
$.each(frappe.utils.sort(child_table_fields, "label", "string"), function(i, df) {
// show fields where user has read access and if report hide flag is not set
if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide)
me.add_field_option(df);
});
}
});
},
add_field_option: function(df) {
var me = this;
if(me.doctype && df.parent==me.doctype) {
var label = __(df.label);
var table = me.doctype;
if(df.fieldtype=='Table') me.table_fields.push(df);
} else {
var label = __(df.label) + ' (' + __(df.parent) + ')';
var table = df.parent;
}
if(frappe.model.no_value_type.indexOf(df.fieldtype) == -1 &&
!(me.fields_by_name[df.parent] && me.fields_by_name[df.parent][df.fieldname])) {
this.options.push({
label: label,
value: table + "." + df.fieldname,
fieldname: df.fieldname,
doctype: df.parent
});
if(!me.fields_by_name[df.parent]) me.fields_by_name[df.parent] = {};
me.fields_by_name[df.parent][df.fieldname] = df;
}
},
})

38
frappe/public/js/frappe/ui/sort_selector.html
View file

@ -1,19 +1,23 @@
<div class="sort-selector pull-right">
<button class="btn btn-default btn-xs pull-right btn-order"
data-value="{{ sort_order }}" style="margin-left: 10px;">
<span class="octicon text-muted octicon-arrow-{{
sort_order==="desc" ? "down" : "up" }}"></span></button>
<div class="dropdown pull-right">
<a class="text-muted dropdown-toggle small"
data-toggle="dropdown" aria-haspopup="true" aria-expanded="true">
<span class="dropdown-text">{{ __(sort_by_label) }}</span>
</a>
<ul class="dropdown-menu" style="max-height: 300px; overflow-y: scroll;">
{% for value in options %}
<li><a class="option" data-value="{{ value.fieldname }}">
{{ __(value.label) }}</a></li>
{% endfor %}
</ul>
<div class="sort-selector">
<div class="dropdown">
<a class="text-muted dropdown-toggle small"
data-toggle="dropdown" aria-haspopup="true" aria-expanded="true">
<span class="dropdown-text">{{ __(sort_by_label) }}</span>
</a>
<ul class="dropdown-menu">
{% for value in options %}
<li>
<a class="option" data-value="{{ value.fieldname }}">
{{ __(value.label) }}
</a>
</li>
{% endfor %}
</ul>
</div>
<button class="btn btn-default btn-xs btn-order"
data-value="{{ sort_order }}">
<span class="octicon text-muted
octicon-arrow-{{ sort_order===" desc " ? "down " : "up " }}">
</span>
</button>
</div>

53
frappe/public/less/filters.less Normal file
View file

@ -0,0 +1,53 @@
@import 'variables';
.active-tag-filters {
.add-filter, .filter-tag {
margin: 0 10px 10px 0;
}
}
.active-tag-filters .btn-group {
white-space: nowrap;
font-size: 0;
}
.active-tag-filters .btn-group .btn-default {
background-color: transparent;
border: 1px solid @border-color;
color: @text-muted;
float: none;
}
.filter-box {
border-bottom: 1px solid @border-color;
padding: 10px 15px 3px;
.remove-filter {
margin-top: 6px;
margin-left: 15px;
}
.filter-field {
padding-right: 15px;
width: calc(100% - 36px);
.frappe-control {
position: relative;
}
}
}
// for sm and above
@media (min-width: @screen-xs) {
.filter-box .row > div[class*="col-sm-"] {
padding-right: 0px;
}
.filter-field {
width: 65% !important;
.frappe-control {
position: relative;
}
}
}

91
frappe/public/less/list.less
View file

@ -33,83 +33,30 @@
}
}
.sort-selector {
.dropdown:hover {
text-decoration: underline;
}
}
.filter-list{
.filter-list {
position: relative;
.tag-filters-area {
padding: 10px 150px 0 10px;
border-bottom: 1px solid @border-color;
}
.sort-selector {
position: absolute;
top: 15px;
right: 15px;
}
}
.tag-filters-area {
padding: 15px 15px 0px;
border-bottom: 1px solid @border-color;
}
.active-tag-filters {
padding-bottom: 4px;
padding-right: 120px;
@media (max-width: @screen-xs) {
padding-right: 80px;
}
}
.active-tag-filters .btn {
margin-bottom: 10px;
}
.active-tag-filters .btn-group {
margin-left: 10px;
white-space: nowrap;
font-size: 0;
}
.active-tag-filters .btn-group .btn-default {
background-color: transparent;
border: 1px solid @border-color;
color: @text-muted;
float: none;
}
.filter-box {
border-bottom: 1px solid @border-color;
padding: 10px 15px 3px;
.remove-filter {
margin-top: 6px;
margin-left: 15px;
}
.filter-field {
padding-right: 15px;
width: calc(100% - 36px);
.frappe-control {
position: relative;
top: 0px;
right: 0px;
margin: 10px;
display: flex;
.dropdown:hover {
text-decoration: underline;
}
}
}
// for sm and above
@media (min-width: @screen-xs) {
.filter-box .row > div[class*="col-sm-"] {
padding-right: 0px;
}
.filter-field {
width: 65% !important;
.frappe-control {
position: relative;
.dropdown-menu {
max-height: 300px;
overflow-y: scroll;
right: 0;
left: auto;
}
button {
margin-left: 10px;
}
}
}

2
frappe/utils/xlsxutils.py
View file

@ -26,7 +26,7 @@ def make_xlsx(data, sheet_name, wb=None):
for row in data:
clean_row = []
for item in row:
if isinstance(item, string_types) and sheet_name != "Data Import Template":
if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']):
value = handle_html(item)
else:
value = item