From 436d7a0d4eb8ec3f0318e331cc55fc046736b148 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 22 Mar 2018 11:30:48 +0530 Subject: [PATCH] Data export tool (#5213) * Data export tool New doctype to export data Child table export option Ability to add filter * Filter view refactor Separated filter css * Moved exporter script from data import doctype to data export doctype * some input validation * removed unwanted file and some style fixes * removed console log * hide sidebar * renamed export method * added data export link in setup module page * minor fix * refactor exporter.py * data export ui tweaks * codacy and bug fix * silly code fixes * [minor] indentation fix --- frappe/config/setup.py | 11 +- frappe/core/doctype/data_export/__init__.py | 0 .../core/doctype/data_export/data_export.js | 138 ++++ .../core/doctype/data_export/data_export.json | 250 +++++++ .../core/doctype/data_export/data_export.py | 9 + frappe/core/doctype/data_export/exporter.py | 340 +++++++++ .../core/doctype/data_import/data_import.js | 33 +- .../core/doctype/data_import/data_import.py | 4 +- frappe/core/doctype/data_import/exporter.py | 309 -------- .../doctype/data_import/test_data_import.py | 22 +- frappe/public/build.json | 3 +- frappe/public/js/frappe/ui/filters/filter.js | 1 + frappe/public/js/frappe/ui/filters/filters.js | 678 ------------------ frappe/public/js/frappe/ui/sort_selector.html | 38 +- frappe/public/less/filters.less | 53 ++ frappe/public/less/list.less | 91 +-- frappe/utils/xlsxutils.py | 2 +- 17 files changed, 872 insertions(+), 1110 deletions(-) create mode 100644 frappe/core/doctype/data_export/__init__.py create mode 100644 frappe/core/doctype/data_export/data_export.js create mode 100644 frappe/core/doctype/data_export/data_export.json create mode 100644 frappe/core/doctype/data_export/data_export.py create mode 100644 frappe/core/doctype/data_export/exporter.py delete mode 100644 frappe/core/doctype/data_import/exporter.py delete mode 100644 frappe/public/js/frappe/ui/filters/filters.js create mode 100644 frappe/public/less/filters.less diff --git a/frappe/config/setup.py b/frappe/config/setup.py index 9c6bc92904..a47bf25ccf 100644 --- a/frappe/config/setup.py +++ b/frappe/config/setup.py @@ -95,9 +95,16 @@ def get_data(): { "type": "doctype", "name": "Data Import", - "label": _("Import / Export Data"), + "label": _("Import Data"), "icon": "octicon octicon-cloud-upload", - "description": _("Import / Export Data from CSV and Excel files.") + "description": _("Import Data from CSV / Excel files.") + }, + { + "type": "doctype", + "name": "Data Export", + "label": _("Export Data"), + "icon": "octicon octicon-cloud-upload", + "description": _("Export Data in CSV / Excel format.") }, { "type": "doctype", diff --git a/frappe/core/doctype/data_export/__init__.py b/frappe/core/doctype/data_export/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/data_export/data_export.js b/frappe/core/doctype/data_export/data_export.js new file mode 100644 index 0000000000..4cbaa81f4e --- /dev/null +++ b/frappe/core/doctype/data_export/data_export.js @@ -0,0 +1,138 @@ +// Copyright (c) 2018, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Export', { + refresh: frm => { + frm.disable_save(); + frm.page.set_primary_action('Export', () => { + can_export(frm) ? export_data(frm) : null; + }); + $(frm.footer.wrapper).toggle(false); + }, + onload: (frm) => { + frm.set_query("reference_doctype", () => { + return { + "filters": { + "issingle": 0, + "istable": 0, + "name": ['in', frappe.boot.user.can_export] + } + }; + }); + }, + reference_doctype: frm => { + const doctype = frm.doc.reference_doctype; + if (doctype) { + frappe.model.with_doctype(doctype, () => set_field_options(frm)); + } else { + reset_filter_and_field(frm); + } + } +}); + +const can_export = frm => { + const doctype = frm.doc.reference_doctype; + const parent_multicheck_options = frm.fields_multicheck[doctype] ? + frm.fields_multicheck[doctype].get_checked_options() : []; + let is_valid_form = false; + if (!doctype) { + frappe.msgprint(__('Please select the Document Type.')); + } else if (!parent_multicheck_options.length) { + frappe.msgprint(__('Atleast one field of Parent Document Type is mandatory')); + } else { + is_valid_form = true; + } + return is_valid_form; +}; + +const export_data = frm => { + let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; + var export_params = () => { + let columns = {}; + Object.keys(frm.fields_multicheck).forEach(dt => { + const options = frm.fields_multicheck[dt].get_checked_options(); + columns[dt] = options; + }); + return { + doctype: frm.doc.reference_doctype, + select_columns: JSON.stringify(columns), + filters: frm.filter_list.get_filters().map(filter => filter.slice(1, 4)), + file_type: frm.doc.file_type, + template: true, + with_data: true + }; + }; + + open_url_post(get_template_url, export_params()); +}; + +const reset_filter_and_field = (frm) => { + const parent_wrapper = frm.fields_dict.fields_multicheck.$wrapper; + const filter_wrapper = frm.fields_dict.filter_list.$wrapper; + parent_wrapper.empty(); + filter_wrapper.empty(); + frm.filter_list = []; + frm.fields_multicheck = {}; +}; + +const set_field_options = (frm) => { + const parent_wrapper = frm.fields_dict.fields_multicheck.$wrapper; + const filter_wrapper = frm.fields_dict.filter_list.$wrapper; + const doctype = frm.doc.reference_doctype; + const related_doctypes = get_doctypes(doctype); + + parent_wrapper.empty(); + filter_wrapper.empty(); + + frm.filter_list = new frappe.ui.FilterGroup({ + parent: filter_wrapper, + doctype: doctype, + on_change: () => { }, + }); + + frm.fields_multicheck = {}; + related_doctypes.forEach(dt => { + frm.fields_multicheck[dt] = add_doctype_field_multicheck_control(dt, parent_wrapper); + }); + + frm.refresh(); +}; + +const get_doctypes = parentdt => { + return [parentdt].concat( + frappe.meta.get_table_fields(parentdt).map(df => df.options) + ); +}; + +const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => { + const fields = get_fields(doctype); + + const options = fields + .map(df => { + return { + label: df.label + (df.reqd ? ' (M)' : ''), + value: df.fieldname, + checked: 1 + }; + }); + + const multicheck_control = frappe.ui.form.make_control({ + parent: parent_wrapper, + df: { + "label": doctype, + "fieldname": doctype + '_fields', + "fieldtype": "MultiCheck", + "options": options, + "select_all": options.length > 5, + "columns": 3, + "hidden": 1, + }, + render_input: true + }); + + multicheck_control.refresh_input(); + return multicheck_control; +}; + +const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; +const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); \ No newline at end of file diff --git a/frappe/core/doctype/data_export/data_export.json b/frappe/core/doctype/data_export/data_export.json new file mode 100644 index 0000000000..f106a0ce02 --- /dev/null +++ b/frappe/core/doctype/data_export/data_export.json @@ -0,0 +1,250 @@ +{ + "allow_copy": 0, + "allow_guest_to_view": 0, + "allow_import": 0, + "allow_rename": 0, + "beta": 0, + "creation": "2018-03-07 10:09:49.794764", + "custom": 0, + "docstatus": 0, + "doctype": "DocType", + "document_type": "", + "editable_grid": 1, + "engine": "InnoDB", + "fields": [ + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "reference_doctype", + "fieldtype": "Link", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Select Doctype", + "length": 0, + "no_copy": 0, + "options": "DocType", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_2", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "CSV", + "fieldname": "file_type", + "fieldtype": "Select", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 1, + "in_standard_filter": 0, + "label": "File Type", + "length": 0, + "no_copy": 0, + "options": "Excel\nCSV", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "reference_doctype", + "fieldname": "section_break", + "fieldtype": "Section Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "filter_list", + "fieldtype": "HTML", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Filter List", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "fields_multicheck", + "fieldtype": "HTML", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Fields Multicheck", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + } + ], + "has_web_view": 0, + "hide_heading": 0, + "hide_toolbar": 1, + "idx": 0, + "image_view": 0, + "in_create": 0, + "is_submittable": 0, + "issingle": 1, + "istable": 0, + "max_attachments": 0, + "modified": "2018-03-21 13:23:05.623052", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Export", + "name_case": "", + "owner": "Administrator", + "permissions": [ + { + "amend": 0, + "apply_user_permissions": 0, + "cancel": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 0, + "if_owner": 0, + "import": 0, + "permlevel": 0, + "print": 1, + "read": 1, + "report": 0, + "role": "System Manager", + "set_user_permissions": 0, + "share": 1, + "submit": 0, + "write": 1 + } + ], + "quick_entry": 1, + "read_only": 0, + "read_only_onload": 0, + "show_name_in_global_search": 0, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 0, + "track_seen": 0 +} \ No newline at end of file diff --git a/frappe/core/doctype/data_export/data_export.py b/frappe/core/doctype/data_export/data_export.py new file mode 100644 index 0000000000..fb4fae26d5 --- /dev/null +++ b/frappe/core/doctype/data_export/data_export.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +from frappe.model.document import Document + +class DataExport(Document): + pass diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py new file mode 100644 index 0000000000..98b657d71c --- /dev/null +++ b/frappe/core/doctype/data_export/exporter.py @@ -0,0 +1,340 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals + +import frappe +from frappe import _ +import frappe.permissions +import re, csv, os +from frappe.utils.csvutils import UnicodeWriter +from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint +from frappe.core.doctype.data_import.importer import get_data_keys +from six import string_types + +reflags = { + "I":re.I, + "L":re.L, + "M":re.M, + "U":re.U, + "S":re.S, + "X":re.X, + "D": re.DEBUG +} + +@frappe.whitelist() +def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, + select_columns=None, file_type='CSV', template=False, filters=None): + exporter = DataExporter(doctype=doctype, parent_doctype=parent_doctype, all_doctypes=all_doctypes, with_data=with_data, + select_columns=select_columns, file_type=file_type, template=template, filters=filters) + exporter.build_response() + +class DataExporter(): + def __init__(self, doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, + select_columns=None, file_type='CSV', template=False, filters=None): + self.doctype = doctype + self.parent_doctype = parent_doctype + self.all_doctypes = all_doctypes + self.with_data = cint(with_data) + self.select_columns = select_columns + self.file_type = file_type + self.template = template + self.filters = filters + self.data_keys = get_data_keys() + + self.prepare_args() + + def prepare_args(self): + if self.select_columns: + self.select_columns = parse_json(self.select_columns) + if self.filters: + self.filters = parse_json(self.filters) + + self.docs_to_export = {} + if self.doctype: + if isinstance(self.doctype, string_types): + self.doctype = [self.doctype] + + if len(self.doctype) > 1: + self.docs_to_export = self.doctype[1] + self.doctype = self.doctype[0] + + if not self.parent_doctype: + self.parent_doctype = self.doctype + + self.column_start_end = {} + + if self.all_doctypes: + self.child_doctypes = [] + for df in frappe.get_meta(self.doctype).get_table_fields(): + self.child_doctypes.append(dict(doctype=df.options, parentfield=df.fieldname)) + + def build_response(self): + self.writer = UnicodeWriter() + self.name_field = 'parent' if self.parent_doctype != self.doctype else 'name' + + if self.template: + self.add_main_header() + + self.writer.writerow(['']) + self.tablerow = [self.data_keys.doctype, ""] + self.labelrow = [_("Column Labels:"), "ID"] + self.fieldrow = [self.data_keys.columns, self.name_field] + self.mandatoryrow = [_("Mandatory:"), _("Yes")] + self.typerow = [_('Type:'), 'Data (text)'] + self.inforow = [_('Info:'), ''] + self.columns = [self.name_field] + + self.build_field_columns(self.doctype) + + if self.all_doctypes: + for d in self.child_doctypes: + self.append_empty_field_column() + if (self.select_columns and self.select_columns.get(d['doctype'], None)) or not self.select_columns: + # if atleast one column is selected for this doctype + self.build_field_columns(d['doctype'], d['parentfield']) + + self.add_field_headings() + self.add_data() + if self.with_data and not self.data: + frappe.respond_as_web_page(_('No Data'), _('There is no data to be exported'), indicator_color='orange') + return + + if self.file_type == 'Excel': + return self.build_response_as_excel() + else: + # write out response as a type csv + frappe.response['result'] = cstr(self.writer.getvalue()) + frappe.response['type'] = 'csv' + frappe.response['doctype'] = self.doctype + + def add_main_header(self): + self.writer.writerow([_('Data Import Template')]) + self.writer.writerow([self.data_keys.main_table, self.doctype]) + + if self.parent_doctype != self.doctype: + self.writer.writerow([self.data_keys.parent_table, self.parent_doctype]) + else: + self.writer.writerow(['']) + + self.writer.writerow(['']) + self.writer.writerow([_('Notes:')]) + self.writer.writerow([_('Please do not change the template headings.')]) + self.writer.writerow([_('First data column must be blank.')]) + self.writer.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')]) + self.writer.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')]) + self.writer.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')]) + self.writer.writerow([_('For updating, you can update only selective columns.')]) + self.writer.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')]) + if self.name_field == "parent": + self.writer.writerow([_('"Parent" signifies the parent table in which this row must be added')]) + self.writer.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')]) + + def build_field_columns(self, dt, parentfield=None): + meta = frappe.get_meta(dt) + + # build list of valid docfields + tablecolumns = [] + for f in frappe.db.sql('desc `tab%s`' % dt): + field = meta.get_field(f[0]) + if field and ((self.select_columns and f[0] in self.select_columns[dt]) or not self.select_columns): + tablecolumns.append(field) + + tablecolumns.sort(key = lambda a: int(a.idx)) + + _column_start_end = frappe._dict(start=0) + + if dt==self.doctype: + _column_start_end = frappe._dict(start=0) + else: + _column_start_end = frappe._dict(start=len(self.columns)) + + self.append_field_column(frappe._dict({ + "fieldname": "name", + "parent": dt, + "label": "ID", + "fieldtype": "Data", + "reqd": 1, + "idx": 0, + "info": _("Leave blank for new records") + }), True) + + for docfield in tablecolumns: + self.append_field_column(docfield, True) + + # all non mandatory fields + for docfield in tablecolumns: + self.append_field_column(docfield, False) + + # if there is one column, add a blank column (?) + if len(self.columns)-_column_start_end.start == 1: + self.append_empty_field_column() + + # append DocType name + self.tablerow[_column_start_end.start + 1] = dt + + if parentfield: + self.tablerow[_column_start_end.start + 2] = parentfield + + _column_start_end.end = len(self.columns) + 1 + + self.column_start_end[(dt, parentfield)] = _column_start_end + + def append_field_column(self, docfield, for_mandatory): + if not docfield: + return + if for_mandatory and not docfield.reqd: + return + if not for_mandatory and docfield.reqd: + return + if docfield.fieldname in ('parenttype', 'trash_reason'): + return + if docfield.hidden: + return + if self.select_columns and docfield.fieldname not in self.select_columns.get(docfield.parent, []): + return + + self.tablerow.append("") + self.fieldrow.append(docfield.fieldname) + self.labelrow.append(_(docfield.label)) + self.mandatoryrow.append(docfield.reqd and 'Yes' or 'No') + self.typerow.append(docfield.fieldtype) + self.inforow.append(self.getinforow(docfield)) + self.columns.append(docfield.fieldname) + + def append_empty_field_column(self): + self.tablerow.append("~") + self.fieldrow.append("~") + self.labelrow.append("") + self.mandatoryrow.append("") + self.typerow.append("") + self.inforow.append("") + self.columns.append("") + + @staticmethod + def getinforow(docfield): + """make info comment for options, links etc.""" + if docfield.fieldtype == 'Select': + if not docfield.options: + return '' + else: + return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n'))) + elif docfield.fieldtype == 'Link': + return 'Valid %s' % docfield.options + elif docfield.fieldtype == 'Int': + return 'Integer' + elif docfield.fieldtype == "Check": + return "0 or 1" + elif docfield.fieldtype in ["Date", "Datetime"]: + return cstr(frappe.defaults.get_defaults().date_format) + elif hasattr(docfield, "info"): + return docfield.info + else: + return '' + + def add_field_headings(self): + self.writer.writerow(self.tablerow) + self.writer.writerow(self.labelrow) + self.writer.writerow(self.fieldrow) + self.writer.writerow(self.mandatoryrow) + self.writer.writerow(self.typerow) + self.writer.writerow(self.inforow) + if self.template: + self.writer.writerow([self.data_keys.data_separator]) + + def add_data(self): + if self.template and not self.with_data: + return + + frappe.permissions.can_export(self.parent_doctype, raise_exception=True) + + # sort nested set doctypes by `lft asc` + order_by = None + table_columns = frappe.db.get_table_columns(self.parent_doctype) + if 'lft' in table_columns and 'rgt' in table_columns: + order_by = '`tab{doctype}`.`lft` asc'.format(doctype=self.parent_doctype) + # get permitted data only + self.data = frappe.get_list(self.doctype, fields=["*"], filters=self.filters, limit_page_length=None, order_by=order_by) + + for doc in self.data: + op = self.docs_to_export.get("op") + names = self.docs_to_export.get("name") + + if names and op: + if op == '=' and doc.name not in names: + continue + elif op == '!=' and doc.name in names: + continue + elif names: + try: + sflags = self.docs_to_export.get("flags", "I,U").upper() + flags = 0 + for a in re.split('\W+',sflags): + flags = flags | reflags.get(a,0) + + c = re.compile(names, flags) + m = c.match(doc.name) + if not m: + continue + except Exception: + if doc.name not in names: + continue + # add main table + rows = [] + + self.add_data_row(rows, self.doctype, None, doc, 0) + + if self.all_doctypes: + # add child tables + for c in self.child_doctypes: + for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}` + where parent=%s and parentfield=%s order by idx""".format(c['doctype']), + (doc.name, c['parentfield']), as_dict=1)): + self.add_data_row(rows, c['doctype'], c['parentfield'], child, ci) + + for row in rows: + self.writer.writerow(row) + + def add_data_row(self, rows, dt, parentfield, doc, rowidx): + d = doc.copy() + meta = frappe.get_meta(dt) + if self.all_doctypes: + d.name = '"'+ d.name+'"' + + if len(rows) < rowidx + 1: + rows.append([""] * (len(self.columns) + 1)) + row = rows[rowidx] + + _column_start_end = self.column_start_end.get((dt, parentfield)) + + if _column_start_end: + for i, c in enumerate(self.columns[_column_start_end.start:_column_start_end.end]): + df = meta.get_field(c) + fieldtype = df.fieldtype if df else "Data" + value = d.get(c, "") + if value: + if fieldtype == "Date": + value = formatdate(value) + elif fieldtype == "Datetime": + value = format_datetime(value) + + row[_column_start_end.start + i + 1] = value + + def build_response_as_excel(self): + filename = frappe.generate_hash("", 10) + with open(filename, 'wb') as f: + f.write(cstr(self.writer.getvalue()).encode("utf-8")) + f = open(filename) + reader = csv.reader(f) + + from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else 'Data Export') + + f.close() + os.remove(filename) + + # write out response as a xlsx type + frappe.response['filename'] = self.doctype + '.xlsx' + frappe.response['filecontent'] = xlsx_file.getvalue() + frappe.response['type'] = 'binary' + diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 02b5adc54e..e03be837ff 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -22,7 +22,7 @@ frappe.ui.form.on('Data Import', { let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar"); if (progress_bar) { $(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped"); - $(progress_bar).css("width", data.progress+"%"); + $(progress_bar).css("width", data.progress + "%"); } } } @@ -38,10 +38,10 @@ frappe.ui.form.on('Data Import', { if (frm.doc.import_status) { const listview_settings = frappe.listview_settings['Data Import']; const indicator = listview_settings.get_indicator(frm.doc); - + frm.page.set_indicator(indicator[0], indicator[1]); - if (frm.doc.import_status==="In Progress") { + if (frm.doc.import_status === "In Progress") { frm.dashboard.add_progress("Data Import Progress", "0"); frm.set_read_only(); frm.refresh_fields(); @@ -57,14 +57,14 @@ frappe.ui.form.on('Data Import', { frappe.help.show_video("6wiriRKPhmg"); }); - if(frm.doc.reference_doctype && frm.doc.docstatus === 0) { + if (frm.doc.reference_doctype && frm.doc.docstatus === 0) { frm.add_custom_button(__("Download template"), function() { frappe.data_import.download_dialog(frm).show(); }); } if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows && - frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status=="Failed")) { + frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) { frm.page.set_primary_action(__("Start Import"), function() { frappe.call({ method: "frappe.core.doctype.data_import.data_import.import_data", @@ -88,10 +88,6 @@ frappe.ui.form.on('Data Import', { } }, - // import_file: function(frm) { - // frm.save(); - // }, - overwrite: function(frm) { if (frm.doc.overwrite === 0) { frm.doc.only_update = 0; @@ -126,8 +122,11 @@ frappe.ui.form.on('Data Import', { create_log_table: function(frm) { let msg = JSON.parse(frm.doc.log_details); var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty(); - $(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors, - import_status: frm.doc.import_status})).appendTo($log_wrapper); + $(frappe.render_template("log_details", { + data: msg.messages, + import_status: frm.doc.import_status, + show_only_errors: frm.doc.show_only_errors, + })).appendTo($log_wrapper); } }); @@ -146,7 +145,7 @@ frappe.data_import.download_dialog = function(frm) { const get_doctype_checkbox_fields = () => { return dialog.fields.filter(df => df.fieldname.endsWith('_fields')) .map(df => dialog.fields_dict[df.fieldname]); - } + }; const doctype_fields = get_fields(frm.doc.reference_doctype) .map(df => ({ @@ -245,13 +244,13 @@ frappe.data_import.download_dialog = function(frm) { doctype: frm.doc.reference_doctype, parent_doctype: frm.doc.reference_doctype, select_columns: JSON.stringify(columns), - with_data: data.with_data ? 'Yes' : 'No', - all_doctypes: 'Yes', - from_data_import: 'Yes', - excel_format: data.file_type === 'Excel' ? 'Yes' : 'No' + with_data: data.with_data, + all_doctypes: true, + file_type: data.file_type, + template: true }; }; - let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template'; + let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; open_url_post(get_template_url, export_params()); } else { frappe.msgprint(__("Please select the Document Type.")); diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 96429a415b..687db5e358 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -99,9 +99,9 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None, order_b def export_csv(doctype, path): - from frappe.core.doctype.data_import.exporter import get_template + from frappe.core.doctype.data_export.exporter import export_data with open(path, "wb") as csvfile: - get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes") + export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True) csvfile.write(frappe.response.result.encode("utf-8")) diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py deleted file mode 100644 index 9a8ca42b24..0000000000 --- a/frappe/core/doctype/data_import/exporter.py +++ /dev/null @@ -1,309 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals - -import frappe, json -from frappe import _ -import frappe.permissions -import re, csv, os -from frappe.utils.csvutils import UnicodeWriter -from frappe.utils import cstr, formatdate, format_datetime -from frappe.core.doctype.data_import.importer import get_data_keys -from six import string_types - -reflags = { - "I":re.I, - "L":re.L, - "M":re.M, - "U":re.U, - "S":re.S, - "X":re.X, - "D": re.DEBUG -} - -@frappe.whitelist() -def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None, - from_data_import="No", excel_format="No"): - all_doctypes = all_doctypes=="Yes" - if select_columns: - select_columns = json.loads(select_columns); - docs_to_export = {} - if doctype: - if isinstance(doctype, string_types): - doctype = [doctype]; - if len(doctype) > 1: - docs_to_export = doctype[1] - doctype = doctype[0] - - if not parent_doctype: - parent_doctype = doctype - - column_start_end = {} - - if all_doctypes: - child_doctypes = [] - for df in frappe.get_meta(doctype).get_table_fields(): - child_doctypes.append(dict(doctype=df.options, parentfield=df.fieldname)) - - def get_data_keys_definition(): - return get_data_keys() - - def add_main_header(): - w.writerow([_('Data Import Template')]) - w.writerow([get_data_keys_definition().main_table, doctype]) - - if parent_doctype != doctype: - w.writerow([get_data_keys_definition().parent_table, parent_doctype]) - else: - w.writerow(['']) - - w.writerow(['']) - w.writerow([_('Notes:')]) - w.writerow([_('Please do not change the template headings.')]) - w.writerow([_('First data column must be blank.')]) - w.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')]) - w.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')]) - w.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')]) - w.writerow([_('For updating, you can update only selective columns.')]) - w.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')]) - if key == "parent": - w.writerow([_('"Parent" signifies the parent table in which this row must be added')]) - w.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')]) - - def build_field_columns(dt, parentfield=None): - meta = frappe.get_meta(dt) - - # build list of valid docfields - tablecolumns = [] - for f in frappe.db.sql('desc `tab%s`' % dt): - field = meta.get_field(f[0]) - if field and ((select_columns and f[0] in select_columns[dt]) or not select_columns): - tablecolumns.append(field) - - tablecolumns.sort(key = lambda a: int(a.idx)) - - _column_start_end = frappe._dict(start=0) - - if dt==doctype: - _column_start_end = frappe._dict(start=0) - else: - _column_start_end = frappe._dict(start=len(columns)) - - append_field_column(frappe._dict({ - "fieldname": "name", - "parent": dt, - "label": "ID", - "fieldtype": "Data", - "reqd": 1, - "idx": 0, - "info": _("Leave blank for new records") - }), True) - - for docfield in tablecolumns: - append_field_column(docfield, True) - - # all non mandatory fields - for docfield in tablecolumns: - append_field_column(docfield, False) - - # if there is one column, add a blank column (?) - if len(columns)-_column_start_end.start == 1: - append_empty_field_column() - - # append DocType name - tablerow[_column_start_end.start + 1] = dt - - if parentfield: - tablerow[_column_start_end.start + 2] = parentfield - - _column_start_end.end = len(columns) + 1 - - column_start_end[(dt, parentfield)] = _column_start_end - - def append_field_column(docfield, for_mandatory): - if not docfield: - return - if for_mandatory and not docfield.reqd: - return - if not for_mandatory and docfield.reqd: - return - if docfield.fieldname in ('parenttype', 'trash_reason'): - return - if docfield.hidden: - return - if select_columns and docfield.fieldname not in select_columns.get(docfield.parent, []): - return - - tablerow.append("") - fieldrow.append(docfield.fieldname) - labelrow.append(_(docfield.label)) - mandatoryrow.append(docfield.reqd and 'Yes' or 'No') - typerow.append(docfield.fieldtype) - inforow.append(getinforow(docfield)) - columns.append(docfield.fieldname) - - def append_empty_field_column(): - tablerow.append("~") - fieldrow.append("~") - labelrow.append("") - mandatoryrow.append("") - typerow.append("") - inforow.append("") - columns.append("") - - def getinforow(docfield): - """make info comment for options, links etc.""" - if docfield.fieldtype == 'Select': - if not docfield.options: - return '' - else: - return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n'))) - elif docfield.fieldtype == 'Link': - return 'Valid %s' % docfield.options - elif docfield.fieldtype == 'Int': - return 'Integer' - elif docfield.fieldtype == "Check": - return "0 or 1" - elif docfield.fieldtype in ["Date", "Datetime"]: - return cstr(frappe.defaults.get_defaults().date_format) - elif hasattr(docfield, "info"): - return docfield.info - else: - return '' - - def add_field_headings(): - w.writerow(tablerow) - w.writerow(labelrow) - w.writerow(fieldrow) - w.writerow(mandatoryrow) - w.writerow(typerow) - w.writerow(inforow) - w.writerow([get_data_keys_definition().data_separator]) - - def add_data(): - def add_data_row(row_group, dt, parentfield, doc, rowidx): - d = doc.copy() - meta = frappe.get_meta(dt) - if all_doctypes: - d.name = '"'+ d.name+'"' - - if len(row_group) < rowidx + 1: - row_group.append([""] * (len(columns) + 1)) - row = row_group[rowidx] - - _column_start_end = column_start_end.get((dt, parentfield)) - - if _column_start_end: - for i, c in enumerate(columns[_column_start_end.start:_column_start_end.end]): - df = meta.get_field(c) - fieldtype = df.fieldtype if df else "Data" - value = d.get(c, "") - if value: - if fieldtype == "Date": - value = formatdate(value) - elif fieldtype == "Datetime": - value = format_datetime(value) - - row[_column_start_end.start + i + 1] = value - - if with_data=='Yes': - frappe.permissions.can_export(parent_doctype, raise_exception=True) - - # sort nested set doctypes by `lft asc` - order_by = None - table_columns = frappe.db.get_table_columns(parent_doctype) - if 'lft' in table_columns and 'rgt' in table_columns: - order_by = '`tab{doctype}`.`lft` asc'.format(doctype=parent_doctype) - - # get permitted data only - data = frappe.get_list(doctype, fields=["*"], limit_page_length=None, order_by=order_by) - - for doc in data: - op = docs_to_export.get("op") - names = docs_to_export.get("name") - - if names and op: - if op == '=' and doc.name not in names: - continue - elif op == '!=' and doc.name in names: - continue - elif names: - try: - sflags = docs_to_export.get("flags", "I,U").upper() - flags = 0 - for a in re.split('\W+',sflags): - flags = flags | reflags.get(a,0) - - c = re.compile(names, flags) - m = c.match(doc.name) - if not m: - continue - except: - if doc.name not in names: - continue - # add main table - row_group = [] - - add_data_row(row_group, doctype, None, doc, 0) - - if all_doctypes: - # add child tables - for c in child_doctypes: - for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}` - where parent=%s and parentfield=%s order by idx""".format(c['doctype']), - (doc.name, c['parentfield']), as_dict=1)): - add_data_row(row_group, c['doctype'], c['parentfield'], child, ci) - - for row in row_group: - w.writerow(row) - - w = UnicodeWriter() - key = 'parent' if parent_doctype != doctype else 'name' - - add_main_header() - - w.writerow(['']) - tablerow = [get_data_keys_definition().doctype, ""] - labelrow = [_("Column Labels:"), "ID"] - fieldrow = [get_data_keys_definition().columns, key] - mandatoryrow = [_("Mandatory:"), _("Yes")] - typerow = [_('Type:'), 'Data (text)'] - inforow = [_('Info:'), ''] - columns = [key] - - build_field_columns(doctype) - - if all_doctypes: - for d in child_doctypes: - append_empty_field_column() - if (select_columns and select_columns.get(d['doctype'], None)) or not select_columns: - # if atleast one column is selected for this doctype - build_field_columns(d['doctype'], d['parentfield']) - - add_field_headings() - add_data() - - if from_data_import == "Yes" and excel_format == "Yes": - filename = frappe.generate_hash("", 10) - with open(filename, 'wb') as f: - f.write(cstr(w.getvalue()).encode("utf-8")) - f = open(filename) - reader = csv.reader(f) - - from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(reader, "Data Import Template") - - f.close() - os.remove(filename) - - # write out response as a xlsx type - frappe.response['filename'] = doctype + '.xlsx' - frappe.response['filecontent'] = xlsx_file.getvalue() - frappe.response['type'] = 'binary' - - else: - # write out response as a type csv - frappe.response['result'] = cstr(w.getvalue()) - frappe.response['type'] = 'csv' - frappe.response['doctype'] = doctype diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py index 413d4edcfd..d181d49457 100644 --- a/frappe/core/doctype/data_import/test_data_import.py +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -4,24 +4,24 @@ from __future__ import unicode_literals import frappe, unittest -from frappe.core.doctype.data_import import exporter +from frappe.core.doctype.data_export import exporter from frappe.core.doctype.data_import import importer from frappe.utils.csvutils import read_csv_content class TestDataImport(unittest.TestCase): def test_export(self): - exporter.get_template("User", all_doctypes="No", with_data="No") + exporter.export_data("User", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) self.assertTrue(content[1][1], "User") def test_export_with_data(self): - exporter.get_template("User", all_doctypes="No", with_data="Yes") + exporter.export_data("User", all_doctypes=True, template=True, with_data=True) content = read_csv_content(frappe.response.result) self.assertTrue(content[1][1], "User") - self.assertTrue("Administrator" in [c[1] for c in content if len(c)>1]) + self.assertTrue('"Administrator"' in [c[1] for c in content if len(c)>1]) def test_export_with_all_doctypes(self): - exporter.get_template("User", all_doctypes="Yes", with_data="Yes") + exporter.export_data("User", all_doctypes="Yes", template=True, with_data=True) content = read_csv_content(frappe.response.result) self.assertTrue(content[1][1], "User") self.assertTrue('"Administrator"' in [c[1] for c in content if len(c)>1]) @@ -33,14 +33,14 @@ class TestDataImport(unittest.TestCase): if frappe.db.exists("Blog Category", "test-category"): frappe.delete_doc("Blog Category", "test-category") - exporter.get_template("Blog Category", all_doctypes="No", with_data="No") + exporter.export_data("Blog Category", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "", "test-category", "Test Cateogry"]) importer.upload(content) self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "Test Category") # export with data - exporter.get_template("Blog Category", all_doctypes="No", with_data="Yes") + exporter.export_data("Blog Category", all_doctypes=True, template=True, with_data=True) content = read_csv_content(frappe.response.result) # overwrite @@ -55,7 +55,7 @@ class TestDataImport(unittest.TestCase): frappe.get_doc({"doctype": "User", "email": user_email, "first_name": "Test Import UserRole"}).insert() - exporter.get_template("Has Role", "User", all_doctypes="No", with_data="No") + exporter.export_data("Has Role", "User", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "test_import_userrole@example.com", "Blogger"]) importer.upload(content) @@ -65,7 +65,7 @@ class TestDataImport(unittest.TestCase): self.assertTrue(user.get("roles")[0].role, "Blogger") # overwrite - exporter.get_template("Has Role", "User", all_doctypes="No", with_data="No") + exporter.export_data("Has Role", "User", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "test_import_userrole@example.com", "Website Manager"]) importer.upload(content, overwrite=True) @@ -77,7 +77,7 @@ class TestDataImport(unittest.TestCase): def test_import_with_children(self): #pylint: disable=R0201 if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") - exporter.get_template("Event", all_doctypes="Yes", with_data="No") + exporter.export_data("Event", all_doctypes="Yes", template=True) content = read_csv_content(frappe.response.result) content.append([None] * len(content[-2])) @@ -93,7 +93,7 @@ class TestDataImport(unittest.TestCase): if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") - exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes") + exporter.export_data("Event", all_doctypes=True, template=True, file_type="Excel") from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent) content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"]) diff --git a/frappe/public/build.json b/frappe/public/build.json index 04e200dbee..163f08548b 100755 --- a/frappe/public/build.json +++ b/frappe/public/build.json @@ -97,7 +97,8 @@ "public/less/mobile.less", "public/less/kanban.less", "public/less/controls.less", - "public/less/chat.less" + "public/less/chat.less", + "public/less/filters.less" ], "css/frappe-rtl.css": [ "public/css/bootstrap-rtl.css", diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js index d37e93a6b3..9cc5c43fee 100644 --- a/frappe/public/js/frappe/ui/filters/filter.js +++ b/frappe/public/js/frappe/ui/filters/filter.js @@ -65,6 +65,7 @@ frappe.ui.Filter = class { this.filter_edit_area.find(".set-filter-and-run").on("click", () => { this.filter_edit_area.removeClass("new-filter"); this.on_change(); + this.update_filter_tag(); }); this.filter_edit_area.find('.condition').change(() => { diff --git a/frappe/public/js/frappe/ui/filters/filters.js b/frappe/public/js/frappe/ui/filters/filters.js deleted file mode 100644 index df161892f7..0000000000 --- a/frappe/public/js/frappe/ui/filters/filters.js +++ /dev/null @@ -1,678 +0,0 @@ -// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -// MIT License. See license.txt - -frappe.ui.FilterList = Class.extend({ - init: function(opts) { - $.extend(this, opts); - this.filters = []; - this.wrapper = this.parent; - this.stats = []; - this.make(); - this.set_events(); - }, - make: function() { - this.wrapper.find('.show_filters, .filter_area').remove(); - this.wrapper.append(` -
-
- -
-
-
`); - }, - set_events: function() { - var me = this; - // show filters - this.wrapper.find('.new-filter').bind('click', function() { - me.add_filter(); - }); - - this.wrapper.find('.clear-filters').bind('click', function() { - me.clear_filters(); - $('.date-range-picker').val('') - me.base_list.run(); - $(this).addClass("hide"); - }); - }, - - show_filters: function() { - this.wrapper.find('.show_filters').toggle(); - if(!this.filters.length) { - this.add_filter(this.doctype, 'name'); - this.filters[0].wrapper.find(".filter_field input").focus(); - } - }, - - clear_filters: function() { - $.each(this.filters, function(i, f) { f.remove(true); }); - if(this.base_list.page.fields_dict) { - $.each(this.base_list.page.fields_dict, (key, value) => { - value.set_input(''); - }); - } - this.filters = []; - }, - - add_filter: function(doctype, fieldname, condition, value, hidden) { - // adds a new filter, returns true if filter has been added - - // allow equal to be used as like - let base_filter = this.base_list.page.fields_dict[fieldname]; - if (base_filter - && (base_filter.df.condition==condition - || (condition==='=' && base_filter.df.condition==='like'))) { - // if filter exists in base_list, then exit - this.base_list.page.fields_dict[fieldname].set_input(value); - - return true; - } - - if(doctype && fieldname - && !frappe.meta.has_field(doctype, fieldname) - && !in_list(frappe.model.std_fields_list, fieldname)) { - frappe.msgprint({ - message: __('Filter {0} missing', [fieldname.bold()]), - title: 'Invalid Filter', - indicator: 'red' - }); - return false; - } - - this.wrapper.find('.show_filters').toggle(true); - var is_new_filter = arguments.length===0; - - if (is_new_filter && this.wrapper.find(".is-new-filter:visible").length) { - // only allow 1 new filter at a time! - return false; - } - - var filter = this.push_new_filter(doctype, fieldname, condition, value); - if (!filter) return; - - if(this.wrapper.find('.clear-filters').hasClass("hide")) { - this.wrapper.find('.clear-filters').removeClass("hide"); - } - - if (filter && is_new_filter) { - filter.wrapper.addClass("is-new-filter"); - } else { - filter.freeze(); - } - - if (hidden) { - filter.$btn_group.addClass("hide"); - } - - return true; - }, - push_new_filter: function(doctype, fieldname, condition, value) { - if(this.filter_exists(doctype, fieldname, condition, value)) { - return; - } - - // if standard filter exists, then clear it. - if(this.base_list.page.fields_dict[fieldname]) { - this.base_list.page.fields_dict[fieldname].set_input(''); - } - - var filter = new frappe.ui.Filter({ - flist: this, - _doctype: doctype, - fieldname: fieldname, - condition: condition, - value: value - }); - - this.filters.push(filter); - - return filter; - }, - - remove: function(filter) { - // remove `filter` from flist - for (var i in this.filters) { - if (this.filters[i] === filter) { - break; - } - } - if (i!==undefined) { - // remove index - this.filters.splice(i, 1); - } - }, - - filter_exists: function(doctype, fieldname, condition, value) { - var flag = false; - for(var i in this.filters) { - if(this.filters[i].field) { - var f = this.filters[i].get_value(); - - if(f[0]==doctype && f[1]==fieldname && f[2]==condition && f[3]==value) { - flag = true; - } else if($.isArray(value) && frappe.utils.arrays_equal(value, f[3])) { - flag = true; - } - } - } - return flag; - }, - - get_filters: function() { - // get filter values as dict - var values = []; - $.each(this.filters, function(i, filter) { - if(filter.field) { - filter.freeze(); - values.push(filter.get_value()); - } - }); - this.base_list.update_standard_filters(values); - - return values; - }, - - // remove hidden filters - update_filters: function() { - var fl = []; - $.each(this.filters, function(i, f) { - if(f.field) fl.push(f); - }) - this.filters = fl; - if(this.filters.length === 0) { - this.wrapper.find('.clear-filters').addClass("hide"); - } - }, - - get_filter: function(fieldname) { - for(var i in this.filters) { - if(this.filters[i].field && this.filters[i].field.df.fieldname==fieldname) - return this.filters[i]; - } - }, - - get_formatted_value: function(field, val){ - var value = val; - - if(field.df.fieldname==="docstatus") { - value = {0:"Draft", 1:"Submitted", 2:"Cancelled"}[value] || value; - } else if(field.df.original_type==="Check") { - value = {0:"No", 1:"Yes"}[cint(value)]; - } - - value = frappe.format(value, field.df, {only_value: 1}); - return value; - } -}); - -frappe.ui.Filter = Class.extend({ - init: function(opts) { - $.extend(this, opts); - - this.doctype = this.flist.doctype; - this.make(); - this.make_select(); - this.set_events(); - }, - make: function() { - this.wrapper = $(frappe.render_template("edit_filter", {})) - .appendTo(this.flist.wrapper.find('.filter_area')); - }, - make_select: function() { - var me = this; - this.fieldselect = new frappe.ui.FieldSelect({ - parent: this.wrapper.find('.fieldname_select_area'), - doctype: this.doctype, - filter_fields: this.filter_fields, - select: function(doctype, fieldname) { - me.set_field(doctype, fieldname); - } - }); - if(this.fieldname) { - this.fieldselect.set_value(this._doctype || this.doctype, this.fieldname); - } - }, - set_events: function() { - var me = this; - - this.wrapper.find("a.remove-filter").on("click", function() { - me.remove(); - }); - - this.wrapper.find(".set-filter-and-run").on("click", function() { - me.wrapper.removeClass("is-new-filter"); - me.flist.base_list.run(); - me.apply(); - }); - - // add help for "in" codition - me.wrapper.find('.condition').change(function() { - if(!me.field) return; - var condition = $(this).val(); - if(in_list(["in", "like", "not in", "not like"], condition)) { - me.set_field(me.field.df.parent, me.field.df.fieldname, 'Data', condition); - if(!me.field.desc_area) { - me.field.desc_area = $('
').appendTo(me.field.wrapper); - } - // set description - me.field.desc_area.html((in_list(["in", "not in"], condition)==="in" - ? __("values separated by commas") - : __("use % as wildcard"))+'
'); - } else { - //if condition selected after refresh - me.set_field(me.field.df.parent, me.field.df.fieldname, null, condition); - } - }); - - // set the field - if(me.fieldname) { - // pre-sets given (could be via tags!) - return this.set_values(me._doctype, me.fieldname, me.condition, me.value); - } else { - me.set_field(me.doctype, 'name'); - } - }, - - apply: function() { - var f = this.get_value(); - - this.flist.remove(this); - this.flist.push_new_filter(f[0], f[1], f[2], f[3]); - this.remove(); - }, - - remove: function(dont_run) { - this.wrapper.remove(); - this.$btn_group && this.$btn_group.remove(); - this.field = null; - this.flist.update_filters(); - - if(!dont_run) { - this.flist.base_list.refresh(true); - } - }, - - set_values: function(doctype, fieldname, condition, value) { - // presents given (could be via tags!) - this.set_field(doctype, fieldname); - - // change 0,1 to Yes, No for check field type - if(this.field.df.original_type==='Check') { - if(value==0) value = 'No'; - else if(value==1) value = 'Yes'; - } - - if(condition) { - this.wrapper.find('.condition').val(condition).change(); - } - if(value!=null) { - return this.field.set_value(value); - } - }, - - set_field: function(doctype, fieldname, fieldtype, condition) { - var me = this; - - // set in fieldname (again) - var cur = me.field ? { - fieldname: me.field.df.fieldname, - fieldtype: me.field.df.fieldtype, - parent: me.field.df.parent, - } : {}; - - var original_docfield = me.fieldselect.fields_by_name[doctype][fieldname]; - if(!original_docfield) { - frappe.msgprint(__("Field {0} is not selectable.", [fieldname])); - return; - } - - var df = copy_dict(me.fieldselect.fields_by_name[doctype][fieldname]); - - // filter field shouldn't be read only or hidden - df.read_only = 0; - df.hidden = 0; - - if(!condition) this.set_default_condition(df, fieldtype); - this.set_fieldtype(df, fieldtype); - - // called when condition is changed, - // don't change if all is well - if(me.field && cur.fieldname == fieldname && df.fieldtype == cur.fieldtype && - df.parent == cur.parent) { - return; - } - - // clear field area and make field - me.fieldselect.selected_doctype = doctype; - me.fieldselect.selected_fieldname = fieldname; - - // save old text - var old_text = null; - if(me.field) { - old_text = me.field.get_value(); - } - - var field_area = me.wrapper.find('.filter_field').empty().get(0); - var f = frappe.ui.form.make_control({ - df: df, - parent: field_area, - only_input: true, - }) - f.refresh(); - - me.field = f; - if(old_text && me.field.df.fieldtype===cur.fieldtype) { - me.field.set_value(old_text); - } - - // run on enter - $(me.field.wrapper).find(':input').keydown(function(ev) { - if(ev.which==13) { - me.flist.base_list.run(); - } - }) - }, - - set_fieldtype: function(df, fieldtype) { - // reset - if(df.original_type) - df.fieldtype = df.original_type; - else - df.original_type = df.fieldtype; - - df.description = ''; df.reqd = 0; - df.ignore_link_validation = true; - - // given - if(fieldtype) { - df.fieldtype = fieldtype; - return; - } - - // scrub - if(df.fieldname=="docstatus") { - df.fieldtype="Select", - df.options=[ - {value:0, label:__("Draft")}, - {value:1, label:__("Submitted")}, - {value:2, label:__("Cancelled")} - ] - } else if(df.fieldtype=='Check') { - df.fieldtype='Select'; - df.options='No\nYes'; - } else if(['Text','Small Text','Text Editor','Code','Tag','Comments', - 'Dynamic Link','Read Only','Assign'].indexOf(df.fieldtype)!=-1) { - df.fieldtype = 'Data'; - } else if(df.fieldtype=='Link' && ['=', '!='].indexOf(this.wrapper.find('.condition').val())==-1) { - df.fieldtype = 'Data'; - } - if(df.fieldtype==="Data" && (df.options || "").toLowerCase()==="email") { - df.options = null; - } - if(this.wrapper.find('.condition').val()== "Between" && (df.fieldtype == 'Date' || df.fieldtype == 'Datetime')){ - df.fieldtype = 'DateRange'; - } - }, - - set_default_condition: function(df, fieldtype) { - if(!fieldtype) { - // set as "like" for data fields - if (df.fieldtype == 'Data') { - this.wrapper.find('.condition').val('like'); - } else if (df.fieldtype == 'Date' || df.fieldtype == 'Datetime'){ - this.wrapper.find('.condition').val('Between'); - }else{ - this.wrapper.find('.condition').val('='); - } - } - }, - - get_value: function() { - return [this.fieldselect.selected_doctype, - this.field.df.fieldname, this.get_condition(), this.get_selected_value()]; - }, - - get_selected_value: function() { - var val = this.field.get_value(); - - if(typeof val==='string') { - val = strip(val); - } - - if(this.field.df.original_type == 'Check') { - val = (val=='Yes' ? 1 :0); - } - - if(this.get_condition().indexOf('like', 'not like')!==-1) { - // automatically append wildcards - if(val) { - if(val.slice(0,1) !== "%") { - val = "%" + val; - } - if(val.slice(-1) !== "%") { - val = val + "%"; - } - } - } else if(in_list(["in", "not in"], this.get_condition())) { - if(val) { - val = $.map(val.split(","), function(v) { return strip(v); }); - } - } if(val === '%') { - val = ""; - } - - return val; - }, - - get_condition: function() { - return this.wrapper.find('.condition').val(); - }, - - freeze: function() { - if(this.$btn_group) { - // already made, just hide the condition setter - this.set_filter_button_text(); - this.wrapper.toggle(false); - return; - } - - var me = this; - - // add a button for new filter if missing - this.$btn_group = $(`
- -
`) - .insertAfter(this.flist.wrapper.find(".set-filters .new-filter")); - - this.set_filter_button_text(); - - this.$btn_group.find(".remove-filter").on("click", function() { - me.remove(); - }); - - this.$btn_group.find(".toggle-filter").on("click", function() { - $(this).closest('.show_filters').find('.filter_area').show(); - me.wrapper.toggle(); - }) - this.wrapper.toggle(false); - }, - - set_filter_button_text: function() { - var value = this.get_selected_value(); - value = this.flist.get_formatted_value(this.field, value); - - // for translations - // __("like"), __("not like"), __("in") - - this.$btn_group.find(".toggle-filter") - .html(repl('%(label)s %(condition)s "%(value)s"', { - label: __(this.field.df.label), - condition: __(this.get_condition()), - value: __(value), - })); - } - -}); - -// ') - .appendTo(this.parent) - .on("click", function () { $(this).select(); }); - this.select_input = this.$select.get(0); - this.awesomplete = new Awesomplete(this.select_input, { - minChars: 0, - maxItems: 99, - autoFirst: true, - list: me.options, - item: function(item, input) { - return $(repl('

  • %(label)s

    • ', item)) - .data("item.autocomplete", item) - .get(0); - } - }); - this.$select.on("awesomplete-select", function(e) { - var o = e.originalEvent; - var value = o.text.value; - var item = me.awesomplete.get_item(value); - me.selected_doctype = item.doctype; - me.selected_fieldname = item.fieldname; - if(me.select) me.select(item.doctype, item.fieldname); - }); - this.$select.on("awesomplete-selectcomplete", function(e) { - var o = e.originalEvent; - var value = o.text.value; - var item = me.awesomplete.get_item(value); - me.$select.val(item.label); - }); - - if(this.filter_fields) { - for(var i in this.filter_fields) - this.add_field_option(this.filter_fields[i]) - } else { - this.build_options(); - } - this.set_value(this.doctype, "name"); - window.last_filter = this; - }, - get_value: function() { - return this.selected_doctype ? this.selected_doctype + "." + this.selected_fieldname : null; - }, - val: function(value) { - if(value===undefined) { - return this.get_value(); - } else { - this.set_value(value); - } - }, - clear: function() { - this.selected_doctype = null; - this.selected_fieldname = null; - this.$select.val(""); - }, - set_value: function(doctype, fieldname) { - var me = this; - this.clear(); - if(!doctype) return; - - // old style - if(doctype.indexOf(".")!==-1) { - var parts = doctype.split("."); - doctype = parts[0]; - fieldname = parts[1]; - } - - $.each(this.options, function(i, v) { - if(v.doctype===doctype && v.fieldname===fieldname) { - me.selected_doctype = doctype; - me.selected_fieldname = fieldname; - me.$select.val(v.label); - return false; - } - }); - }, - build_options: function() { - var me = this; - me.table_fields = []; - var std_filters = $.map(frappe.model.std_fields, function(d) { - var opts = {parent: me.doctype} - if(d.fieldname=="name") opts.options = me.doctype; - return $.extend(copy_dict(d), opts); - }); - - // add parenttype column - var doctype_obj = locals['DocType'][me.doctype]; - if(doctype_obj && cint(doctype_obj.istable)) { - std_filters = std_filters.concat([{ - fieldname: 'parent', - fieldtype: 'Data', - label: 'Parent', - parent: me.doctype, - }]); - } - - // blank - if(this.with_blank) { - this.options.push({ - label:"", - value:"", - }) - } - - // main table - var main_table_fields = std_filters.concat(frappe.meta.docfield_list[me.doctype]); - $.each(frappe.utils.sort(main_table_fields, "label", "string"), function(i, df) { - // show fields where user has read access and if report hide flag is not set - if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide) - me.add_field_option(df); - }); - - // child tables - $.each(me.table_fields, function(i, table_df) { - if(table_df.options) { - var child_table_fields = [].concat(frappe.meta.docfield_list[table_df.options]); - $.each(frappe.utils.sort(child_table_fields, "label", "string"), function(i, df) { - // show fields where user has read access and if report hide flag is not set - if(frappe.perm.has_perm(me.doctype, df.permlevel, "read") && !df.report_hide) - me.add_field_option(df); - }); - } - }); - }, - - add_field_option: function(df) { - var me = this; - if(me.doctype && df.parent==me.doctype) { - var label = __(df.label); - var table = me.doctype; - if(df.fieldtype=='Table') me.table_fields.push(df); - } else { - var label = __(df.label) + ' (' + __(df.parent) + ')'; - var table = df.parent; - } - if(frappe.model.no_value_type.indexOf(df.fieldtype) == -1 && - !(me.fields_by_name[df.parent] && me.fields_by_name[df.parent][df.fieldname])) { - this.options.push({ - label: label, - value: table + "." + df.fieldname, - fieldname: df.fieldname, - doctype: df.parent - }); - if(!me.fields_by_name[df.parent]) me.fields_by_name[df.parent] = {}; - me.fields_by_name[df.parent][df.fieldname] = df; - } - }, -}) diff --git a/frappe/public/js/frappe/ui/sort_selector.html b/frappe/public/js/frappe/ui/sort_selector.html index 6312b711a7..2452c40d5e 100644 --- a/frappe/public/js/frappe/ui/sort_selector.html +++ b/frappe/public/js/frappe/ui/sort_selector.html @@ -1,19 +1,23 @@ -
    - -
    - - {{ __(sort_by_label) }} - - - +
    +
    + + {{ __(sort_by_label) }} + +
    +
    \ No newline at end of file diff --git a/frappe/public/less/filters.less b/frappe/public/less/filters.less new file mode 100644 index 0000000000..c419d49e47 --- /dev/null +++ b/frappe/public/less/filters.less @@ -0,0 +1,53 @@ +@import 'variables'; + +.active-tag-filters { + .add-filter, .filter-tag { + margin: 0 10px 10px 0; + } +} + +.active-tag-filters .btn-group { + white-space: nowrap; + font-size: 0; +} + +.active-tag-filters .btn-group .btn-default { + background-color: transparent; + border: 1px solid @border-color; + color: @text-muted; + float: none; +} + +.filter-box { + border-bottom: 1px solid @border-color; + padding: 10px 15px 3px; + + .remove-filter { + margin-top: 6px; + margin-left: 15px; + } + + .filter-field { + padding-right: 15px; + width: calc(100% - 36px); + + .frappe-control { + position: relative; + } + } +} + +// for sm and above +@media (min-width: @screen-xs) { + .filter-box .row > div[class*="col-sm-"] { + padding-right: 0px; + } + + .filter-field { + width: 65% !important; + + .frappe-control { + position: relative; + } + } +} diff --git a/frappe/public/less/list.less b/frappe/public/less/list.less index 8d9c9d147a..e39affdeb3 100644 --- a/frappe/public/less/list.less +++ b/frappe/public/less/list.less @@ -33,83 +33,30 @@ } } -.sort-selector { - .dropdown:hover { - text-decoration: underline; - } -} -.filter-list{ +.filter-list { position: relative; - + .tag-filters-area { + padding: 10px 150px 0 10px; + border-bottom: 1px solid @border-color; + } .sort-selector { position: absolute; - top: 15px; - right: 15px; - } -} - -.tag-filters-area { - padding: 15px 15px 0px; - border-bottom: 1px solid @border-color; -} - -.active-tag-filters { - padding-bottom: 4px; - padding-right: 120px; - - @media (max-width: @screen-xs) { - padding-right: 80px; - } -} - -.active-tag-filters .btn { - margin-bottom: 10px; -} - -.active-tag-filters .btn-group { - margin-left: 10px; - white-space: nowrap; - font-size: 0; -} - -.active-tag-filters .btn-group .btn-default { - background-color: transparent; - border: 1px solid @border-color; - color: @text-muted; - float: none; -} - -.filter-box { - border-bottom: 1px solid @border-color; - padding: 10px 15px 3px; - - .remove-filter { - margin-top: 6px; - margin-left: 15px; - } - - .filter-field { - padding-right: 15px; - width: calc(100% - 36px); - - .frappe-control { - position: relative; + top: 0px; + right: 0px; + margin: 10px; + display: flex; + .dropdown:hover { + text-decoration: underline; } - } -} - -// for sm and above -@media (min-width: @screen-xs) { - .filter-box .row > div[class*="col-sm-"] { - padding-right: 0px; - } - - .filter-field { - width: 65% !important; - - .frappe-control { - position: relative; + .dropdown-menu { + max-height: 300px; + overflow-y: scroll; + right: 0; + left: auto; + } + button { + margin-left: 10px; } } } diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 97581c0069..d047a9cec8 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -26,7 +26,7 @@ def make_xlsx(data, sheet_name, wb=None): for row in data: clean_row = [] for item in row: - if isinstance(item, string_types) and sheet_name != "Data Import Template": + if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']): value = handle_html(item) else: value = item