fix: Data Exporter
- De duplicate child table rows - Add name column for child tables too - Show hidden tables in export
This commit is contained in:
Faris Ansari
parent
a16efb37b8
commit
a205917dac
4 changed files with 125 additions and 66 deletions
|
|
@ -6,9 +6,18 @@ import frappe
|
|||
from frappe import _
|
||||
from frappe.model import display_fieldtypes, no_value_fields, table_fields
|
||||
from frappe.utils.csvutils import build_csv_response
|
||||
from .importer_new import INVALID_VALUES
|
||||
|
||||
|
||||
class Exporter:
|
||||
def __init__(self, doctype, export_fields=None, export_data=False, export_filters=None, file_type='CSV'):
|
||||
def __init__(
|
||||
self,
|
||||
doctype,
|
||||
export_fields=None,
|
||||
export_data=False,
|
||||
export_filters=None,
|
||||
file_type="CSV",
|
||||
):
|
||||
"""
|
||||
Exports records of a DocType for use with Importer
|
||||
:param doctype: Document Type to export
|
||||
|
|
@ -20,6 +29,7 @@ class Exporter:
|
|||
self.doctype = doctype
|
||||
self.meta = frappe.get_meta(doctype)
|
||||
self.export_fields = export_fields
|
||||
self.export_filters = export_filters
|
||||
|
||||
# this will contain the csv content
|
||||
self.csv_array = []
|
||||
|
|
@ -30,40 +40,32 @@ class Exporter:
|
|||
self.add_header()
|
||||
|
||||
if export_data:
|
||||
self.data = self.get_data_to_export(export_filters)
|
||||
self.data = self.get_data_to_export()
|
||||
else:
|
||||
self.data = []
|
||||
self.add_data()
|
||||
|
||||
|
||||
def get_all_exportable_fields(self):
|
||||
return self.get_exportable_parent_fields() + self.get_exportable_children_fields()
|
||||
|
||||
|
||||
def get_exportable_parent_fields(self):
|
||||
name_field = frappe._dict({
|
||||
'fieldtype': 'Data',
|
||||
'fieldname': 'name',
|
||||
'label': 'ID',
|
||||
'reqd': 1,
|
||||
'parent': self.doctype
|
||||
})
|
||||
|
||||
parent_fields = self.get_exportable_fields(self.doctype)
|
||||
|
||||
# if autoname is based on field
|
||||
# then merge ID and the field column title as "ID (Autoname Field)"
|
||||
autoname = self.meta.autoname
|
||||
if autoname and autoname.startswith('field:'):
|
||||
fieldname = autoname[len('field:'):]
|
||||
if autoname and autoname.startswith("field:"):
|
||||
fieldname = autoname[len("field:") :]
|
||||
autoname_field = self.meta.get_field(fieldname)
|
||||
if autoname_field:
|
||||
name_field.label = 'ID ({})'.format(autoname_field.label)
|
||||
name_field = parent_fields[0]
|
||||
name_field.label = "ID ({})".format(autoname_field.label)
|
||||
# remove the autoname field as it is a duplicate of ID field
|
||||
parent_fields = [df for df in parent_fields if df.fieldname != autoname_field.fieldname]
|
||||
|
||||
return [name_field] + parent_fields
|
||||
parent_fields = [
|
||||
df for df in parent_fields if df.fieldname != autoname_field.fieldname
|
||||
]
|
||||
|
||||
return parent_fields
|
||||
|
||||
def get_exportable_children_fields(self):
|
||||
children = [df.options for df in self.meta.fields if df.fieldtype in table_fields]
|
||||
|
|
@ -73,8 +75,9 @@ class Exporter:
|
|||
|
||||
return children_fields
|
||||
|
||||
|
||||
def get_exportable_fields(self, doctype):
|
||||
fields = []
|
||||
|
||||
def is_exportable(df):
|
||||
return (
|
||||
df.fieldtype not in display_fieldtypes
|
||||
|
|
@ -87,83 +90,146 @@ class Exporter:
|
|||
# filter out layout fields
|
||||
fields = [df for df in meta.fields if is_exportable(df)]
|
||||
|
||||
if self.export_fields == 'All':
|
||||
return fields
|
||||
if self.export_fields == "Mandatory":
|
||||
fields = [df for df in fields if df.reqd]
|
||||
|
||||
if self.export_fields == 'Mandatory':
|
||||
return [df for df in fields if df.reqd]
|
||||
|
||||
if isinstance(self.export_fields, dict):
|
||||
elif isinstance(self.export_fields, dict):
|
||||
whitelist = self.export_fields.get(doctype, [])
|
||||
return [df for df in fields if df.fieldname in whitelist]
|
||||
fields = [df for df in fields if df.fieldname in whitelist]
|
||||
else:
|
||||
fields = [df for df in fields if df.reqd or df.in_list_view or df.bold]
|
||||
|
||||
return [df for df in fields if df.reqd or df.in_list_view or df.bold]
|
||||
name_field = frappe._dict(
|
||||
{
|
||||
"fieldtype": "Data",
|
||||
"fieldname": "name",
|
||||
"label": "ID",
|
||||
"reqd": 1,
|
||||
"parent": doctype,
|
||||
}
|
||||
)
|
||||
|
||||
if fields:
|
||||
return [name_field] + fields
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_data_to_export(self, filters=None):
|
||||
def get_data_to_export(self):
|
||||
frappe.permissions.can_export(self.doctype, raise_exception=True)
|
||||
|
||||
def get_column_name(df):
|
||||
return '`tab{0}`.`{1}`'.format(df.parent, df.fieldname)
|
||||
return "`tab{0}`.`{1}`".format(df.parent, df.fieldname)
|
||||
|
||||
fieldnames = [get_column_name(df) for df in self.fields]
|
||||
fields = [get_column_name(df) for df in self.fields]
|
||||
filters = self.export_filters
|
||||
|
||||
if self.meta.is_nested_set():
|
||||
order_by = '`tab{0}`.`lft` ASC'.format(self.doctype)
|
||||
order_by = "`tab{0}`.`lft` ASC".format(self.doctype)
|
||||
else:
|
||||
order_by = '`tab{0}`.`creation` DESC'.format(self.doctype)
|
||||
order_by = "`tab{0}`.`creation` DESC".format(self.doctype)
|
||||
|
||||
data = frappe.db.get_list(self.doctype,
|
||||
data = frappe.db.get_list(
|
||||
self.doctype,
|
||||
filters=filters,
|
||||
fields=fieldnames,
|
||||
fields=fields,
|
||||
limit_page_length=None,
|
||||
order_by=order_by,
|
||||
as_list=1,
|
||||
)
|
||||
|
||||
return self.remove_duplicate_parent_values(data)
|
||||
data = self.remove_duplicate_values(data)
|
||||
data = self.remove_row_gaps(data)
|
||||
data = self.remove_empty_rows(data)
|
||||
|
||||
def remove_duplicate_parent_values(self, data):
|
||||
return data
|
||||
|
||||
def remove_duplicate_values(self, data):
|
||||
out = []
|
||||
|
||||
parent_fields = self.get_exportable_parent_fields()
|
||||
parent_fields_count = len(parent_fields)
|
||||
doctypes = set([df.parent for df in self.fields])
|
||||
|
||||
def name_exists_in_column_before_row(name, column_index, row_index):
|
||||
column_values = [row[column_index] for i, row in enumerate(data) if i < row_index]
|
||||
return name in column_values
|
||||
|
||||
for i, row in enumerate(data):
|
||||
# first row is fine
|
||||
if i == 0:
|
||||
out.append(row)
|
||||
continue
|
||||
|
||||
current_name = None
|
||||
# first column is always name, can be used to find duplicate rows
|
||||
for row in data:
|
||||
row = list(row)
|
||||
name = row[0]
|
||||
for doctype in doctypes:
|
||||
name_index = self.get_name_column_index(doctype)
|
||||
name = row[name_index]
|
||||
column_indexes = self.get_column_indexes(doctype)
|
||||
|
||||
if name != current_name:
|
||||
current_name = name
|
||||
out.append(row)
|
||||
if name_exists_in_column_before_row(name, name_index, i):
|
||||
# remove the values from the row
|
||||
row = [None if i in column_indexes else d for i, d in enumerate(row)]
|
||||
|
||||
elif name == current_name:
|
||||
# remove the parent values from the row
|
||||
row = row[parent_fields_count:]
|
||||
# replace them with None values and add back the row contents
|
||||
row = [None] * parent_fields_count + row
|
||||
out.append(row)
|
||||
out.append(row)
|
||||
|
||||
return out
|
||||
|
||||
def remove_row_gaps(self, data):
|
||||
doctypes = set([df.parent for df in self.fields if df.parent != self.doctype])
|
||||
|
||||
def get_nearest_empty_row_index(col_index, row_index):
|
||||
col_values = [row[col_index] for row in data]
|
||||
i = row_index - 1
|
||||
while not col_values[i]:
|
||||
i = i - 1
|
||||
out = i + 1
|
||||
if row_index != out:
|
||||
return out
|
||||
|
||||
for i, row in enumerate(data):
|
||||
# if this is the row that contains parent values then skip
|
||||
if row[0]:
|
||||
continue
|
||||
|
||||
for doctype in doctypes:
|
||||
name_index = self.get_name_column_index(doctype)
|
||||
name = row[name_index]
|
||||
column_indexes = self.get_column_indexes(doctype)
|
||||
|
||||
if not name:
|
||||
continue
|
||||
|
||||
row_index = get_nearest_empty_row_index(name_index, i)
|
||||
if row_index:
|
||||
for col_index in column_indexes:
|
||||
data[row_index][col_index] = row[col_index]
|
||||
row[col_index] = None
|
||||
|
||||
return data
|
||||
|
||||
def remove_empty_rows(self, data):
|
||||
return [row for row in data if any(v not in INVALID_VALUES for v in row)]
|
||||
|
||||
def get_name_column_index(self, doctype):
|
||||
for i, df in enumerate(self.fields):
|
||||
if df.parent == doctype and df.fieldname == 'name':
|
||||
return i
|
||||
return -1
|
||||
|
||||
def get_column_indexes(self, doctype):
|
||||
return [i for i, df in enumerate(self.fields) if df.parent == doctype]
|
||||
|
||||
def add_header(self):
|
||||
def get_label(df):
|
||||
if df.parent == self.doctype:
|
||||
return df.label
|
||||
else:
|
||||
return '{0} ({1})'.format(df.label, df.parent)
|
||||
return "{0} ({1})".format(df.label, df.parent)
|
||||
|
||||
header = [get_label(df) for df in self.fields]
|
||||
self.csv_array.append(header)
|
||||
|
||||
|
||||
def add_data(self):
|
||||
self.csv_array += self.data
|
||||
|
||||
|
||||
def get_csv_array(self):
|
||||
return self.csv_array
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,7 @@ export default class ColumnPickerFields extends frappe.views.ReportView {
|
|||
get_fields_as_options() {
|
||||
let column_map = this.get_columns_for_picker();
|
||||
let doctypes = [this.doctype].concat(
|
||||
...frappe.meta
|
||||
.get_table_fields(this.doctype)
|
||||
.filter(df => !df.hidden)
|
||||
.map(df => df.options)
|
||||
...frappe.meta.get_table_fields(this.doctype).map(df => df.options)
|
||||
);
|
||||
// flatten array
|
||||
return [].concat(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ frappe.provide('frappe.data_import');
|
|||
|
||||
frappe.data_import.DataExporter = class DataExporter {
|
||||
constructor(doctype) {
|
||||
frappe.data_exporter = this;
|
||||
this.doctype = doctype;
|
||||
frappe.model.with_doctype(doctype, () => {
|
||||
this.make_dialog();
|
||||
|
|
@ -11,10 +12,7 @@ frappe.data_import.DataExporter = class DataExporter {
|
|||
|
||||
make_dialog() {
|
||||
let doctypes = [this.doctype].concat(
|
||||
...frappe.meta
|
||||
.get_table_fields(this.doctype)
|
||||
.filter(df => !df.hidden)
|
||||
.map(df => df.options)
|
||||
...frappe.meta.get_table_fields(this.doctype).map(df => df.options)
|
||||
);
|
||||
|
||||
this.dialog = new frappe.ui.Dialog({
|
||||
|
|
|
|||
|
|
@ -772,8 +772,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
|
|||
get_columns_for_picker() {
|
||||
let out = {};
|
||||
|
||||
const standard_fields_filter = df =>
|
||||
!in_list(frappe.model.no_value_type, df.fieldtype) && !df.report_hide;
|
||||
const standard_fields_filter = df => !in_list(frappe.model.no_value_type, df.fieldtype);
|
||||
|
||||
let doctype_fields = frappe.meta.get_docfields(this.doctype).filter(standard_fields_filter);
|
||||
|
||||
|
|
@ -786,8 +785,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
|
|||
|
||||
out[this.doctype] = doctype_fields;
|
||||
|
||||
const table_fields = frappe.meta.get_table_fields(this.doctype)
|
||||
.filter(df => !df.hidden);
|
||||
const table_fields = frappe.meta.get_table_fields(this.doctype);
|
||||
|
||||
table_fields.forEach(df => {
|
||||
const cdt = df.options;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue