feat!: faster generation and formatting utils for excel exports (#36323)
* feat: Style builder for report xlsx formatting * fix: update report to use direct import for query report execution * refactor: simplify module method retrieval in report execution * feat: get xlsx styles for report * refactor: enhance XLSXStyleBuilder with currency formatting and default style registration * feat: add xlsxwriter dependency for enhanced XLSX report generation * refactor: enhance XLSXStyleBuilder with improved style registration and formatting methods * feat: enhance XLSX export functionality with improved styling and metadata support * refactor: default formatting of currency * chore: remove some typo * feat: update make_xlsx function to use xlsxwriter for improved Excel file generation and styling * perf: some micro optimisations * refactor: inline generator back and improve condition * refactor: replace frappe.request_cache with functools.cache * fix: handle styling in email * fix: fix old test case to handle styles in export * refactor: enhance XLSX style handling and registration methods * refactor: improve currency formatting logic * fix: update make_xlsx to use constant_memory for large datasets and improve row style handling * fix: handle None style_id in XLSXStyleBuilder methods to prevent errors * fix: include owner field with proper doctype naming * fix: set default date format in XLSX workbook creation * fix: pass applied filters to metadata * fix: getting accurate field info for report view exporting * chore: Minor changes * feat: add function to generate default XLSX styles for exports * feat: integrate default XLSX styles into builder report export functionality * feat: styles on export docs xlsx * feat: enhance make_xlsx function to support file path saving * feat: add make_xls function for creating Excel files in old format and improve sheet name sanitization * fix: handle default date formatting * refactor: changes xlsx builder usage * refactor: update xlsx style builder usage * refactor: enhance field info retrieval with default field support * fix: handle update key in report data * refactor: enhance get_field_info to include options and improve label retrieval * fix: improve error handling for unsupported file formats and ensure applied filters are set correctly * refactor: update XLSX header index handling and improve metadata structure * fix: handle currency formatting in reportview export * fix: update default date format to datetime format in XLSX creation * fix: update serial number field in auto email report to use 'sr' instead of 'idx' * fix: enhance XLSX styling by adding right alignment for specific field types * chore: remove unused code * fix: update XLSXMetadata attributes for improved report styling options * perf: further improve currency styling * fix: correct column index mapping in XLSX export header * refactor: optimize indentation style registration in XLSXStyleBuilder * perf: improve apply_indentations * fix: reduce more attr lookup * refactor: remove duplication * fix: use report name in XLSX export instead of hardcoded title * fix: remove ignore_visible_idx from XLSXMetadata * fix: review * fix: update XLSX style fetching logic in build_xlsx_data function * fix: add right alignment to date, time, and datetime styles in XLSXStyleBuilder * fix: simplify number format handling in XLSXStyleBuilder * fix: register common styles in XLSXStyleBuilder for improved style management * test: add tests for XLSX styles structure and fieldtype column styles in XLSXStyleBuilder --------- Co-authored-by: Sagar Vora <16315650+sagarvora@users.noreply.github.com>
This commit is contained in:
Abdeali Chharchhodawala
GitHub
parent
9f5b45167d
commit
690826ff9b
8 changed files with 879 additions and 158 deletions
|
|
@ -7,7 +7,7 @@ from frappe.model import display_fieldtypes, no_value_fields
|
|||
from frappe.model import table_fields as table_fieldtypes
|
||||
from frappe.utils import flt, format_duration, groupby_metric
|
||||
from frappe.utils.csvutils import build_csv_response
|
||||
from frappe.utils.xlsxutils import build_xlsx_response
|
||||
from frappe.utils.xlsxutils import build_xlsx_response, get_default_xlsx_styles
|
||||
|
||||
|
||||
class Exporter:
|
||||
|
|
@ -253,7 +253,17 @@ class Exporter:
|
|||
if self.file_type == "CSV":
|
||||
build_csv_response(self.get_csv_array_for_export(), _(self.doctype))
|
||||
elif self.file_type == "Excel":
|
||||
build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype))
|
||||
data = self.get_csv_array_for_export()
|
||||
styles = get_default_xlsx_styles(
|
||||
columns=self.fields,
|
||||
# exclude header row
|
||||
data=data[1:],
|
||||
# from the second child row onwards, parent values will be empty
|
||||
# so currency value from parent doc may be absent, avoid inconsistency
|
||||
currency_formatting=False,
|
||||
)
|
||||
|
||||
build_xlsx_response(data, _(self.doctype), styles=styles)
|
||||
|
||||
def group_children_data_by_parent(self, children_data: dict[str, list]):
|
||||
return groupby_metric(children_data, key="parent")
|
||||
|
|
|
|||
|
|
@ -5,16 +5,17 @@ import json
|
|||
import threading
|
||||
|
||||
import frappe
|
||||
import frappe.desk.query_report
|
||||
from frappe import _, scrub
|
||||
from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles
|
||||
from frappe.core.doctype.page.page import delete_custom_role
|
||||
from frappe.desk.query_report import run
|
||||
from frappe.desk.reportview import append_totals_row
|
||||
from frappe.model.document import Document
|
||||
from frappe.modules import make_boilerplate
|
||||
from frappe.modules.export_file import export_to_files
|
||||
from frappe.utils import cint, cstr
|
||||
from frappe.utils.safe_exec import check_safe_sql_query, safe_exec
|
||||
from frappe.utils.xlsxutils import XLSXMetadata, XLSXStyleBuilder
|
||||
|
||||
|
||||
class Report(Document):
|
||||
|
|
@ -211,11 +212,14 @@ class Report(Document):
|
|||
|
||||
return res
|
||||
|
||||
def get_module_method(self, method):
|
||||
module = self.module or frappe.db.get_value("DocType", self.ref_doctype, "module")
|
||||
method_path = get_report_module_dotted_path(module, self.name) + "." + method
|
||||
return frappe.get_attr(method_path)
|
||||
|
||||
def execute_module(self, filters):
|
||||
# report in python module
|
||||
module = self.module or frappe.db.get_value("DocType", self.ref_doctype, "module")
|
||||
method_name = get_report_module_dotted_path(module, self.name) + ".execute"
|
||||
return frappe.get_attr(method_name)(frappe._dict(filters))
|
||||
return self.get_module_method("execute")(frappe._dict(filters))
|
||||
|
||||
def execute_script(self, filters):
|
||||
# server script
|
||||
|
|
@ -251,7 +255,7 @@ class Report(Document):
|
|||
self, filters=None, user=None, ignore_prepared_report=False, are_default_filters=True
|
||||
):
|
||||
columns, result = [], []
|
||||
data = frappe.desk.query_report.run(
|
||||
data = run(
|
||||
self.name,
|
||||
filters=filters,
|
||||
user=user,
|
||||
|
|
@ -323,8 +327,6 @@ class Report(Document):
|
|||
columns = params.get("fields")
|
||||
elif params.get("columns"):
|
||||
columns = params.get("columns")
|
||||
elif params.get("fields"):
|
||||
columns = params.get("fields")
|
||||
else:
|
||||
columns = [["name", self.ref_doctype]]
|
||||
columns.extend(
|
||||
|
|
@ -457,6 +459,18 @@ class Report(Document):
|
|||
|
||||
self.db_set("disabled", cint(disable))
|
||||
|
||||
def get_xlsx_styles_from_module(self, metadata: XLSXMetadata) -> dict:
|
||||
if self.is_standard != "Yes" or self.report_type not in ("Query Report", "Script Report"):
|
||||
return
|
||||
|
||||
try:
|
||||
method = self.get_module_method("get_xlsx_styles")
|
||||
except AttributeError:
|
||||
# Ignore if hook(method) is not defined
|
||||
return
|
||||
|
||||
return method(metadata)
|
||||
|
||||
|
||||
def is_prepared_report_enabled(report):
|
||||
return cint(frappe.db.get_value("Report", report, "prepared_report"))
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from frappe.monitor import add_data_to_monitor
|
|||
from frappe.permissions import get_role_permissions, get_roles, has_permission
|
||||
from frappe.utils import cint, cstr, flt, format_datetime, format_duration, formatdate, get_html_format, sbool
|
||||
from frappe.utils.caching import request_cache
|
||||
from frappe.utils.xlsxutils import XLSXMetadata, XLSXStyleBuilder, handle_html, make_xlsx
|
||||
|
||||
|
||||
def get_report_doc(report_name):
|
||||
|
|
@ -368,7 +369,6 @@ def run_export_query_job(user_email: str, form_params, csv_params):
|
|||
|
||||
def _export_query(form_params, csv_params, populate_response=True):
|
||||
from frappe.desk.utils import get_csv_bytes, provide_binary_file
|
||||
from frappe.utils.xlsxutils import handle_html, make_xlsx
|
||||
|
||||
report_name = form_params.report_name
|
||||
file_format_type = form_params.file_format_type
|
||||
|
|
@ -390,7 +390,10 @@ def _export_query(form_params, csv_params, populate_response=True):
|
|||
)
|
||||
|
||||
data = frappe._dict(data)
|
||||
data.filters = form_params.applied_filters
|
||||
|
||||
data.report_name = report_name
|
||||
data.filters = form_params.filters
|
||||
data.applied_filters = form_params.applied_filters
|
||||
|
||||
if not data.columns:
|
||||
frappe.respond_as_web_page(
|
||||
|
|
@ -417,28 +420,33 @@ def _export_query(form_params, csv_params, populate_response=True):
|
|||
|
||||
format_fields(data)
|
||||
|
||||
xlsx_data, column_widths, header_index = build_xlsx_data(
|
||||
xlsx_data, column_widths, styles = build_xlsx_data(
|
||||
data,
|
||||
include_indentation=include_indentation,
|
||||
include_filters=include_filters,
|
||||
include_hidden_columns=include_hidden_columns,
|
||||
build_styles=file_format_type == "Excel",
|
||||
)
|
||||
|
||||
if file_format_type == "CSV":
|
||||
file_extension = "csv"
|
||||
content = get_csv_bytes(
|
||||
[[handle_html(v) if isinstance(v, str) else v for v in r] for r in xlsx_data],
|
||||
csv_params,
|
||||
)
|
||||
file_extension = "csv"
|
||||
elif file_format_type == "Excel":
|
||||
file_extension = "xlsx"
|
||||
content = make_xlsx(
|
||||
xlsx_data,
|
||||
"Query Report",
|
||||
report_name,
|
||||
column_widths=column_widths,
|
||||
header_index=header_index,
|
||||
has_filters=bool(include_filters),
|
||||
styles=styles,
|
||||
).getvalue()
|
||||
else:
|
||||
frappe.throw(
|
||||
title=_("Unsupported file format: {0}").format(file_format_type),
|
||||
msg=_("Only CSV and Excel formats are supported for export"),
|
||||
)
|
||||
|
||||
if include_filters:
|
||||
for value in (data.filters or {}).values():
|
||||
|
|
@ -491,6 +499,10 @@ def format_fields(data: frappe._dict) -> None:
|
|||
row[index] = format_datetime(val)
|
||||
|
||||
|
||||
def format_filter_value(value):
|
||||
return ", ".join([cstr(x) for x in value]) if isinstance(value, list) else cstr(value)
|
||||
|
||||
|
||||
def build_xlsx_data(
|
||||
data: frappe._dict,
|
||||
visible_idx: list[int] | None = None,
|
||||
|
|
@ -498,24 +510,29 @@ def build_xlsx_data(
|
|||
include_filters: bool = False,
|
||||
ignore_visible_idx: bool = False,
|
||||
include_hidden_columns: bool = False,
|
||||
) -> tuple[list[list[Any]], list[int], int]:
|
||||
*,
|
||||
build_styles: bool = False,
|
||||
) -> tuple[list[list[Any]], list[int], dict | None]:
|
||||
"""
|
||||
Build Excel data structure from report data with proper formatting.
|
||||
|
||||
Args:
|
||||
data: Report data containing columns, result, and filters
|
||||
data: Report data containing columns, result, filters, applied_filters, report_name etc.
|
||||
visible_idx: Deprecated (v17). Row indices to include.
|
||||
include_indentation: Whether to include indentation for tree-like data
|
||||
include_filters: Whether to include filter rows at the top of the Excel sheet
|
||||
ignore_visible_idx: Deprecated (v17). Skips visible_idx filtering.
|
||||
include_hidden_columns: Whether to include columns marked as hidden
|
||||
build_styles: Whether to build style metadata for Excel formatting
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing:
|
||||
- result: List of rows for the Excel sheet
|
||||
- column_widths: List of column widths for the Excel sheet
|
||||
- header_index: Index of the header row in the result
|
||||
- styles: Dictionary of styles for Excel formatting (if applicable)
|
||||
"""
|
||||
metadata = None
|
||||
|
||||
EXCEL_TYPES = (
|
||||
str,
|
||||
bool,
|
||||
|
|
@ -546,41 +563,65 @@ def build_xlsx_data(
|
|||
visible_idx = set(visible_idx)
|
||||
|
||||
result = []
|
||||
column_data = []
|
||||
column_widths = []
|
||||
header_index = 0
|
||||
|
||||
include_hidden_columns = cint(include_hidden_columns)
|
||||
excel_row_idx = 0
|
||||
|
||||
include_filters = cint(include_filters)
|
||||
include_indentation = cint(include_indentation)
|
||||
include_hidden_columns = cint(include_hidden_columns)
|
||||
has_total_row = sbool(data.get("add_total_row"))
|
||||
|
||||
if cint(include_filters) and data.filters:
|
||||
if build_styles:
|
||||
metadata = XLSXMetadata(
|
||||
report_name=data.report_name,
|
||||
filters=data.filters,
|
||||
has_total_row=has_total_row,
|
||||
has_indentation=include_indentation,
|
||||
)
|
||||
|
||||
# adding applied filter rows
|
||||
if include_filters and data.applied_filters:
|
||||
filter_data = []
|
||||
for filter_name, filter_value in data.filters.items():
|
||||
for filter_name, filter_value in data.applied_filters.items():
|
||||
if not filter_value:
|
||||
continue
|
||||
filter_value = (
|
||||
", ".join([cstr(x) for x in filter_value])
|
||||
if isinstance(filter_value, list)
|
||||
else cstr(filter_value)
|
||||
)
|
||||
filter_data.append([cstr(filter_name), filter_value])
|
||||
|
||||
applied_filter = [cstr(filter_name), format_filter_value(filter_value)]
|
||||
|
||||
if build_styles:
|
||||
metadata.applied_filters_map[excel_row_idx] = applied_filter
|
||||
excel_row_idx += 1
|
||||
|
||||
filter_data.append(applied_filter)
|
||||
|
||||
# empty row after filters
|
||||
filter_data.append([])
|
||||
excel_row_idx += 1
|
||||
result += filter_data
|
||||
|
||||
# header is after filters + 1 empty row
|
||||
header_index = len(result)
|
||||
|
||||
column_data = []
|
||||
# adding header row
|
||||
column_idx = 0
|
||||
for column in data.columns:
|
||||
if column.get("hidden") and not include_hidden_columns:
|
||||
continue
|
||||
|
||||
if build_styles:
|
||||
metadata.column_map[column_idx] = column
|
||||
column_idx += 1
|
||||
|
||||
column_data.append(_(column.get("label")))
|
||||
column_width = cint(column.get("width", 0))
|
||||
# to convert into scale accepted by openpyxl
|
||||
# to convert into scale accepted by xlsxwriter
|
||||
column_width /= 10
|
||||
column_widths.append(column_width)
|
||||
|
||||
result.append(column_data)
|
||||
excel_row_idx += 1
|
||||
|
||||
# build table from result
|
||||
handle_indentation = include_indentation and not build_styles
|
||||
for row_idx, row in enumerate(data.result):
|
||||
# NOTE: for backwards compatibility. remove in v17.
|
||||
if not (ignore_visible_idx or row_idx in visible_idx):
|
||||
|
|
@ -589,6 +630,16 @@ def build_xlsx_data(
|
|||
row_data = []
|
||||
row_is_dict = isinstance(row, dict)
|
||||
|
||||
indent = 0
|
||||
if row_is_dict and handle_indentation:
|
||||
indent = row.get("indent") or 0
|
||||
if indent:
|
||||
indent = cint(indent)
|
||||
|
||||
if build_styles:
|
||||
metadata.row_map[excel_row_idx] = row
|
||||
excel_row_idx += 1
|
||||
|
||||
for col_idx, column in enumerate(data.columns):
|
||||
if column.get("hidden") and not include_hidden_columns:
|
||||
continue
|
||||
|
|
@ -600,14 +651,31 @@ def build_xlsx_data(
|
|||
if not isinstance(cell_value, EXCEL_TYPES):
|
||||
cell_value = cstr(cell_value)
|
||||
|
||||
if row_is_dict and include_indentation and "indent" in row and col_idx == 0:
|
||||
cell_value = (" " * cint(row["indent"])) + cstr(cell_value)
|
||||
if handle_indentation and indent and col_idx == 0:
|
||||
cell_value = (" " * indent) + cstr(cell_value)
|
||||
|
||||
row_data.append(cell_value)
|
||||
|
||||
result.append(row_data)
|
||||
|
||||
return result, column_widths, header_index
|
||||
return result, column_widths, get_xlsx_styles(metadata, data.report_name) if build_styles else None
|
||||
|
||||
|
||||
def get_xlsx_styles(metadata: XLSXMetadata, report_name: str | None = None) -> dict | None:
|
||||
"""
|
||||
Returns styles for XLSX export.
|
||||
|
||||
If report_name is provided, it tries to fetch styles defined in the report's module.
|
||||
"""
|
||||
styles = None
|
||||
if report_name:
|
||||
report = frappe.get_doc("Report", report_name)
|
||||
styles = report.get_xlsx_styles_from_module(metadata)
|
||||
|
||||
if not styles:
|
||||
styles = XLSXStyleBuilder(metadata).result
|
||||
|
||||
return styles
|
||||
|
||||
|
||||
def add_total_row(
|
||||
|
|
|
|||
|
|
@ -415,13 +415,14 @@ def run_report_view_export_job(user_email, form_params, csv_params):
|
|||
|
||||
def _export_query(form_params, csv_params, populate_response=True):
|
||||
from frappe.desk.utils import get_csv_bytes, provide_binary_file
|
||||
from frappe.utils.xlsxutils import handle_html, make_xlsx
|
||||
from frappe.utils.xlsxutils import get_default_xlsx_styles, handle_html, make_xlsx
|
||||
|
||||
doctype = form_params.pop("doctype")
|
||||
owner_field = f"`tab{doctype}`.`owner`"
|
||||
if isinstance(form_params["fields"], list):
|
||||
form_params["fields"].append("owner")
|
||||
form_params["fields"].append(owner_field)
|
||||
elif isinstance(form_params["fields"], tuple):
|
||||
form_params["fields"] = form_params["fields"] + ("owner",)
|
||||
form_params["fields"] = form_params["fields"] + (owner_field,)
|
||||
file_format_type = form_params.pop("file_format_type")
|
||||
title = form_params.pop("title", doctype)
|
||||
add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None
|
||||
|
|
@ -456,7 +457,8 @@ def _export_query(form_params, csv_params, populate_response=True):
|
|||
fields_info = get_field_info(db_query.fields, doctype)
|
||||
|
||||
labels = [info["label"] for info in fields_info]
|
||||
data = [[_("Sr"), *labels]]
|
||||
sr_label = _("Sr")
|
||||
data = [[sr_label, *labels]]
|
||||
processed_data = []
|
||||
|
||||
if frappe.local.lang == "en" or not translate_values:
|
||||
|
|
@ -481,7 +483,21 @@ def _export_query(form_params, csv_params, populate_response=True):
|
|||
)
|
||||
elif file_format_type == "Excel":
|
||||
file_extension = "xlsx"
|
||||
content = make_xlsx(data, doctype).getvalue()
|
||||
|
||||
styles = get_default_xlsx_styles(
|
||||
columns=[
|
||||
{
|
||||
"fieldname": "sr",
|
||||
"label": sr_label,
|
||||
"fieldtype": "Int",
|
||||
},
|
||||
*fields_info,
|
||||
],
|
||||
data=data[1:], # exclude header row
|
||||
has_total_row=bool(add_totals_row),
|
||||
)
|
||||
|
||||
content = make_xlsx(data, doctype, styles=styles).getvalue()
|
||||
|
||||
if not populate_response:
|
||||
return title, file_extension, content
|
||||
|
|
@ -509,69 +525,91 @@ def append_totals_row(data):
|
|||
return data
|
||||
|
||||
|
||||
def get_field_info(fields, doctype):
|
||||
"""Get column names, labels, field types, and translatable properties based on column names."""
|
||||
def get_field_info(fields, parent_doctype):
|
||||
"""
|
||||
Get field's
|
||||
- fieldname
|
||||
- label
|
||||
- fieldtype
|
||||
- translatable
|
||||
- options (if any)
|
||||
|
||||
:param fields: List of field names (can include child table fields and aggregate functions).
|
||||
:param parent_doctype: The main doctype from which the report is generated.
|
||||
"""
|
||||
from frappe.model.meta import get_default_df
|
||||
|
||||
field_info = []
|
||||
for key in fields:
|
||||
|
||||
for field in fields:
|
||||
df = None
|
||||
doctype = None
|
||||
try:
|
||||
parenttype, fieldname = parse_field(key)
|
||||
doctype, fieldname = parse_field(field)
|
||||
except ValueError:
|
||||
# handles aggregate functions
|
||||
parenttype = doctype
|
||||
if isinstance(key, dict):
|
||||
fieldname = next(k for k in key if k != "as")
|
||||
if isinstance(field, dict):
|
||||
# Eg: {"COUNT": "name", "as": "count_name"} -> "COUNT"
|
||||
fieldname = next(f for f in field if f != "as")
|
||||
else:
|
||||
fieldname = key.split("(", 1)[0]
|
||||
# Eg: "count(name)" -> "count"
|
||||
fieldname = field.split("(", 1)[0]
|
||||
fieldname = fieldname.capitalize()
|
||||
|
||||
parenttype = parenttype or doctype
|
||||
doctype = doctype or parent_doctype
|
||||
options = None
|
||||
|
||||
if parenttype == doctype and fieldname == "name":
|
||||
name = fieldname
|
||||
# Special-case the primary `name` column on the parent doctype
|
||||
if doctype == parent_doctype and fieldname == "name":
|
||||
label = _("ID", context="Label of name column in report")
|
||||
fieldtype = "Data"
|
||||
translatable = True
|
||||
else:
|
||||
df = frappe.get_meta(parenttype).get_field(fieldname)
|
||||
if df and df.fieldtype in ("Data", "Select", "Small Text", "Text"):
|
||||
name = df.name
|
||||
label = _(df.label)
|
||||
meta = frappe.get_meta(doctype)
|
||||
meta_df = meta.get_field(fieldname)
|
||||
df = meta_df or get_default_df(fieldname)
|
||||
|
||||
if df:
|
||||
fieldname = df.fieldname
|
||||
label = _(df.label or "") if meta_df else meta.get_label(fieldname)
|
||||
fieldtype = df.fieldtype
|
||||
translatable = getattr(df, "translatable", False)
|
||||
elif df and df.fieldtype == "Link" and frappe.get_meta(df.options).translated_doctype:
|
||||
name = df.name
|
||||
label = _(df.label)
|
||||
fieldtype = df.fieldtype
|
||||
translatable = True
|
||||
translatable = df.translatable or False
|
||||
options = df.options
|
||||
|
||||
if df.fieldtype == "Link" and options and frappe.get_meta(options).translated_doctype:
|
||||
translatable = True
|
||||
else:
|
||||
name = fieldname
|
||||
label = _(df.label) if df else _(fieldname)
|
||||
label = _(frappe.unscrub(fieldname))
|
||||
fieldtype = "Data"
|
||||
translatable = False
|
||||
|
||||
if parenttype != doctype:
|
||||
if doctype != parent_doctype:
|
||||
# If the column is from a child table, append the child doctype.
|
||||
# For example, "Item Code (Sales Invoice Item)".
|
||||
label += f" ({_(parenttype)})"
|
||||
label += f" ({_(doctype)})"
|
||||
|
||||
field_info.append(
|
||||
{"name": name, "label": label, "fieldtype": fieldtype, "translatable": translatable}
|
||||
{
|
||||
"fieldname": fieldname,
|
||||
"label": label,
|
||||
"fieldtype": fieldtype,
|
||||
"translatable": translatable,
|
||||
"options": options,
|
||||
}
|
||||
)
|
||||
|
||||
return field_info
|
||||
|
||||
|
||||
def handle_duration_fieldtype_values(doctype, data, fields):
|
||||
def handle_duration_fieldtype_values(parent_doctype, data, fields):
|
||||
for field in fields:
|
||||
try:
|
||||
parenttype, fieldname = parse_field(field)
|
||||
doctype, fieldname = parse_field(field)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
parenttype = parenttype or doctype
|
||||
df = frappe.get_meta(parenttype).get_field(fieldname)
|
||||
doctype = doctype or parent_doctype
|
||||
df = frappe.get_meta(doctype).get_field(fieldname)
|
||||
|
||||
if df and df.fieldtype == "Duration":
|
||||
index = fields.index(field) + 1
|
||||
|
|
@ -584,7 +622,14 @@ def handle_duration_fieldtype_values(doctype, data, fields):
|
|||
|
||||
|
||||
def parse_field(field: str | dict) -> tuple[str | None, str]:
|
||||
"""Parse a field into parenttype and fieldname."""
|
||||
"""
|
||||
Parse a field into doctype and fieldname.
|
||||
|
||||
:param field: The field string to parse.
|
||||
:returns: A tuple of (doctype, fieldname). Doctype is None if not specified.
|
||||
|
||||
:raises ValueError: If the field contains aggregate functions.
|
||||
"""
|
||||
if isinstance(field, dict): # for aggregates via qb
|
||||
raise ValueError
|
||||
|
||||
|
|
@ -690,7 +735,7 @@ def get_stats(stats: str, doctype: str, filters: str | None = None):
|
|||
|
||||
try:
|
||||
db_columns = frappe.db.get_table_columns(doctype)
|
||||
except (frappe.db.InternalError, frappe.db.ProgrammingError):
|
||||
except frappe.db.InternalError, frappe.db.ProgrammingError:
|
||||
# raised when _user_tags column is added on the fly
|
||||
# raised if its a virtual doctype
|
||||
db_columns = []
|
||||
|
|
|
|||
|
|
@ -159,9 +159,9 @@ class AutoEmailReport(Document):
|
|||
)
|
||||
|
||||
# add serial numbers
|
||||
columns.insert(0, frappe._dict(fieldname="idx", label="", width="30px"))
|
||||
columns.insert(0, frappe._dict(fieldname="sr", label=_("Sr"), fieldtype="Int", width="30px"))
|
||||
for i in range(len(data)):
|
||||
data[i]["idx"] = i + 1
|
||||
data[i]["sr"] = i + 1
|
||||
|
||||
if len(data) == 0 and self.send_if_data:
|
||||
return None
|
||||
|
|
@ -172,21 +172,23 @@ class AutoEmailReport(Document):
|
|||
return self.get_html_table(columns, data)
|
||||
|
||||
elif self.format in ("XLSX", "CSV"):
|
||||
report_data = frappe._dict()
|
||||
report_data["columns"] = columns
|
||||
report_data["result"] = data
|
||||
report_data = frappe._dict(
|
||||
{
|
||||
"report_name": self.report,
|
||||
"filters": self.filters,
|
||||
"columns": columns,
|
||||
"result": data,
|
||||
}
|
||||
)
|
||||
is_excel = self.format == "XLSX"
|
||||
|
||||
xlsx_data, column_widths, header_index = build_xlsx_data(
|
||||
report_data, [], 1, ignore_visible_idx=True
|
||||
xlsx_data, column_widths, styles = build_xlsx_data(
|
||||
report_data, [], 1, ignore_visible_idx=True, build_styles=is_excel
|
||||
)
|
||||
|
||||
if self.format == "XLSX":
|
||||
if is_excel:
|
||||
xlsx_file = make_xlsx(
|
||||
xlsx_data,
|
||||
"Auto Email Report",
|
||||
column_widths=column_widths,
|
||||
header_index=header_index,
|
||||
has_filters=bool(self.filters),
|
||||
xlsx_data, "Auto Email Report", column_widths=column_widths, styles=styles
|
||||
)
|
||||
|
||||
return xlsx_file.getvalue()
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
# License: MIT. See LICENSE
|
||||
|
||||
import frappe
|
||||
import frappe.utils
|
||||
from frappe.desk.query_report import build_xlsx_data, export_query, run
|
||||
from frappe.tests import IntegrationTestCase
|
||||
from frappe.utils.xlsxutils import make_xlsx
|
||||
from frappe.utils.xlsxutils import XLSXMetadata, XLSXStyleBuilder, make_xlsx
|
||||
|
||||
|
||||
class TestQueryReport(IntegrationTestCase):
|
||||
|
|
@ -31,7 +30,7 @@ class TestQueryReport(IntegrationTestCase):
|
|||
|
||||
self.assertEqual(type(xlsx_data), list)
|
||||
self.assertEqual(len(xlsx_data), 4) # columns + data
|
||||
# column widths are divided by 10 to match the scale that is supported by openpyxl
|
||||
# column widths are divided by 10 to match the scale that is supported by xlsxwriter
|
||||
self.assertListEqual(column_widths, [0, 10, 15])
|
||||
|
||||
for row in xlsx_data:
|
||||
|
|
@ -47,19 +46,19 @@ class TestQueryReport(IntegrationTestCase):
|
|||
|
||||
# Create mock data
|
||||
data = create_mock_data()
|
||||
data.filters = {"Label 1": "Filter Value", "Label 2": None, "Label 3": list(range(5))}
|
||||
|
||||
# Define the visible rows
|
||||
visible_idx = [0, 2, 3]
|
||||
|
||||
# Build the result
|
||||
xlsx_data, _column_widths, header_index = build_xlsx_data(
|
||||
data, visible_idx, include_indentation=False, include_filters=True
|
||||
xlsx_data, _column_widths, _ = build_xlsx_data(
|
||||
data,
|
||||
visible_idx,
|
||||
include_indentation=False,
|
||||
include_filters=True,
|
||||
)
|
||||
|
||||
self.assertEqual(header_index, 3) # 2 filter rows + 1 empty row
|
||||
|
||||
# Check if unset filters are skipped | Rows - 2 filters + 1 empty + 1 column + 3 data
|
||||
# Check if unset filters are skipped | Rows -> 2 filters + 1 empty + 1 column + 3 data
|
||||
self.assertEqual(len(xlsx_data), 7)
|
||||
|
||||
# Check filter formatting
|
||||
|
|
@ -69,6 +68,7 @@ class TestQueryReport(IntegrationTestCase):
|
|||
"""Test excel export using rows with composite cell value"""
|
||||
|
||||
data = frappe._dict()
|
||||
|
||||
data.columns = [
|
||||
{"label": "Column A", "fieldname": "column_a", "fieldtype": "Float"},
|
||||
{"label": "Column B", "fieldname": "column_b", "width": 150, "fieldtype": "Data"},
|
||||
|
|
@ -82,9 +82,9 @@ class TestQueryReport(IntegrationTestCase):
|
|||
visible_idx = [0, 1]
|
||||
|
||||
# Build the result
|
||||
xlsx_data, column_widths, header_index = build_xlsx_data(data, visible_idx, include_indentation=0)
|
||||
xlsx_data, column_widths, _ = build_xlsx_data(data, visible_idx, include_indentation=0)
|
||||
# Export to excel
|
||||
make_xlsx(xlsx_data, "Query Report", column_widths=column_widths, header_index=header_index)
|
||||
make_xlsx(xlsx_data, "Query Report", column_widths=column_widths)
|
||||
|
||||
for row in xlsx_data:
|
||||
# column_b should be 'str' even with composite cell value
|
||||
|
|
@ -250,6 +250,75 @@ data = columns, result
|
|||
raise e
|
||||
frappe.db.rollback()
|
||||
|
||||
def test_xlsx_styles_structure(self):
|
||||
"""build_xlsx_data with build_styles=True returns a well-formed styles dict"""
|
||||
data = create_mock_data()
|
||||
data.pop("report_name") # module not needed for this test
|
||||
|
||||
_, _, styles = build_xlsx_data(data, build_styles=True)
|
||||
|
||||
self.assertIsNotNone(styles)
|
||||
for key in ("styles", "column_styles", "row_styles", "cell_styles"):
|
||||
self.assertIn(key, styles)
|
||||
|
||||
# style registry must be non-empty
|
||||
self.assertGreater(len(styles["styles"]), 0)
|
||||
|
||||
# header row (index 0, no filters included) must have bold style
|
||||
self.assertIn(0, styles["row_styles"])
|
||||
|
||||
# resolve the header row's style IDs and confirm bold is set
|
||||
registry = styles["styles"]
|
||||
header_style_ids = styles["row_styles"][0]
|
||||
header_merged = {}
|
||||
for sid in header_style_ids:
|
||||
header_merged.update(registry[sid])
|
||||
self.assertTrue(header_merged.get("bold"))
|
||||
|
||||
def test_xlsx_style_builder_fieldtype_column_styles(self):
|
||||
"""XLSXStyleBuilder applies column styles for Float/Percent/Date but not Data"""
|
||||
column_map = {
|
||||
0: {"fieldname": "name", "fieldtype": "Data", "label": "Name"},
|
||||
1: {"fieldname": "score", "fieldtype": "Float", "label": "Score"},
|
||||
2: {"fieldname": "pct", "fieldtype": "Percent", "label": "Pct"},
|
||||
3: {"fieldname": "dt", "fieldtype": "Date", "label": "Date"},
|
||||
}
|
||||
row_map = {1: {"name": "A", "score": 1.0, "pct": 10.0, "dt": "2025-01-01"}}
|
||||
|
||||
metadata = XLSXMetadata(column_map=column_map, row_map=row_map)
|
||||
builder = XLSXStyleBuilder(metadata, default_styling=False)
|
||||
builder.apply_default_fieldtype_formats(currency_formatting=False)
|
||||
|
||||
def resolve(col_idx):
|
||||
"""Merge all style dicts registered for a column into one dict."""
|
||||
merged = {}
|
||||
for sid in builder.column_styles[col_idx]:
|
||||
merged.update(builder.styles[sid])
|
||||
return merged
|
||||
|
||||
# Float, Percent, Date → column-level styles
|
||||
self.assertIn(1, builder.column_styles)
|
||||
self.assertIn(2, builder.column_styles)
|
||||
self.assertIn(3, builder.column_styles)
|
||||
|
||||
# Data column → no column style
|
||||
self.assertNotIn(0, builder.column_styles)
|
||||
|
||||
# Float → has num_format, no alignment override
|
||||
float_style = resolve(1)
|
||||
self.assertIn("num_format", float_style)
|
||||
self.assertNotIn("align", float_style)
|
||||
|
||||
# Percent → num_format contains "%"
|
||||
percent_style = resolve(2)
|
||||
self.assertIn("num_format", percent_style)
|
||||
self.assertIn("%", percent_style["num_format"])
|
||||
|
||||
# Date → has num_format and explicitly right-aligned
|
||||
date_style = resolve(3)
|
||||
self.assertIn("num_format", date_style)
|
||||
self.assertEqual(date_style.get("align"), "right")
|
||||
|
||||
def test_export_report_via_email(self):
|
||||
REPORT_NAME = "Test CSV Report"
|
||||
REF_DOCTYPE = "DocType"
|
||||
|
|
@ -287,15 +356,22 @@ data = columns, result
|
|||
|
||||
def create_mock_data():
|
||||
data = frappe._dict()
|
||||
data.report_name = "Mock Report"
|
||||
|
||||
data.columns = [
|
||||
{"label": "Column A", "fieldname": "column_a", "fieldtype": "Float"},
|
||||
{"label": "Column B", "fieldname": "column_b", "width": 100, "fieldtype": "Float"},
|
||||
{"label": "Column C", "fieldname": "column_c", "width": 150, "fieldtype": "Duration"},
|
||||
]
|
||||
|
||||
data.result = [
|
||||
[1.0, 3.0, 600],
|
||||
{"column_a": 22.1, "column_b": 21.8, "column_c": 86412},
|
||||
{"column_b": 5.1, "column_c": 53234, "column_a": 11.1},
|
||||
[3.0, 1.5, 333],
|
||||
]
|
||||
|
||||
data.applied_filters = {"Label 1": "Filter Value", "Label 2": None, "Label 3": list(range(5))}
|
||||
data.filters = {"label_1": "Filter Value", "label_2": None, "label_3": list(range(5))}
|
||||
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -1,46 +1,488 @@
|
|||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# License: MIT. See LICENSE
|
||||
import datetime
|
||||
import functools
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import field as dataclass_field
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar, Literal
|
||||
|
||||
import openpyxl
|
||||
import xlrd
|
||||
import xlsxwriter
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.cell import WriteOnlyCell
|
||||
from openpyxl.styles import Font
|
||||
from openpyxl.utils import get_column_letter
|
||||
from openpyxl.workbook.child import INVALID_TITLE_REGEX
|
||||
from xlsxwriter.format import Format
|
||||
|
||||
import frappe
|
||||
from frappe import _
|
||||
from frappe.core.utils import html2text
|
||||
from frappe.utils import cint
|
||||
from frappe.utils.html_utils import unescape_html
|
||||
|
||||
ILLEGAL_CHARACTERS_RE = re.compile(
|
||||
r"[\000-\010]|[\013-\014]|[\016-\037]|\uFEFF|\uFFFE|\uFFFF|[\uD800-\uDFFF]"
|
||||
)
|
||||
|
||||
|
||||
def get_excel_date_format():
|
||||
date_format = frappe.get_system_settings("date_format")
|
||||
time_format = frappe.get_system_settings("time_format")
|
||||
|
||||
# Excel-compatible format
|
||||
date_format = date_format.replace("mm", "MM")
|
||||
|
||||
return date_format, time_format
|
||||
# as required by XLSXWriter
|
||||
INVALID_SHEET_NAME_RE = re.compile(r"[\[\]:*?/\\]")
|
||||
MAX_SHEET_NAME_LENGTH = 31
|
||||
|
||||
|
||||
# return xlsx file object
|
||||
### XLSX Formatter ###
|
||||
@dataclass(slots=True)
|
||||
class XLSXMetadata:
|
||||
"""
|
||||
Metadata container for XLSX report styling.
|
||||
|
||||
- All indexes must be 0-based respecting xlsxwriter's indexing.
|
||||
|
||||
Attributes:
|
||||
column_map: Maps column index to column dict (fieldname, fieldtype, etc.).
|
||||
row_map: Maps row index to row data (dict or list).
|
||||
applied_filters_map: Maps row index to list of applied filter label-value pairs.
|
||||
has_total_row: Whether the last row is a total row.
|
||||
has_indentation: Whether indentation styling should be applied.
|
||||
|
||||
# optional metadata for custom style builders
|
||||
report_name: Name of the report.
|
||||
filters: Raw filter values.
|
||||
"""
|
||||
|
||||
column_map: dict[int, dict] = dataclass_field(default_factory=dict)
|
||||
row_map: dict[int, dict | list] = dataclass_field(default_factory=dict)
|
||||
applied_filters_map: dict[int, list] = dataclass_field(default_factory=dict)
|
||||
|
||||
has_total_row: bool = False
|
||||
has_indentation: bool = False
|
||||
|
||||
# optional
|
||||
report_name: str = ""
|
||||
filters: dict = dataclass_field(default_factory=dict)
|
||||
|
||||
def get_column(self, fieldname: str) -> dict | None:
|
||||
"""
|
||||
Get column dict by fieldname, or None if not found.
|
||||
"""
|
||||
return next((col for col in self.column_map.values() if col.get("fieldname") == fieldname), None)
|
||||
|
||||
def get_header_index(self) -> int:
|
||||
"""
|
||||
Get header row index based on applied filters.
|
||||
Assumes header is always 1 row after the last filter row.
|
||||
"""
|
||||
count = len(self.applied_filters_map)
|
||||
return count + 1 if count else 0
|
||||
|
||||
def get_first_row_index(self) -> int:
|
||||
return min(self.row_map.keys()) if self.row_map else 0
|
||||
|
||||
def get_last_row_index(self) -> int:
|
||||
return max(self.row_map.keys()) if self.row_map else 0
|
||||
|
||||
|
||||
class XLSXStyleBuilder:
|
||||
"""
|
||||
Builder for XLSX cell styles based on report metadata.
|
||||
|
||||
Builds a style dictionary with:
|
||||
- styles: List of style definitions (xlsxwriter format properties). List index is the style ID.
|
||||
- column_styles: Maps column index to list of style IDs.
|
||||
- row_styles: Maps row index to list of style IDs.
|
||||
- cell_styles: Maps (row, col) tuple to list of style IDs.
|
||||
|
||||
**Usage:**
|
||||
|
||||
```
|
||||
builder = XLSXStyleBuilder(metadata)
|
||||
builder.style_column(0, builder.register_style({"bold": True}))
|
||||
styles = builder.result
|
||||
```
|
||||
"""
|
||||
|
||||
RIGHT_ALIGN_FIELDTYPES: ClassVar[set[str]] = {
|
||||
*frappe.model.numeric_fieldtypes,
|
||||
*frappe.model.datetime_fields,
|
||||
"Rating",
|
||||
}
|
||||
|
||||
def __init__(self, metadata: XLSXMetadata, default_styling: bool = True):
|
||||
self.metadata = metadata
|
||||
|
||||
# column fieldname -> index mapping
|
||||
self.field_index_map = {
|
||||
col["fieldname"]: idx for idx, col in self.metadata.column_map.items() if col.get("fieldname")
|
||||
}
|
||||
|
||||
self.styles: list[dict] = []
|
||||
self.column_styles: dict[int, list[int]] = {}
|
||||
self.row_styles: dict[int, list[int]] = {}
|
||||
self.cell_styles: dict[tuple[int, int], list[int]] = {}
|
||||
|
||||
self.result = {
|
||||
"styles": self.styles,
|
||||
"column_styles": self.column_styles,
|
||||
"row_styles": self.row_styles,
|
||||
"cell_styles": self.cell_styles,
|
||||
}
|
||||
|
||||
# metadata indexes for quick access
|
||||
self.header_index = self.metadata.get_header_index()
|
||||
self.first_row_index = self.metadata.get_first_row_index()
|
||||
self.last_row_index = self.metadata.get_last_row_index()
|
||||
self.row_is_dict = isinstance(self.metadata.row_map.get(self.first_row_index), dict)
|
||||
|
||||
self._register_common_styles()
|
||||
|
||||
if default_styling:
|
||||
self.apply_default_styles()
|
||||
|
||||
### STYLE REGISTRATION ###
|
||||
def _register_common_styles(self):
|
||||
self.bold_style_id = self.register_style({"bold": True})
|
||||
|
||||
def register_style(self, style: dict) -> int:
|
||||
"""
|
||||
Register a style and return its ID.
|
||||
|
||||
Style dict uses xlsxwriter format properties.
|
||||
"""
|
||||
if not style:
|
||||
frappe.throw(_("Cannot register an empty XLSX style"))
|
||||
|
||||
style_id = len(self.styles)
|
||||
self.styles.append(style)
|
||||
|
||||
return style_id
|
||||
|
||||
### STYLE APPLICATION ###
|
||||
def style_column(self, col_idx: int, style_id: int):
|
||||
"""
|
||||
Apply a style to all cells in a column.
|
||||
|
||||
Args:
|
||||
col_idx: 0-based column index
|
||||
style_id: ID of the style to apply (from register_style)
|
||||
"""
|
||||
if col_idx not in self.column_styles:
|
||||
self.column_styles[col_idx] = []
|
||||
|
||||
self.column_styles[col_idx].append(style_id)
|
||||
|
||||
return self
|
||||
|
||||
def style_row(self, row_idx: int, style_id: int):
|
||||
"""
|
||||
Apply a style to all cells in a row.
|
||||
|
||||
Args:
|
||||
row_idx: 0-based row index
|
||||
style_id: ID of the style to apply (from register_style)
|
||||
"""
|
||||
if row_idx not in self.row_styles:
|
||||
self.row_styles[row_idx] = []
|
||||
|
||||
self.row_styles[row_idx].append(style_id)
|
||||
|
||||
return self
|
||||
|
||||
def style_cell(self, row_idx: int, col_idx: int, style_id: int):
|
||||
"""
|
||||
Apply a style to a specific cell.
|
||||
|
||||
Args:
|
||||
row_idx: 0-based row index
|
||||
col_idx: 0-based column index
|
||||
style_id: ID of the style to apply (from register_style)
|
||||
"""
|
||||
key = (row_idx, col_idx)
|
||||
cell_styles = self.cell_styles
|
||||
|
||||
if key not in cell_styles:
|
||||
cell_styles[key] = []
|
||||
|
||||
cell_styles[key].append(style_id)
|
||||
|
||||
return self
|
||||
|
||||
### UTILITY METHODS FOR STYLING ###
|
||||
def apply_default_styles(self, currency_formatting: bool = True):
|
||||
"""
|
||||
Apply all default styles:
|
||||
|
||||
- Header row styling
|
||||
- Filter rows styling
|
||||
- Total row styling (if has_total_row)
|
||||
- Indentation styling (if has_indentation)
|
||||
- Default fieldtype formatting (numbers, dates, etc.)
|
||||
- Currency formatting can be toggled with currency_formatting flag
|
||||
"""
|
||||
self.style_header()
|
||||
|
||||
if self.metadata.applied_filters_map:
|
||||
self.style_filters()
|
||||
|
||||
if self.metadata.has_total_row:
|
||||
self.style_total_row()
|
||||
|
||||
if self.metadata.has_indentation:
|
||||
self.apply_indentations()
|
||||
|
||||
self.apply_default_fieldtype_formats(currency_formatting)
|
||||
|
||||
return self
|
||||
|
||||
def style_header(self):
|
||||
header_index = self.header_index
|
||||
|
||||
self.style_row(header_index, self.bold_style_id)
|
||||
|
||||
right_align = self.register_style({"align": "right"})
|
||||
left_align = self.register_style({"align": "left"})
|
||||
|
||||
for col_idx, col in self.metadata.column_map.items():
|
||||
self.style_cell(
|
||||
header_index,
|
||||
col_idx,
|
||||
right_align if col.get("fieldtype") in self.RIGHT_ALIGN_FIELDTYPES else left_align,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def style_filters(self):
|
||||
for row_idx in self.metadata.applied_filters_map.keys():
|
||||
# style only the label column (0th index)
|
||||
self.style_cell(row_idx, 0, self.bold_style_id)
|
||||
return self
|
||||
|
||||
def apply_indentations(self, col_idx: int = 0, field: str = "indent", pt: int = 2):
|
||||
if not self.row_is_dict:
|
||||
return self
|
||||
|
||||
@functools.cache
|
||||
def register_indent_style(indent: int) -> int:
|
||||
return self.register_style({"align": "left", "indent": indent * pt})
|
||||
|
||||
# quick access for hot loop
|
||||
last_row_index = self.last_row_index
|
||||
skip_last_row = self.metadata.has_total_row
|
||||
style_cell = self.style_cell
|
||||
|
||||
for row_idx, row in self.metadata.row_map.items():
|
||||
if skip_last_row and row_idx == last_row_index:
|
||||
continue
|
||||
|
||||
if indent := row.get(field):
|
||||
style_cell(row_idx, col_idx, register_indent_style(indent))
|
||||
|
||||
return self
|
||||
|
||||
def style_total_row(self):
|
||||
return self.style_row(self.last_row_index, self.bold_style_id)
|
||||
|
||||
def apply_default_fieldtype_formats(self, currency_formatting: bool = True):
|
||||
formats: dict[str, int] = {
|
||||
"Float": self.register_style({"num_format": self.get_number_format("Float")}),
|
||||
"Percent": self.register_style({"num_format": self.get_number_format("Percent")}),
|
||||
"Date": self.register_style({"num_format": self.get_date_format(), "align": "right"}),
|
||||
"Time": self.register_style({"num_format": self.get_time_format(), "align": "right"}),
|
||||
"Datetime": self.register_style({"num_format": self.get_datetime_format(), "align": "right"}),
|
||||
}
|
||||
|
||||
for idx, col in self.metadata.column_map.items():
|
||||
style_id = formats.get(col.get("fieldtype"))
|
||||
|
||||
if style_id is not None:
|
||||
self.style_column(idx, style_id)
|
||||
|
||||
if currency_formatting:
|
||||
self.apply_currency_fieldtype_formats()
|
||||
|
||||
return self
|
||||
|
||||
def apply_currency_fieldtype_formats(self):
|
||||
currency_options = {
|
||||
col_idx: col.get("options")
|
||||
for col_idx, col in self.metadata.column_map.items()
|
||||
if col.get("fieldtype") == "Currency"
|
||||
}
|
||||
|
||||
if not currency_options:
|
||||
return self
|
||||
|
||||
default_currency = frappe.db.get_default("currency")
|
||||
|
||||
# quick access for hot loop
|
||||
last_row_index = self.last_row_index
|
||||
skip_last_row = self.metadata.has_total_row
|
||||
currency_options_items = currency_options.items()
|
||||
style_cell = self.style_cell
|
||||
|
||||
# helpers
|
||||
@functools.cache
|
||||
def _get_value(doctype: str, docname: str, fieldname: str) -> str | None:
|
||||
return frappe.db.get_value(doctype, docname, fieldname)
|
||||
|
||||
@functools.cache
|
||||
def parse_options(options: str) -> tuple:
|
||||
parts = options.split(":")
|
||||
return parts if len(parts) == 3 else (None, None, None)
|
||||
|
||||
@functools.cache
|
||||
def register_currency_style(currency: str) -> int:
|
||||
return self.register_style({"num_format": self.get_number_format("Currency", currency)})
|
||||
|
||||
# dispatch dict/list row access once, not per cell
|
||||
if self.row_is_dict:
|
||||
|
||||
def get_row_value(row, field):
|
||||
return row.get(field)
|
||||
else:
|
||||
_field_index_get = self.field_index_map.get
|
||||
|
||||
def get_row_value(row, field):
|
||||
idx = _field_index_get(field)
|
||||
return row[idx] if idx is not None else None
|
||||
|
||||
# currency formatting
|
||||
for row_idx, row in self.metadata.row_map.items():
|
||||
if skip_last_row and row_idx == last_row_index:
|
||||
continue
|
||||
|
||||
for col_idx, options in currency_options_items:
|
||||
currency = None
|
||||
|
||||
if options:
|
||||
if ":" not in options:
|
||||
currency = get_row_value(row, options)
|
||||
else:
|
||||
doctype, link_field, currency_field = parse_options(options)
|
||||
if doctype is not None and (link_value := get_row_value(row, link_field)):
|
||||
currency = _get_value(doctype, link_value, currency_field)
|
||||
|
||||
style_cell(row_idx, col_idx, register_currency_style(currency or default_currency))
|
||||
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _get_currency_symbol_info(currency: str | None) -> tuple[str, bool]:
|
||||
if not currency or frappe.db.get_default("hide_currency_symbol") == "Yes":
|
||||
return "", False
|
||||
|
||||
symbol, on_right = frappe.db.get_value("Currency", currency, ["symbol", "symbol_on_right"])
|
||||
|
||||
return (symbol or currency), bool(on_right)
|
||||
|
||||
@staticmethod
|
||||
def _build_currency_format(
|
||||
format_string: str,
|
||||
currency_symbol: str | None = None,
|
||||
symbol_on_right: bool = False,
|
||||
) -> str:
|
||||
if not currency_symbol:
|
||||
return format_string
|
||||
|
||||
if symbol_on_right:
|
||||
return f'{format_string}" {currency_symbol}";-{format_string}" {currency_symbol}"'
|
||||
|
||||
return f'"{currency_symbol} "{format_string};"{currency_symbol} "-{format_string}'
|
||||
|
||||
### FORMAT GETTERS ###
|
||||
@staticmethod
|
||||
def get_number_format(
|
||||
fieldtype: Literal["Currency", "Float", "Percent"],
|
||||
currency: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get Excel number format string for the given fieldtype.
|
||||
"""
|
||||
from frappe.locale import get_number_format as _get_format
|
||||
|
||||
number_format = _get_format()
|
||||
thousands_sep = number_format.thousands_separator
|
||||
precision = number_format.precision
|
||||
|
||||
if fieldtype == "Currency":
|
||||
precision = cint(frappe.db.get_default("currency_precision")) or precision
|
||||
format_str = XLSXStyleBuilder._build_number_format(thousands_sep, precision)
|
||||
currency_symbol, symbol_on_right = XLSXStyleBuilder._get_currency_symbol_info(currency)
|
||||
return XLSXStyleBuilder._build_currency_format(format_str, currency_symbol, symbol_on_right)
|
||||
|
||||
elif fieldtype in ("Float", "Percent"):
|
||||
precision = cint(frappe.db.get_default("float_precision")) or precision
|
||||
format_str = XLSXStyleBuilder._build_number_format(thousands_sep, precision)
|
||||
return f'{format_str}"%" ' if fieldtype == "Percent" else format_str
|
||||
|
||||
return "General"
|
||||
|
||||
@staticmethod
|
||||
def _build_number_format(thousands_sep: str, precision: int = 0) -> str:
|
||||
# Decimal separator is hardcoded to '.' because Excel only understands '.' in format strings.
|
||||
# The system decimal separator is intentionally ignored here.
|
||||
# TODO: can be improved by passing a language/locale to xlsxwriter's Workbook for locale-aware formatting.
|
||||
integer_part = "#,##0" if thousands_sep else "#0"
|
||||
decimal_part = ("." + "0" * precision) if precision > 0 else ""
|
||||
|
||||
return f"{integer_part}{decimal_part}"
|
||||
|
||||
@staticmethod
|
||||
def get_date_format() -> str:
|
||||
return frappe.get_system_settings("date_format")
|
||||
|
||||
@staticmethod
|
||||
def get_time_format() -> str:
|
||||
return frappe.get_system_settings("time_format")
|
||||
|
||||
@staticmethod
|
||||
def get_datetime_format() -> str:
|
||||
return f"{XLSXStyleBuilder.get_date_format()} {XLSXStyleBuilder.get_time_format()}"
|
||||
|
||||
|
||||
def get_default_xlsx_styles(
|
||||
columns: list[dict],
|
||||
data: list[list | dict],
|
||||
applied_filters: list[list] | None = None,
|
||||
*,
|
||||
has_total_row: bool = False,
|
||||
has_indentation: bool = False,
|
||||
currency_formatting: bool = True,
|
||||
) -> dict:
|
||||
"""
|
||||
Generate default XLSX styles for xlsx exports.
|
||||
|
||||
Args:
|
||||
columns: Column definitions with keys: fieldname, fieldtype, label, options.
|
||||
data: Row data as list of dicts or lists (excluding header and filter rows).
|
||||
applied_filters: Filter rows to display at top of sheet. Each item is [label, value].
|
||||
has_total_row: If True, applies bold styling to the last row.
|
||||
has_indentation: If True, applies indent styles based on row's 'indent' key.
|
||||
currency_formatting: If True, applies currency number formats to Currency fields.
|
||||
"""
|
||||
applied_filters = applied_filters or []
|
||||
header_index = len(applied_filters) + 1 if applied_filters else 0
|
||||
|
||||
applied_filters_map = dict(enumerate(applied_filters))
|
||||
column_map = dict(enumerate(columns))
|
||||
row_map = dict(enumerate(data, start=header_index + 1)) # +1 for header row
|
||||
|
||||
metadata = XLSXMetadata(
|
||||
column_map=column_map,
|
||||
row_map=row_map,
|
||||
applied_filters_map=applied_filters_map,
|
||||
has_total_row=has_total_row,
|
||||
has_indentation=has_indentation,
|
||||
)
|
||||
|
||||
return XLSXStyleBuilder(metadata, default_styling=False).apply_default_styles(currency_formatting).result
|
||||
|
||||
|
||||
### Excel Creation ###
|
||||
def make_xlsx(
|
||||
data: list[list[Any]],
|
||||
sheet_name: str,
|
||||
wb: openpyxl.Workbook | None = None,
|
||||
wb: xlsxwriter.Workbook | None = None,
|
||||
column_widths: list[int] | None = None,
|
||||
header_index: int = 0,
|
||||
has_filters: bool = False,
|
||||
) -> BytesIO:
|
||||
styles: dict | None = None,
|
||||
) -> BytesIO | None:
|
||||
"""
|
||||
Create an Excel file with the given data and formatting options.
|
||||
|
||||
|
|
@ -48,64 +490,127 @@ def make_xlsx(
|
|||
data: List of rows, where each row is a list of cell values
|
||||
sheet_name: Name of the Excel sheet
|
||||
wb: Existing workbook to add sheet to. If None, creates new workbook
|
||||
- Workbook must be closed by caller if provided
|
||||
- Should be created with constant_memory=True for large datasets
|
||||
column_widths: List of column widths in Excel units. If None, auto-sized
|
||||
header_index: Row index (0-based) that should be formatted as header making it bold
|
||||
has_filters: If True, applies bold formatting to the first column of filter rows
|
||||
|
||||
styles: Dictionary defining styles for cells, rows, and columns
|
||||
- as returned by XLSXStyleBuilder.result
|
||||
Returns:
|
||||
BytesIO: object containing the Excel file data
|
||||
BytesIO | None: BytesIO object containing the Excel file data if a new workbook was created, otherwise None
|
||||
|
||||
"""
|
||||
column_widths = column_widths or []
|
||||
styles = styles or {}
|
||||
|
||||
# creating workbook
|
||||
xlsx_file = None
|
||||
created_wb = False # to know to close it later
|
||||
|
||||
if wb is None:
|
||||
wb = openpyxl.Workbook(write_only=True)
|
||||
xlsx_file = BytesIO()
|
||||
options = {"constant_memory": True}
|
||||
|
||||
sheet_name_sanitized = INVALID_TITLE_REGEX.sub(" ", sheet_name)
|
||||
ws = wb.create_sheet(sheet_name_sanitized, 0)
|
||||
if not styles:
|
||||
options["default_date_format"] = XLSXStyleBuilder.get_datetime_format()
|
||||
|
||||
wb = xlsxwriter.Workbook(xlsx_file, options)
|
||||
created_wb = True
|
||||
|
||||
ws = wb.add_worksheet(get_sanitized_sheet_name(sheet_name))
|
||||
|
||||
# extract style components
|
||||
def _extract_ids(key: str) -> dict:
|
||||
return {k: tuple(v) for k, v in (styles.get(key) or {}).items() if v}
|
||||
|
||||
style_registry: list[dict] = styles.get("styles") or []
|
||||
col_style_ids: dict[int, tuple[int, ...]] = _extract_ids("column_styles")
|
||||
row_style_ids: dict[int, tuple[int, ...]] = _extract_ids("row_styles")
|
||||
cell_style_ids: dict[tuple[int, int], tuple[int, ...]] = _extract_ids("cell_styles")
|
||||
|
||||
styling_enabled = bool(col_style_ids or row_style_ids or cell_style_ids)
|
||||
|
||||
if not styling_enabled:
|
||||
ws.set_row(0, cell_format=wb.add_format({"bold": True}))
|
||||
|
||||
def resolve_style_ids(style_ids: tuple[int, ...]) -> dict:
|
||||
if len(style_ids) == 1:
|
||||
return style_registry[style_ids[0]]
|
||||
|
||||
result = {}
|
||||
|
||||
for sid in style_ids:
|
||||
result.update(style_registry[sid])
|
||||
return result
|
||||
|
||||
@functools.cache
|
||||
def get_format(style_ids: tuple[int, ...]) -> Format:
|
||||
return wb.add_format(resolve_style_ids(style_ids))
|
||||
|
||||
# set column widths
|
||||
for i, column_width in enumerate(column_widths):
|
||||
if column_width:
|
||||
ws.column_dimensions[get_column_letter(i + 1)].width = column_width
|
||||
ws.set_column(i, i, column_width)
|
||||
|
||||
date_format, time_format = get_excel_date_format()
|
||||
bold_font = Font(name="Calibri", bold=True)
|
||||
# column level styles
|
||||
for col_idx, style_ids in col_style_ids.items():
|
||||
ws.set_column(col_idx, col_idx, cell_format=get_format(style_ids))
|
||||
|
||||
# row level styles (sorted because constant_memory mode requires writing rows in order)
|
||||
for row_idx, style_ids in sorted(row_style_ids.items()):
|
||||
ws.set_row(row_idx, cell_format=get_format(style_ids))
|
||||
|
||||
# priority: column < row < cell (later in tuple = higher priority)
|
||||
cell_formats: dict[tuple[int, int], Format] = {}
|
||||
|
||||
# process explicit cell styles
|
||||
for pos, cell_ids in cell_style_ids.items():
|
||||
row_idx, col_idx = pos
|
||||
col_ids = col_style_ids.get(col_idx, ())
|
||||
row_ids = row_style_ids.get(row_idx, ())
|
||||
|
||||
cell_formats[pos] = get_format(col_ids + row_ids + cell_ids)
|
||||
|
||||
# process row x column intersections (no explicit cell style)
|
||||
for row_idx, row_ids in row_style_ids.items():
|
||||
for col_idx, col_ids in col_style_ids.items():
|
||||
pos = (row_idx, col_idx)
|
||||
if pos not in cell_formats:
|
||||
cell_formats[pos] = get_format(col_ids + row_ids)
|
||||
|
||||
# quick access for hot loop
|
||||
handle_html_content = sheet_name not in {"Data Import Template", "Data Export"}
|
||||
illegal_chars_search = ILLEGAL_CHARACTERS_RE.search
|
||||
illegal_chars_sub = ILLEGAL_CHARACTERS_RE.sub
|
||||
|
||||
write = ws.write
|
||||
has_cell_formats = bool(cell_formats)
|
||||
get_cell_format = cell_formats.get
|
||||
|
||||
for row_idx, row in enumerate(data):
|
||||
clean_row = []
|
||||
is_header_row = row_idx == header_index
|
||||
is_filter_row = has_filters and row_idx < header_index
|
||||
for col_idx, value in enumerate(row):
|
||||
if isinstance(value, str):
|
||||
if handle_html_content:
|
||||
value = handle_html(value)
|
||||
|
||||
for col_idx, item in enumerate(row):
|
||||
if isinstance(item, str) and (sheet_name not in ["Data Import Template", "Data Export"]):
|
||||
value = handle_html(item)
|
||||
else:
|
||||
value = item
|
||||
if illegal_chars_search(value):
|
||||
value = illegal_chars_sub("", value)
|
||||
|
||||
if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
|
||||
# Remove illegal characters from the string
|
||||
value = ILLEGAL_CHARACTERS_RE.sub("", value)
|
||||
cell_format = get_cell_format((row_idx, col_idx)) if has_cell_formats else None
|
||||
write(row_idx, col_idx, value, cell_format)
|
||||
|
||||
cell = WriteOnlyCell(ws, value=value)
|
||||
if not created_wb:
|
||||
return
|
||||
|
||||
if isinstance(value, datetime.date | datetime.datetime):
|
||||
number_format = date_format
|
||||
if isinstance(value, datetime.datetime):
|
||||
number_format = f"{date_format} {time_format}"
|
||||
cell.number_format = number_format
|
||||
|
||||
# Apply bold font for header row or first column of filter rows
|
||||
if is_header_row or (is_filter_row and col_idx == 0):
|
||||
cell.font = bold_font
|
||||
|
||||
clean_row.append(cell)
|
||||
|
||||
ws.append(clean_row)
|
||||
|
||||
xlsx_file = BytesIO()
|
||||
wb.save(xlsx_file)
|
||||
wb.close()
|
||||
xlsx_file.seek(0)
|
||||
return xlsx_file
|
||||
|
||||
|
||||
### Utilities ###
|
||||
def get_sanitized_sheet_name(name: str) -> str:
|
||||
return INVALID_SHEET_NAME_RE.sub(" ", name)[:MAX_SHEET_NAME_LENGTH]
|
||||
|
||||
|
||||
def handle_html(data: str) -> str:
|
||||
# return if no html tags found
|
||||
if "<" not in data or ">" not in data:
|
||||
|
|
@ -148,7 +653,7 @@ def read_xls_file_from_attached_file(content):
|
|||
return [sheet.row_values(i) for i in range(sheet.nrows)]
|
||||
|
||||
|
||||
def build_xlsx_response(data, filename):
|
||||
def build_xlsx_response(data, filename, styles: dict | None = None):
|
||||
from frappe.desk.utils import provide_binary_file
|
||||
|
||||
provide_binary_file(filename, "xlsx", make_xlsx(data, filename).getvalue())
|
||||
provide_binary_file(filename, "xlsx", make_xlsx(data, filename, styles=styles).getvalue())
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ dependencies = [
|
|||
"num2words~=0.5.14",
|
||||
"oauthlib~=3.3.1",
|
||||
"openpyxl~=3.1.5",
|
||||
"xlsxwriter~=3.2.9",
|
||||
"orjson~=3.11.5",
|
||||
"passlib~=1.7.4",
|
||||
"pdfkit~=1.0.0",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue