Merge branch 'staging-fixes' of https://github.com/frappe/frappe into staging-fixes

This commit is contained in:
Suraj Shetty 2018-12-10 16:27:00 +05:30
commit 9a0ff3bad0
19 changed files with 247 additions and 126 deletions

View file

@ -17,7 +17,7 @@ from faker import Faker
from .exceptions import *
from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
__version__ = '10.1.65'
__version__ = '10.1.67'
__title__ = "Frappe Framework"
local = Local()

View file

@ -15,7 +15,7 @@ frappe.ui.form.on('Prepared Report', {
<tbody></tbody>
</table>`);
const filters = JSON.parse(JSON.parse(frm.doc.filters));
const filters = JSON.parse(frm.doc.filters);
Object.keys(filters).forEach(key => {
const filter_row = $(`<tr>

View file

@ -10,12 +10,11 @@ import json
import frappe
from frappe.model.document import Document
from frappe.utils.background_jobs import enqueue
from frappe.desk.query_report import generate_report_result, get_columns_dict
from frappe.desk.query_report import generate_report_result
from frappe.utils.file_manager import save_file, remove_all
from frappe.utils.csvutils import to_csv, read_csv_content_from_attached_file
from frappe.desk.form.load import get_attachments
from frappe.utils.file_manager import download_file
from frappe.utils.file_manager import get_file
from frappe.utils import gzip_compress, gzip_decompress
class PreparedReport(Document):
@ -35,8 +34,8 @@ class PreparedReport(Document):
def run_background(instance):
report = frappe.get_doc("Report", instance.ref_report_doctype)
result = generate_report_result(report, filters=json.loads(instance.filters), user=instance.owner)
create_csv_file(result['columns'], result['result'], 'Prepared Report', instance.name)
result = generate_report_result(report, filters=instance.filters, user=instance.owner)
create_json_gz_file(result['result'], 'Prepared Report', instance.name)
instance.status = "Completed"
instance.columns = json.dumps(result["columns"])
@ -45,65 +44,31 @@ def run_background(instance):
frappe.publish_realtime(
'report_generated',
{"report_name": instance.report_name},
{"report_name": instance.report_name, "name": instance.name},
user=frappe.session.user
)
def remove_header_meta(columns):
column_list = []
columns_header = get_columns_dict(columns)
for idx in range(len(columns)):
column_list.append(columns_header[idx]['label'])
return column_list
def create_json_gz_file(data, dt, dn):
# Storing data in CSV file causes information loss
# Reports like P&L Statement were completely unsuable because of this
json_filename = '{0}.json.gz'.format(frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M"))
encoded_content = frappe.safe_encode(frappe.as_json(data))
def create_csv_file(columns, data, dt, dn):
csv_filename = '{0}.csv'.format(frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M"))
rows = []
if data:
columns_without_meta = remove_header_meta(columns)
row = data[0]
if type(row) == list:
rows = [tuple(columns_without_meta)] + data
else:
for row in data:
new_row = []
for col in columns:
key = col.get('fieldname') or col.get('label')
new_row.append(row.get(key, ''))
rows.append(new_row)
rows = [tuple(columns_without_meta)] + rows
encoded = base64.b64encode(frappe.safe_encode(to_csv(rows)))
# Call save_file function to upload and attach the file
# GZip compression seems to reduce storage requirements by 80-90%
compressed_content = gzip_compress(encoded_content)
save_file(
fname=csv_filename,
content=encoded,
fname=json_filename,
content=compressed_content,
dt=dt,
dn=dn,
folder=None,
decode=True,
is_private=False)
@frappe.whitelist()
def get_report_attachment_data(dn):
doc = frappe.get_doc("Prepared Report", dn)
data = read_csv_content_from_attached_file(doc)
return {
'columns': data[0],
'result': data[1:]
}
@frappe.whitelist()
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
download_file(attachment.file_url)
frappe.local.response.filename = attachment.file_name[:-2]
frappe.local.response.filecontent = gzip_decompress(get_file(attachment.name)[1])
frappe.local.response.type = "binary"

View file

@ -12,10 +12,11 @@ from frappe.utils import flt, cint, get_html_format, cstr, get_url_to_form
from frappe.model.utils import render_include
from frappe.translate import send_translations
import frappe.desk.reportview
from frappe.utils.csvutils import read_csv_content_from_attached_file
from frappe.permissions import get_role_permissions
from six import string_types, iteritems
from datetime import timedelta
from frappe.utils.file_manager import get_file
from frappe.utils import gzip_decompress
def get_report_doc(report_name):
doc = frappe.get_doc("Report", report_name)
@ -80,9 +81,6 @@ def generate_report_result(report, filters=None, user=None):
if result:
result = get_filtered_data(report.ref_doctype, columns, result, user)
if cint(report.add_total_row) and result:
result = add_total_row(result, columns)
return {
"result": result,
"columns": columns,
@ -102,7 +100,9 @@ def background_enqueue_run(report_name, filters=None, user=None):
frappe.get_doc({
"doctype": "Prepared Report",
"report_name": report_name,
"filters": json.dumps(filters),
# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
"filters": json.dumps(json.loads(filters)),
"ref_report_doctype": report_name,
"report_type": report.report_type,
"query": report.query,
@ -111,6 +111,7 @@ def background_enqueue_run(report_name, filters=None, user=None):
track_instance.insert(ignore_permissions=True)
frappe.db.commit()
return {
"name": track_instance.name,
"redirect_url": get_url_to_form("Prepared Report", track_instance.name)
}
@ -167,9 +168,10 @@ def run(report_name, filters=None, user=None):
filters = json.loads(filters)
dn = filters.get("prepared_report_name")
filters.pop("prepared_report_name", None)
else:
dn = ""
result = get_prepared_report_result(report, filters, dn)
result = get_prepared_report_result(report, filters, dn, user)
else:
result = generate_report_result(report, filters, user)
@ -178,9 +180,10 @@ def run(report_name, filters=None, user=None):
return result
def get_prepared_report_result(report, filters, dn=""):
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name})
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name, "filters": json.dumps(filters), "owner": user})
doc = None
if len(doc_list):
if dn:
@ -190,11 +193,15 @@ def get_prepared_report_result(report, filters, dn=""):
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
data = read_csv_content_from_attached_file(doc)
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
compressed_content = get_file(attached_file_name)[1]
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content)
if data:
latest_report_data = {
"columns": json.loads(doc.columns) if doc.columns else data[0],
"result": data[1:]
"result": data
}
latest_report_data.update({

View file

@ -12,7 +12,7 @@ source_link = "https://github.com/frappe/frappe"
app_license = "MIT"
develop_version = '12.x.x-develop'
staging_version = '11.0.3-beta.32'
staging_version = '11.0.3-beta.34'
app_email = "info@frappe.io"

View file

@ -230,4 +230,5 @@ frappe.patches.v10_0.enhance_security
frappe.patches.v11_0.multiple_references_in_events
frappe.patches.v11_0.set_allow_self_approval_in_workflow
frappe.patches.v11_0.remove_skip_for_doctype
frappe.patches.v11_0.migrate_report_settings_for_new_listview
frappe.patches.v11_0.migrate_report_settings_for_new_listview
frappe.patches.v11_0.delete_all_prepared_reports

View file

@ -0,0 +1,6 @@
import frappe
def execute():
prepared_reports = frappe.get_all("Prepared Report")
for report in prepared_reports:
frappe.delete_doc("Prepared Report", report.name)

View file

@ -5,6 +5,10 @@ const Block = Quill.import('blots/block');
Block.tagName = 'DIV';
Quill.register(Block, true);
const CodeBlockContainer = Quill.import('formats/code-block-container');
CodeBlockContainer.tagName = 'PRE';
Quill.register(CodeBlockContainer, true);
// table
const Table = Quill.import('formats/table-container');
const superCreate = Table.create.bind(Table);

View file

@ -301,7 +301,7 @@ frappe.ui.form.Dashboard = Class.extend({
}
}
frappe.set_route("List", doctype);
frappe.set_route("List", doctype, "List");
},
get_document_filter: function(doctype) {
// return the default filter for the given document

View file

@ -392,6 +392,10 @@ frappe.views.BaseList = class BaseList {
// for child classes
}
on_filter_change() {
// fired when filters are added or removed
}
toggle_result_area() {
this.$result.toggle(this.data.length > 0);
this.$paging_area.toggle(this.data.length > 0);
@ -480,6 +484,7 @@ class FilterArea {
if (this.trigger_refresh) {
this.list_view.start = 0;
this.list_view.refresh();
this.list_view.on_filter_change();
}
}

View file

@ -52,6 +52,21 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
this.sort_by = this.view_user_settings.sort_by || 'modified';
this.sort_order = this.view_user_settings.sort_order || 'desc';
// set filters from user_settings or list_settings
if (this.view_user_settings.filters && this.view_user_settings.filters.length) {
// Priority 1: user_settings
const saved_filters = this.view_user_settings.filters;
this.filters = this.validate_filters(saved_filters);
} else {
// Priority 2: filters in listview_settings
this.filters = (this.settings.filters || []).map(f => {
if (f.length === 3) {
f = [this.doctype, f[0], f[1], f[2]];
}
return f;
});
}
// build menu items
this.menu_items = this.menu_items.concat(this.get_menu_items());
@ -266,23 +281,8 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
before_refresh() {
if (frappe.route_options) {
// Priority 1: route filters
this.filters = this.parse_filters_from_route_options();
} else if (this.view_user_settings.filters && this.view_user_settings.filters.length) {
// Priority 2: saved filters
const saved_filters = this.view_user_settings.filters;
this.filters = this.validate_filters(saved_filters);
} else {
// Priority 3: filters in listview_settings
this.filters = (this.settings.filters || []).map(f => {
if (f.length === 3) {
f = [this.doctype, f[0], f[1], f[2]];
}
return f;
});
}
if (this.filters.length) {
return this.filter_area.clear(false)
.then(() => this.filter_area.set(this.filters));
}
@ -290,6 +290,15 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
return Promise.resolve();
}
parse_filters_from_settings() {
return (this.settings.filters || []).map(f => {
if (f.length === 3) {
f = [this.doctype, f[0], f[1], f[2]];
}
return f;
});
}
toggle_result_area() {
super.toggle_result_area();
this.toggle_actions_menu_button(

View file

@ -125,7 +125,7 @@ frappe.views.GanttView = class GanttView extends frappe.views.ListView {
var html =
`<h5>${task.name}</h5>
<p>${task._start.format('MMM D')} - ${task._end.format('MMM D')}</p>`;
<p>${moment(task._start).format('MMM D')} - ${moment(task._end).format('MMM D')}</p>`;
// custom html in doctype settings
var custom = me.settings.gantt_custom_popup_html;
@ -207,3 +207,4 @@ frappe.views.GanttView = class GanttView extends frappe.views.ListView {
];
}
};

View file

@ -32,10 +32,14 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
this.page_title = this.board_name;
this.card_meta = this.get_card_meta();
return this.get_board()
.then(() => {
this.filters = this.board.filters_array;
});
this.menu_items.push({
label: __('Save filters'),
action: () => {
this.save_kanban_board_filters();
}
});
return this.get_board();
}
get_board() {
@ -43,9 +47,14 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
.then(board => {
this.board = board;
this.board.filters_array = JSON.parse(this.board.filters || '[]');
this.filters = this.board.filters_array;
});
}
before_refresh() {
}
setup_view() {
}
@ -60,13 +69,40 @@ frappe.views.KanbanView = class KanbanView extends frappe.views.ListView {
this.save_view_user_settings({
last_kanban_board: this.board_name
});
}
on_filter_change() {
if (JSON.stringify(this.board.filters_array) !== JSON.stringify(this.filter_area.get())) {
this.page.set_indicator(__('Not Saved'), 'orange');
} else {
this.page.clear_indicator();
}
}
save_kanban_board_filters() {
const filters = this.filter_area.get();
frappe.call({
method: 'frappe.desk.doctype.kanban_board.kanban_board.save_filters',
args: {
board_name: this.board_name,
filters: this.filter_area.get()
filters: filters
}
}).then(r => {
if (r.exc) {
frappe.show_alert({
indicator: 'red',
message: __('There was an error saving filters')
});
return;
}
frappe.show_alert({
indicator: 'green',
message: __('Filters saved')
});
this.board.filters_array = filters;
this.on_filter_change();
});
}

View file

@ -63,9 +63,17 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
setup_events() {
frappe.realtime.on("report_generated", (data) => {
if(data.report_name) {
let alert_message = `Report ${this.report_name} generated.
<a target='_blank' href="#query-report/${this.report_name}">View</a>`;
frappe.show_alert({message: alert_message, indicator: 'orange'});
this.prepared_report_action = "Rebuild";
// If generated report and currently active Prepared Report has same fiters
// then refresh the Prepared Report
// Otherwise show alert with the link to the Prepared Report
if(data.name == this.prepared_report_doc_name) {
this.refresh();
} else {
let alert_message = `Report ${this.report_name} generated.
<a target='_blank' href="#query-report/${this.report_name}/?prepared_report_name=${data.name}">View</a>`;
frappe.show_alert({message: alert_message, indicator: 'orange'});
}
}
});
}
@ -93,6 +101,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
this.page_title = __(this.report_name);
this.menu_items = this.get_menu_items();
this.datatable = null;
this.prepared_report_action = "New";
frappe.run_serially([
() => this.get_report_doc(),
@ -266,8 +275,22 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
this.hide_status();
if (data.prepared_report){
if (data.prepared_report) {
this.prepared_report = true;
const query_string = frappe.utils.get_query_string(frappe.get_route_str());
const query_params = frappe.utils.get_query_params(query_string);
// If query_string contains prepared_report_name then set filters
// to match the mentioned prepared report doc and disable editing
if(query_params.prepared_report_name) {
this.prepared_report_action = "Edit";
const filters_from_report = JSON.parse(data.doc.filters);
Object.values(this.filters).forEach(function(field) {
if (filters_from_report[field.fieldname]) {
field.set_input(filters_from_report[field.fieldname]);
}
field.input.disabled = true;
});
}
this.add_prepared_report_buttons(data.doc);
}
this.toggle_message(false);
@ -297,19 +320,43 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
+"dn="+encodeURIComponent(doc.name)));
});
this.show_status(__(`
<span class="indicator orange">This report was <a href=#Form/Prepared%20Report/${doc.name}>generated</a>
on ${frappe.datetime.convert_to_user_tz(doc.report_end_time)}.
<a href=#List/Prepared%20Report>See all past reports</a>.</span>
`));
const part1 = __('This report was generated {0}.', [frappe.datetime.comment_when(doc.report_end_time)]);
const part2 = __('To get the updated report, click on {0}.', [__('Rebuild')]);
const part3 = __('See all past reports.');
this.show_status(`
<span class="indicator orange">
${part1}
${part2}
<a href="#List/Prepared%20Report?report_name=${this.report_name}">${part3}</a>
</span>
`);
};
// if
this.page.set_primary_action(
__("Generate New Report"),
this.generate_background_report.bind(this)
);
// Three cases
// 1. First time with given filters, no data.
// 2. Showing data from specific report
// 3. Showing data from an old report without specific report name
if(this.prepared_report_action == "New") {
this.page.set_primary_action(
__("Generate New Report"),
() => {
this.generate_background_report();
}
);
} else if(this.prepared_report_action == "Edit") {
this.page.set_primary_action(
__("Edit"),
() => {
frappe.set_route(frappe.get_route());
}
);
} else if(this.prepared_report_action == "Rebuild"){
this.page.set_primary_action(
__("Rebuild"),
this.generate_background_report.bind(this)
);
}
}
generate_background_report() {
@ -327,6 +374,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
callback: resolve
})).then(r => {
const data = r.message;
// Rememeber the name of Prepared Report doc
this.prepared_report_doc_name = data.name;
let alert_message = `Report initiated. You can track its status
<a class='text-info' target='_blank' href=${data.redirect_url}>here</a>`;
frappe.show_alert({message: alert_message, indicator: 'orange'});
@ -347,23 +396,25 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
if (this.datatable) {
this.datatable.options.treeView = this.tree_report;
this.datatable.refresh(this.data, this.columns);
return;
} else {
let datatable_options = {
columns: this.columns,
data: this.data,
inlineFilters: true,
treeView: this.tree_report,
layout: 'fixed',
cellHeight: 33
};
if (this.report_settings.get_datatable_options) {
datatable_options = this.report_settings.get_datatable_options(datatable_options);
}
this.datatable = new DataTable(this.$report[0], datatable_options);
}
let datatable_options = {
columns: this.columns,
data: this.data,
inlineFilters: true,
treeView: this.tree_report,
layout: 'fixed',
cellHeight: 33
};
if (this.report_settings.get_datatable_options) {
datatable_options = this.report_settings.get_datatable_options(datatable_options);
if (this.report_settings.after_datatable_render) {
this.report_settings.after_datatable_render(this.datatable);
}
this.datatable = new DataTable(this.$report[0], datatable_options);
}
get_chart_options(data) {
@ -896,6 +947,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
reset_report_view() {
this.hide_status();
this.toggle_nothing_to_show(true);
this.refresh();
}
toggle_loading(flag) {
@ -910,6 +962,10 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
Please set the appropriate filters and then generate a new one.`);
}
this.toggle_message(flag, message);
if(flag){
this.prepared_report_action = "New";
}
this.add_prepared_report_buttons();
}
toggle_message(flag, message) {

View file

@ -121,6 +121,16 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
}
}
on_filter_change() {
if (this.report_doc) {
if (JSON.stringify(this.filters) !== JSON.stringify(this.filter_area.get())) {
this.page.set_indicator(__('Not Saved'), 'orange');
} else {
this.page.clear_indicator();
}
}
}
update_row(doc, flash_row) {
const to_refresh = [];
@ -899,6 +909,10 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
if(r.message != this.report_name) {
frappe.set_route('List', this.doctype, 'Report', r.message);
}
// reset dirty state
this.filters = this.filter_area.get();
this.on_filter_change();
}
});

View file

@ -14,6 +14,8 @@ from email.utils import parseaddr, formataddr
from frappe.utils.data import *
from six.moves.urllib.parse import quote
from six import text_type, string_types
import io
from gzip import GzipFile
default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by',
'parent', 'parentfield', 'parenttype', 'idx', 'docstatus']
@ -620,3 +622,21 @@ def call(fn, *args, **kwargs):
bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}'''
"""
return json.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
# Following methods are aken as-is from Python 3 codebase
# since gzip.compress and gzip.decompress are not available in Python 2.7
def gzip_compress(data, compresslevel=9):
"""Compress data in one shot and return the compressed string.
Optional argument is the compression level, in range of 0-9.
"""
buf = io.BytesIO()
with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f:
f.write(data)
return buf.getvalue()
def gzip_decompress(data):
"""Decompress a gzip compressed string in one shot.
Return the decompressed string.
"""
with GzipFile(fileobj=io.BytesIO(data)) as f:
return f.read()

View file

@ -173,9 +173,6 @@ def build_page(path):
frappe.local.path = path
context = get_context(path)
if context.title and "{{" in cstr(context.title):
title_template = context.pop('title')
context.title = frappe.render_template(title_template, context)
if context.source:
html = frappe.render_template(context.source, context)

View file

@ -19,7 +19,7 @@
"awesomplete": "^1.1.2",
"cookie": "^0.3.1",
"express": "^4.16.2",
"frappe-datatable": "^1.5.5",
"frappe-datatable": "^1.6.1",
"frappe-gantt": "^0.1.0",
"fuse.js": "^3.2.0",
"highlight.js": "^9.12.0",

View file

@ -1219,10 +1219,10 @@ forwarded@~0.1.2:
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=
frappe-datatable@^1.5.5:
version "1.5.5"
resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.5.5.tgz#52c4e4263b2598d1685cdfc72dff248365091868"
integrity sha512-0Euo4otAzkpm1S+NrvhYHQ+Bug0aPus/k2W5FvyCXdRyKyDu5aSYAku+Xquj2mhES/EJGhyfhcuTcGLt0Q1h7g==
frappe-datatable@^1.6.1:
version "1.6.1"
resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.6.1.tgz#e57850923b5f307fd02328c522c5abe35a17dd89"
integrity sha512-u2l4I2Pwu4jTLSBF7vW7EDwzcRdfrlKbgaUCyhDUYlOhWl0sRt6rO2ZmIhU7znSYz7TNkKkAt7hkI+x+Mg6ONw==
dependencies:
hyperlist "^1.0.0-beta"
lodash "^4.17.5"