diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js
index 002e069874..6a7cf2728c 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.js
+++ b/frappe/core/doctype/prepared_report/prepared_report.js
@@ -15,7 +15,7 @@ frappe.ui.form.on('Prepared Report', {
`);
- const filters = JSON.parse(JSON.parse(frm.doc.filters));
+ const filters = JSON.parse(frm.doc.filters);
Object.keys(filters).forEach(key => {
const filter_row = $(`
diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py
index c85a9f4589..59f6748c56 100644
--- a/frappe/core/doctype/prepared_report/prepared_report.py
+++ b/frappe/core/doctype/prepared_report/prepared_report.py
@@ -6,6 +6,7 @@
from __future__ import unicode_literals
import base64
import json
+import io
import frappe
from frappe.model.document import Document
@@ -15,7 +16,9 @@ from frappe.core.doctype.file.file import remove_all
from frappe.utils.csvutils import to_csv, read_csv_content_from_attached_file
from frappe.desk.form.load import get_attachments
from frappe.core.doctype.file.file import download_file
-
+from frappe.utils import gzip_compress, gzip_decompress
+from six import PY2
+from frappe.utils import encode
class PreparedReport(Document):
@@ -35,8 +38,8 @@ class PreparedReport(Document):
def run_background(instance):
report = frappe.get_doc("Report", instance.ref_report_doctype)
- result = generate_report_result(report, filters=json.loads(instance.filters), user=instance.owner)
- create_csv_file(result['columns'], result['result'], 'Prepared Report', instance.name)
+ result = generate_report_result(report, filters=instance.filters, user=instance.owner)
+ create_json_gz_file(result['result'], 'Prepared Report', instance.name)
instance.status = "Completed"
instance.columns = json.dumps(result["columns"])
@@ -45,65 +48,52 @@ def run_background(instance):
frappe.publish_realtime(
'report_generated',
- {"report_name": instance.report_name},
+ {"report_name": instance.report_name, "name": instance.name},
user=frappe.session.user
)
-def remove_header_meta(columns):
- column_list = []
- columns_header = get_columns_dict(columns)
- for idx in range(len(columns)):
- column_list.append(columns_header[idx]['label'])
- return column_list
+def create_json_gz_file(data, dt, dn):
+ # Storing data in CSV file causes information loss
+ # Reports like P&L Statement were completely unsuable because of this
+ json_filename = '{0}.json.gz'.format(frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M"))
+ encoded_content = frappe.safe_encode(frappe.as_json(data))
+ compressed_content = gzip_compress(encoded_content)
-
-def create_csv_file(columns, data, dt, dn):
- csv_filename = '{0}.csv'.format(frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M"))
-
- rows = []
-
- if data:
- columns_without_meta = remove_header_meta(columns)
-
- row = data[0]
- if type(row) == list:
- rows = [tuple(columns_without_meta)] + data
- else:
- for row in data:
- new_row = []
- for col in columns:
- key = col.get('fieldname') or col.get('label')
- new_row.append(row.get(key, ''))
- rows.append(new_row)
-
- rows = [tuple(columns_without_meta)] + rows
-
- encoded = base64.b64encode(frappe.safe_encode(to_csv(rows)))
# Call save() file function to upload and attach the file
_file = frappe.get_doc({
"doctype": "File",
- "file_name": csv_filename,
+ "file_name": json_filename,
"attached_to_doctype": dt,
"attached_to_name": dn,
- "content": encoded,
+ "content": compressed_content,
"decode": True})
_file.save()
-
-@frappe.whitelist()
-def get_report_attachment_data(dn):
-
- doc = frappe.get_doc("Prepared Report", dn)
- data = read_csv_content_from_attached_file(doc)
-
- return {
- 'columns': data[0],
- 'result': data[1:]
- }
-
-
@frappe.whitelist()
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
- download_file(attachment.file_url)
+ frappe.local.response.filename = attachment.file_name[:-2]
+ frappe.local.response.filecontent = gzip_decompress(get_file(attachment.name)[1])
+ frappe.local.response.type = "binary"
+
+def get_file(fname):
+ """Returns [`file_name`, `content`] for given file name `fname`"""
+ _file = frappe.get_doc("File", {"file_name": fname})
+ file_path = _file.get_full_path()
+
+ # read the file
+ if PY2:
+ with open(encode(file_path)) as f:
+ content = f.read()
+ else:
+ with io.open(encode(file_path), mode='rb') as f:
+ content = f.read()
+ try:
+ # for plain text files
+ content = content.decode()
+ except UnicodeDecodeError:
+ # for .png, .jpg, etc
+ pass
+
+ return [file_path.rsplit("/", 1)[-1], content]
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index c5fda2ca66..2a511e83cd 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -12,10 +12,11 @@ from frappe.utils import flt, cint, get_html_format, cstr, get_url_to_form
from frappe.model.utils import render_include
from frappe.translate import send_translations
import frappe.desk.reportview
-from frappe.utils.csvutils import read_csv_content_from_attached_file
from frappe.permissions import get_role_permissions
from six import string_types, iteritems
from datetime import timedelta
+from frappe.utils.file_manager import get_file
+from frappe.utils import gzip_decompress
def get_report_doc(report_name):
doc = frappe.get_doc("Report", report_name)
@@ -99,7 +100,9 @@ def background_enqueue_run(report_name, filters=None, user=None):
frappe.get_doc({
"doctype": "Prepared Report",
"report_name": report_name,
- "filters": json.dumps(filters),
+ # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
+ # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
+ "filters": json.dumps(json.loads(filters)),
"ref_report_doctype": report_name,
"report_type": report.report_type,
"query": report.query,
@@ -108,6 +111,7 @@ def background_enqueue_run(report_name, filters=None, user=None):
track_instance.insert(ignore_permissions=True)
frappe.db.commit()
return {
+ "name": track_instance.name,
"redirect_url": get_url_to_form("Prepared Report", track_instance.name)
}
@@ -164,9 +168,10 @@ def run(report_name, filters=None, user=None):
filters = json.loads(filters)
dn = filters.get("prepared_report_name")
+ filters.pop("prepared_report_name", None)
else:
dn = ""
- result = get_prepared_report_result(report, filters, dn)
+ result = get_prepared_report_result(report, filters, dn, user)
else:
result = generate_report_result(report, filters, user)
@@ -175,9 +180,10 @@ def run(report_name, filters=None, user=None):
return result
-def get_prepared_report_result(report, filters, dn=""):
+def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
- doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name})
+ # Only look for completed prepared reports with given filters.
+ doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name, "filters": json.dumps(filters), "owner": user})
doc = None
if len(doc_list):
if dn:
@@ -187,11 +193,15 @@ def get_prepared_report_result(report, filters, dn=""):
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
- data = read_csv_content_from_attached_file(doc)
+ # Prepared Report data is stored in a GZip compressed JSON file
+ attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
+ compressed_content = get_file(attached_file_name)[1]
+ uncompressed_content = gzip_decompress(compressed_content)
+ data = json.loads(uncompressed_content)
if data:
latest_report_data = {
"columns": json.loads(doc.columns) if doc.columns else data[0],
- "result": data[1:]
+ "result": data
}
latest_report_data.update({
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 08ec07df46..bc3f9db205 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -12,7 +12,7 @@ source_link = "https://github.com/frappe/frappe"
app_license = "MIT"
develop_version = '12.x.x-develop'
-staging_version = '11.0.3-beta.33'
+staging_version = '11.0.3-beta.34'
app_email = "info@frappe.io"
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 7e1e0b982a..c057611ca9 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -232,3 +232,4 @@ frappe.patches.v11_0.multiple_references_in_events
frappe.patches.v11_0.set_allow_self_approval_in_workflow
execute:frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)')
frappe.patches.v11_0.migrate_report_settings_for_new_listview
+frappe.patches.v11_0.delete_all_prepared_reports
diff --git a/frappe/patches/v11_0/delete_all_prepared_reports.py b/frappe/patches/v11_0/delete_all_prepared_reports.py
new file mode 100644
index 0000000000..ee4b1dbd08
--- /dev/null
+++ b/frappe/patches/v11_0/delete_all_prepared_reports.py
@@ -0,0 +1,6 @@
+import frappe
+
+def execute():
+ prepared_reports = frappe.get_all("Prepared Report")
+ for report in prepared_reports:
+ frappe.delete_doc("Prepared Report", report.name)
diff --git a/frappe/public/js/frappe/form/controls/text_editor.js b/frappe/public/js/frappe/form/controls/text_editor.js
index 41de01e6f7..9e27dcf0f8 100644
--- a/frappe/public/js/frappe/form/controls/text_editor.js
+++ b/frappe/public/js/frappe/form/controls/text_editor.js
@@ -5,6 +5,10 @@ const Block = Quill.import('blots/block');
Block.tagName = 'DIV';
Quill.register(Block, true);
+const CodeBlockContainer = Quill.import('formats/code-block-container');
+CodeBlockContainer.tagName = 'PRE';
+Quill.register(CodeBlockContainer, true);
+
// table
const Table = Quill.import('formats/table-container');
const superCreate = Table.create.bind(Table);
diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js
index 6c3e699d85..4fc6d4da69 100644
--- a/frappe/public/js/frappe/views/reports/query_report.js
+++ b/frappe/public/js/frappe/views/reports/query_report.js
@@ -63,9 +63,17 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
setup_events() {
frappe.realtime.on("report_generated", (data) => {
if(data.report_name) {
- let alert_message = `Report ${this.report_name} generated.
- View`;
- frappe.show_alert({message: alert_message, indicator: 'orange'});
+ this.prepared_report_action = "Rebuild";
+ // If generated report and currently active Prepared Report has same fiters
+ // then refresh the Prepared Report
+ // Otherwise show alert with the link to the Prepared Report
+ if(data.name == this.prepared_report_doc_name) {
+ this.refresh();
+ } else {
+ let alert_message = `Report ${this.report_name} generated.
+ View`;
+ frappe.show_alert({message: alert_message, indicator: 'orange'});
+ }
}
});
}
@@ -93,6 +101,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
this.page_title = __(this.report_name);
this.menu_items = this.get_menu_items();
this.datatable = null;
+ this.prepared_report_action = "New";
frappe.run_serially([
() => this.get_report_doc(),
@@ -266,8 +275,22 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
this.hide_status();
- if (data.prepared_report){
+ if (data.prepared_report) {
this.prepared_report = true;
+ const query_string = frappe.utils.get_query_string(frappe.get_route_str());
+ const query_params = frappe.utils.get_query_params(query_string);
+ // If query_string contains prepared_report_name then set filters
+ // to match the mentioned prepared report doc and disable editing
+ if(query_params.prepared_report_name) {
+ this.prepared_report_action = "Edit";
+ const filters_from_report = JSON.parse(data.doc.filters);
+ Object.values(this.filters).forEach(function(field) {
+ if (filters_from_report[field.fieldname]) {
+ field.set_input(filters_from_report[field.fieldname]);
+ }
+ field.input.disabled = true;
+ });
+ }
this.add_prepared_report_buttons(data.doc);
}
this.toggle_message(false);
@@ -297,19 +320,43 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
+"dn="+encodeURIComponent(doc.name)));
});
- this.show_status(__(`
- This report was generated
- on ${frappe.datetime.convert_to_user_tz(doc.report_end_time)}.
- See all past reports.
- `));
+ const part1 = __('This report was generated {0}.', [frappe.datetime.comment_when(doc.report_end_time)]);
+ const part2 = __('To get the updated report, click on {0}.', [__('Rebuild')]);
+ const part3 = __('See all past reports.');
+
+ this.show_status(`
+
+ ${part1}
+ ${part2}
+ ${part3}
+
+ `);
};
- // if
-
- this.page.set_primary_action(
- __("Generate New Report"),
- this.generate_background_report.bind(this)
- );
+ // Three cases
+ // 1. First time with given filters, no data.
+ // 2. Showing data from specific report
+ // 3. Showing data from an old report without specific report name
+ if(this.prepared_report_action == "New") {
+ this.page.set_primary_action(
+ __("Generate New Report"),
+ () => {
+ this.generate_background_report();
+ }
+ );
+ } else if(this.prepared_report_action == "Edit") {
+ this.page.set_primary_action(
+ __("Edit"),
+ () => {
+ frappe.set_route(frappe.get_route());
+ }
+ );
+ } else if(this.prepared_report_action == "Rebuild"){
+ this.page.set_primary_action(
+ __("Rebuild"),
+ this.generate_background_report.bind(this)
+ );
+ }
}
generate_background_report() {
@@ -327,6 +374,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
callback: resolve
})).then(r => {
const data = r.message;
+ // Rememeber the name of Prepared Report doc
+ this.prepared_report_doc_name = data.name;
let alert_message = `Report initiated. You can track its status
here`;
frappe.show_alert({message: alert_message, indicator: 'orange'});
@@ -898,6 +947,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
reset_report_view() {
this.hide_status();
this.toggle_nothing_to_show(true);
+ this.refresh();
}
toggle_loading(flag) {
@@ -912,6 +962,10 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
Please set the appropriate filters and then generate a new one.`);
}
this.toggle_message(flag, message);
+ if(flag){
+ this.prepared_report_action = "New";
+ }
+ this.add_prepared_report_buttons();
}
toggle_message(flag, message) {
diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py
index 403353ee59..b3b46696bb 100644
--- a/frappe/utils/__init__.py
+++ b/frappe/utils/__init__.py
@@ -14,6 +14,8 @@ from email.utils import parseaddr, formataddr
from frappe.utils.data import *
from six.moves.urllib.parse import quote
from six import text_type, string_types
+import io
+from gzip import GzipFile
default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by',
'parent', 'parentfield', 'parenttype', 'idx', 'docstatus']
@@ -620,3 +622,21 @@ def call(fn, *args, **kwargs):
bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}'''
"""
return json.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
+
+# Following methods are aken as-is from Python 3 codebase
+# since gzip.compress and gzip.decompress are not available in Python 2.7
+def gzip_compress(data, compresslevel=9):
+ """Compress data in one shot and return the compressed string.
+ Optional argument is the compression level, in range of 0-9.
+ """
+ buf = io.BytesIO()
+ with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f:
+ f.write(data)
+ return buf.getvalue()
+
+def gzip_decompress(data):
+ """Decompress a gzip compressed string in one shot.
+ Return the decompressed string.
+ """
+ with GzipFile(fileobj=io.BytesIO(data)) as f:
+ return f.read()
diff --git a/package.json b/package.json
index 2aafb5a61e..747b1e95dd 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,7 @@
"awesomplete": "^1.1.2",
"cookie": "^0.3.1",
"express": "^4.16.2",
- "frappe-datatable": "^1.6.0",
+ "frappe-datatable": "^1.6.1",
"frappe-gantt": "^0.1.0",
"fuse.js": "^3.2.0",
"highlight.js": "^9.12.0",
diff --git a/yarn.lock b/yarn.lock
index f7fdc44b3e..9a0b9670ff 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1517,10 +1517,10 @@ forwarded@~0.1.2:
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=
-frappe-datatable@^1.6.0:
- version "1.6.0"
- resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.6.0.tgz#c86c2c7fc054a500c70cdf6c6362b5fed63c6fe6"
- integrity sha512-40iwguZr0w+X0MV/yTzsYDoKlH7b/GuOq6o2lEOrbVT3p4dZYpalxQmO8mkM9gKjNjSgGMVL1r6Se8peq80Qqg==
+frappe-datatable@^1.6.1:
+ version "1.6.1"
+ resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.6.1.tgz#e57850923b5f307fd02328c522c5abe35a17dd89"
+ integrity sha512-u2l4I2Pwu4jTLSBF7vW7EDwzcRdfrlKbgaUCyhDUYlOhWl0sRt6rO2ZmIhU7znSYz7TNkKkAt7hkI+x+Mg6ONw==
dependencies:
hyperlist "^1.0.0-beta"
lodash "^4.17.5"