From c3e636914fd94c61113d0178b29d682707ee4613 Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 28 Sep 2020 02:08:24 +0530 Subject: [PATCH 01/43] fix(integration-request): handle response and update status --- .../doctype/integration_request/integration_request.py | 10 ++++++++++ frappe/integrations/utils.py | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index 2cb656459b..33e87c4796 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -6,6 +6,8 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document import json +from six import string_types +from frappe.integrations.utils import json_handler class IntegrationRequest(Document): def autoname(self): @@ -20,3 +22,11 @@ class IntegrationRequest(Document): self.status = status self.save(ignore_permissions=True) frappe.db.commit() + + def process_response(self, ref_field, response): + if isinstance(response, string_types): + response = json.loads(response) + + status = "Completed" if ref_field == "output" else "Failed" + self.db_set(ref_field, json.dumps(response, default=json_handler)) + self.db_set("status", status) \ No newline at end of file diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 808affe47a..1af9682073 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -49,16 +49,20 @@ def make_post_request(url, auth=None, headers=None, data=None): frappe.log_error() raise exc -def create_request_log(data, integration_type, service_name, name=None): +def create_request_log(data, integration_type, service_name, name=None, error=None): if isinstance(data, string_types): data = json.loads(data) + if isinstance(error, string_types): + error = json.loads(error) + integration_request = frappe.get_doc({ "doctype": "Integration Request", "integration_type": integration_type, "integration_request_service": service_name, "reference_doctype": data.get("reference_doctype"), "reference_docname": data.get("reference_docname"), + "error": json.dumps(error, default=json_handler), "data": json.dumps(data, default=json_handler) }) From 80c75d9806acb6509196b4e1f07d55b4f3b00b8d Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 28 Sep 2020 19:04:09 +0200 Subject: [PATCH 02/43] feat: option to anonymize ip for google analytics --- frappe/data/sample_site_config.json | 3 +++ .../includes/app_analytics/google_analytics.html | 3 +++ .../doctype/website_settings/website_settings.json | 12 +++++++++--- frappe/www/desk.py | 1 + frappe/www/website_script.js | 3 +++ frappe/www/website_script.py | 10 ++++++++-- 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/frappe/data/sample_site_config.json b/frappe/data/sample_site_config.json index 36818ef286..715cd7b9fa 100644 --- a/frappe/data/sample_site_config.json +++ b/frappe/data/sample_site_config.json @@ -22,6 +22,9 @@ "use_ssl": 0, "auto_email_id": "hello@example.com", + "google_analytics_id": "google_analytics_id", + "google_analytics_anonymize_ip": 1, + "google_login": { "client_id": "google_client_id", "client_secret": "google_client_secret" diff --git a/frappe/templates/includes/app_analytics/google_analytics.html b/frappe/templates/includes/app_analytics/google_analytics.html index 65199548b9..7c3b165a8f 100644 --- a/frappe/templates/includes/app_analytics/google_analytics.html +++ b/frappe/templates/includes/app_analytics/google_analytics.html @@ -6,6 +6,9 @@ })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); ga('create', '{{ google_analytics_id }}', 'auto'); + {% if google_analytics_anonymize_ip %} + ga('set', 'anonymizeIp', true); + {% endif %} $(document).on("mousedown", function(event) { if(!frappe && !frappe.get_route) return; diff --git a/frappe/website/doctype/website_settings/website_settings.json b/frappe/website/doctype/website_settings/website_settings.json index 6ad0f2cd01..721bfd2d19 100644 --- a/frappe/website/doctype/website_settings/website_settings.json +++ b/frappe/website/doctype/website_settings/website_settings.json @@ -52,6 +52,7 @@ "indexing_authorization_code", "column_break_17", "google_analytics_id", + "google_analytics_anonymize_ip", "misc_section", "subdomain", "disable_signup", @@ -206,7 +207,6 @@ "label": "Integrations" }, { - "description": "Add Google Analytics ID: eg. UA-89XXX57-1. Please search help on Google Analytics for more information.", "fieldname": "google_analytics_id", "fieldtype": "Data", "label": "Google Analytics ID" @@ -401,6 +401,12 @@ "fieldname": "edit_footer_template_values", "fieldtype": "Button", "label": "Edit Values" + }, + { + "default": "1", + "fieldname": "google_analytics_anonymize_ip", + "fieldtype": "Check", + "label": "Google Analytics Anonymize IP" } ], "icon": "fa fa-cog", @@ -409,7 +415,7 @@ "issingle": 1, "links": [], "max_attachments": 10, - "modified": "2020-08-21 14:02:55.168829", + "modified": "2020-09-28 18:47:18.506700", "modified_by": "Administrator", "module": "Website", "name": "Website Settings", @@ -433,4 +439,4 @@ "sort_field": "modified", "sort_order": "ASC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/www/desk.py b/frappe/www/desk.py index 6cb7c8a077..c6bce850a5 100644 --- a/frappe/www/desk.py +++ b/frappe/www/desk.py @@ -43,6 +43,7 @@ def get_context(context): "boot": boot if context.get("for_mobile") else boot_json, "csrf_token": csrf_token, "google_analytics_id": frappe.conf.get("google_analytics_id"), + "google_analytics_anonymize_ip": frappe.conf.get("google_analytics_anonymize_ip"), "mixpanel_id": frappe.conf.get("mixpanel_id") }) diff --git a/frappe/www/website_script.js b/frappe/www/website_script.js index e31b6812d5..ce9c28e9d9 100644 --- a/frappe/www/website_script.js +++ b/frappe/www/website_script.js @@ -9,6 +9,9 @@ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); ga('create', '{{ google_analytics_id }}', 'auto'); +{% if google_analytics_anonymize_ip %} +ga('set', 'anonymizeIp', true); +{% endif %} ga('send', 'pageview'); // End Google Analytics {%- endif %} diff --git a/frappe/www/website_script.py b/frappe/www/website_script.py index 9d6ba1065e..0bb5a8a80b 100644 --- a/frappe/www/website_script.py +++ b/frappe/www/website_script.py @@ -18,5 +18,11 @@ def get_context(context): context.javascript += "\n" + js if not frappe.conf.developer_mode: - context["google_analytics_id"] = (frappe.db.get_single_value("Website Settings", "google_analytics_id") - or frappe.conf.get("google_analytics_id")) + context['google_analytics_id'] = get_setting('google_analytics_id') + context['google_analytics_anonymize_ip'] = get_setting('google_analytics_anonymize_ip') + +def get_setting(field_name): + """Return value of field_name frok Website Settings or Site Config.""" + website_settings = frappe.db.get_single_value('Website Settings', field_name) + conf = frappe.conf.get(field_name) + return website_settings or conf From 1849c4f2cf3715e7b3afabb823a066ebee845ba9 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Wed, 30 Sep 2020 11:24:08 +0200 Subject: [PATCH 03/43] fix: improve ux of standard web templates --- frappe/website/doctype/web_template/web_template.js | 12 +++++++++++- frappe/website/doctype/web_template/web_template.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_template/web_template.js b/frappe/website/doctype/web_template/web_template.js index 99dc02bfbb..1488dbf1be 100644 --- a/frappe/website/doctype/web_template/web_template.js +++ b/frappe/website/doctype/web_template/web_template.js @@ -2,11 +2,21 @@ // For license information, please see license.txt frappe.ui.form.on('Web Template', { - refresh(frm) { + refresh: function(frm) { if (!frappe.boot.developer_mode && frm.doc.standard) { frm.disable_form(); } frm.toggle_display('standard', frappe.boot.developer_mode); + frm.toggle_display('template', !frm.doc.standard); + }, + standard: function(frm) { + if (!frm.doc.standard) { + // If standard changes from true to false, hide template until + // the next save. Changes will get overwritten from the backend + // on save and should not be possible in the UI. + frm.toggle_display('template', false); + frm.save(); + } } }); diff --git a/frappe/website/doctype/web_template/web_template.py b/frappe/website/doctype/web_template/web_template.py index 1b6e67a9d0..811e4f5ad9 100644 --- a/frappe/website/doctype/web_template/web_template.py +++ b/frappe/website/doctype/web_template/web_template.py @@ -35,6 +35,7 @@ class WebTemplate(Document): if self.standard: export_to_files(record_list=[["Web Template", self.name]], create_init=True) self.create_template_file() + self.template = "" # standard to custom was_standard = (self.get_doc_before_save() or {}).get("standard") From 4f82ebcef717665265efc2230fc936dfa206ddaf Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Oct 2020 15:32:03 +0530 Subject: [PATCH 04/43] refactor: Fix excel export - Remove unnecessary code --- frappe/desk/query_report.py | 110 +++++++++++++----------------------- 1 file changed, 38 insertions(+), 72 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 5a9aae8435..95f93b612e 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -74,23 +74,25 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) res = report.execute_script_report(filters) columns, result, message, chart, report_summary, skip_total_row = ljust_list(res, 6) + columns = [get_column_as_dict(col) for col in columns] - if report.custom_columns: - # Original query columns, needed to reorder data as per custom columns - query_columns = columns - # Reordered columns - columns = json.loads(report.custom_columns) - - result = reorder_data_for_custom_columns(columns, query_columns, result) - - result = add_data_to_custom_columns(columns, result) + # convert to list of dicts + result = normalize_result(result, columns) if custom_columns: - result = add_data_to_custom_columns(custom_columns, result) - for custom_column in custom_columns: columns.insert(custom_column["insert_after_index"] + 1, custom_column) + custom_columns = custom_columns or [] + + if isinstance(report.custom_columns, string_types): + custom_columns += json.loads(report.custom_columns) or [] + else: + custom_columns += report.custom_columns or [] + + if custom_columns: + result = add_custom_column_data(custom_columns, result) + if result: result = get_filtered_data(report.ref_doctype, columns, result, user) @@ -109,6 +111,20 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) or 0, } +def normalize_result(result, columns): + # Converts to list of dicts from list of lists/tuples + data = [] + column_names = [column["fieldname"] for column in columns] + if result and isinstance(result[0], (list, tuple)): + for row in result: + row_obj = {} + for idx, column_name in enumerate(column_names): + row_obj[column_name] = row[idx] + data.append(row_obj) + else: + data = result + + return data @frappe.whitelist() def background_enqueue_run(report_name, filters=None, user=None): @@ -221,69 +237,17 @@ def run( return result -def add_data_to_custom_columns(columns, result): - custom_fields_data = get_data_for_custom_report(columns) +def add_custom_column_data(custom_columns, result): + custom_column_data = get_data_for_custom_report(custom_columns) - data = [] - for row in result: - row_obj = {} - if isinstance(row, tuple): - row = list(row) + for column in custom_columns: + key = (column.get('doctype'), column.get('fieldname')) + if key in custom_column_data: + for row in result: + row_reference = row[column.get('link_field')] + row[column.get('fieldname')] = custom_column_data.get(key).get(row_reference) - if isinstance(row, list): - for idx, column in enumerate(columns): - if column.get("link_field"): - row_obj[column["fieldname"]] = None - row.insert(idx, None) - else: - row_obj[column["fieldname"]] = row[idx] - data.append(row_obj) - else: - data.append(row) - - for row in data: - for column in columns: - if column.get("link_field"): - fieldname = column["fieldname"] - key = (column["doctype"], fieldname) - link_field = column["link_field"] - row[fieldname] = custom_fields_data.get(key, {}).get( - row.get(link_field) - ) - - return data - - -def reorder_data_for_custom_columns(custom_columns, columns, result): - if not result: - return [] - - columns = [get_column_as_dict(col) for col in columns] - if isinstance(result[0], list) or isinstance(result[0], tuple): - # If the result is a list of lists - custom_column_names = [col["label"] for col in custom_columns] - original_column_names = [col["label"] for col in columns] - return get_columns_from_list(custom_column_names, original_column_names, result) - else: - # columns do not need to be reordered if result is a list of dicts - return result - - -def get_columns_from_list(columns, target_columns, result): - reordered_result = [] - - for res in result: - r = [] - for col_name in columns: - try: - idx = target_columns.index(col_name) - r.append(res[idx]) - except ValueError: - pass - - reordered_result.append(r) - - return reordered_result + return result def get_prepared_report_result(report, filters, dn="", user=None): @@ -755,6 +719,8 @@ def get_column_as_dict(col): col_dict["fieldtype"], col_dict["options"] = col[1].split("/") else: col_dict["fieldtype"] = col[1] + if len(col) == 3: + col_dict["width"] = col[2] col_dict["label"] = col[0] col_dict["fieldname"] = frappe.scrub(col[0]) From f8afe32ea4e0cd610add5fef0a654f5a81fe0f49 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Oct 2020 22:20:59 +0530 Subject: [PATCH 05/43] fix: Scrub fieldname [wip] --- frappe/desk/query_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 95f93b612e..e8a03a1552 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -244,7 +244,7 @@ def add_custom_column_data(custom_columns, result): key = (column.get('doctype'), column.get('fieldname')) if key in custom_column_data: for row in result: - row_reference = row[column.get('link_field')] + row_reference = row[frappe.scrub(column.get('link_field'))] row[column.get('fieldname')] = custom_column_data.get(key).get(row_reference) return result From 51f5bb4811032efa8ec468218d5a9fcd54530c66 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Oct 2020 23:11:10 +0530 Subject: [PATCH 06/43] fix: Preserve column order --- frappe/desk/query_report.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index e8a03a1552..922c50b036 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -75,23 +75,25 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) columns, result, message, chart, report_summary, skip_total_row = ljust_list(res, 6) columns = [get_column_as_dict(col) for col in columns] + report_column_names = [col["fieldname"] for col in columns] # convert to list of dicts result = normalize_result(result, columns) + if report.custom_columns: + # saved columns (with custom columns / with different column order) + columns = json.loads(report.custom_columns) + + # unsaved custom_columns if custom_columns: for custom_column in custom_columns: columns.insert(custom_column["insert_after_index"] + 1, custom_column) - custom_columns = custom_columns or [] + # all columns which are not in original report + report_custom_columns = [column for column in columns if column["fieldname"] not in report_column_names] - if isinstance(report.custom_columns, string_types): - custom_columns += json.loads(report.custom_columns) or [] - else: - custom_columns += report.custom_columns or [] - - if custom_columns: - result = add_custom_column_data(custom_columns, result) + if report_custom_columns: + result = add_custom_column_data(report_custom_columns, result) if result: result = get_filtered_data(report.ref_doctype, columns, result, user) From 6274ec19a7424ab8f18760f6569047449ef46d4a Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Fri, 9 Oct 2020 12:51:20 +0530 Subject: [PATCH 07/43] fix: handle success and failure separately --- .../doctype/integration_request/integration_request.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index 33e87c4796..7668ce683a 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -23,7 +23,16 @@ class IntegrationRequest(Document): self.save(ignore_permissions=True) frappe.db.commit() + def handle_sucess(self, response): + """update the output field with the response along with the relevant status""" + self.process_response("output", response) + + def handle_failure(self, response): + """update the error field with the response along with the relevant status""" + self.process_response("error", response) + def process_response(self, ref_field, response): + """Update the response in integration request with status based on reference""" if isinstance(response, string_types): response = json.loads(response) From 50d2f6bb0816888ef4dd9c5103259125de84c9ae Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 10 Oct 2020 09:54:17 +0530 Subject: [PATCH 08/43] test: Use .get to avoid KeyError --- frappe/core/doctype/report/test_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 805b903300..78904afc5f 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -27,7 +27,7 @@ class TestReport(unittest.TestCase): columns, data = report.get_data(filters={'user': 'Administrator', 'doctype': 'DocType'}) self.assertEqual(columns[0].get('label'), 'Name') self.assertEqual(columns[1].get('label'), 'Module') - self.assertTrue('User' in [d[0] for d in data]) + self.assertTrue('User' in [d.get('name') for d in data]) def test_report_permissions(self): frappe.set_user('test@example.com') From e07b534fbbbe520320b35e72478856f596b9bc01 Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sat, 10 Oct 2020 12:58:55 +0700 Subject: [PATCH 09/43] refactor(email): Helpful info in Email Domain logging Making it easier for user and/or server sysadmin to diagnose what went wrong. --- .../doctype/email_domain/email_domain.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index 08583dc228..d6304c9275 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -17,6 +17,8 @@ class EmailDomain(Document): def validate(self): """Validate email id and check POP3/IMAP and SMTP connections is enabled.""" + logger = frappe.logger() + if self.email_id: validate_email_address(self.email_id, True) @@ -26,19 +28,25 @@ class EmailDomain(Document): if not frappe.local.flags.in_install and not frappe.local.flags.in_patch: try: if self.use_imap: + logger.info('Checking incoming IMAP email server {host}:{port} ssl={ssl}...'.format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl)) if self.use_ssl: test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self)) else: test = imaplib.IMAP4(self.email_server, port=get_port(self)) else: + logger.info('Checking incoming POP3 email server {host}:{port} ssl={ssl}...'.format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl)) if self.use_ssl: test = poplib.POP3_SSL(self.email_server, port=get_port(self)) else: test = poplib.POP3(self.email_server, port=get_port(self)) - except Exception: - frappe.throw(_("Incoming email account not correct")) + except Exception as e: + logger.warn('Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e) + frappe.throw(title=_("Incoming email account not correct"), + msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e)) finally: try: @@ -54,15 +62,21 @@ class EmailDomain(Document): if not self.get('smtp_port'): self.smtp_port = 465 + logger.info('Checking outgoing SMTPS email server {host}:{port}...'.format( + host=self.smtp_server, port=self.smtp_port)) sess = smtplib.SMTP_SSL((self.smtp_server or "").encode('utf-8'), cint(self.smtp_port) or None) else: if self.use_tls and not self.smtp_port: self.smtp_port = 587 + logger.info('Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...'.format( + host=self.smtp_server, port=self.get('smtp_port'), tls=self.use_tls)) sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None) sess.quit() - except Exception: - frappe.throw(_("Outgoing email account not correct")) + except Exception as e: + logger.warn('Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e) + frappe.throw(title=_("Outgoing email account not correct"), + msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e)) def on_update(self): """update all email accounts using this domain""" From 5dea5e227cf191dac63c7f914cf2ed339c1dc8e9 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 13 Oct 2020 16:08:59 +0530 Subject: [PATCH 10/43] test: Add tests to validate custom report data --- frappe/core/doctype/report/test_report.py | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 78904afc5f..2d2290e448 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import frappe, json, os import unittest +from frappe.desk.query_report import run, save_report, get_report_doc test_records = frappe.get_test_records('Report') test_dependencies = ['User'] @@ -29,6 +30,52 @@ class TestReport(unittest.TestCase): self.assertEqual(columns[1].get('label'), 'Module') self.assertTrue('User' in [d.get('name') for d in data]) + def test_custom_report(self): + custom_report_name = save_report( + 'Permitted Documents For User', + 'Permitted Documents For User Custom', + json.dumps([{ + 'fieldname': 'email', + 'fieldtype': 'Data', + 'label': 'Email', + 'insert_after_index': 0, + 'link_field': 'name', + 'doctype': 'User', + 'options': 'Email', + 'width': 100, + 'id':'email', + 'name': 'Email' + }])) + custom_report = frappe.get_doc('Report', custom_report_name) + columns, result = custom_report.run_query_report( + filters={ + 'user': 'Administrator', + 'doctype': 'User' + }, user=frappe.session.user) + + self.assertListEqual(['email'], [column.get('fieldname') for column in columns]) + self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, result[0]) + + def test_report_with_custom_column(self): + response = run('Permitted Documents For User', + filters={'user': 'Administrator', 'doctype': 'User'}, + custom_columns=[{ + 'fieldname': 'email', + 'fieldtype': 'Data', + 'label': 'Email', + 'insert_after_index': 0, + 'link_field': 'name', + 'doctype': 'User', + 'options': 'Email', + 'width': 100, + 'id':'email', + 'name': 'Email' + }]) + result = response.get('result') + columns = response.get('columns') + self.assertListEqual(['name', 'email', 'user_type'], [column.get('fieldname') for column in columns]) + self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, result[0]) + def test_report_permissions(self): frappe.set_user('test@example.com') frappe.db.sql("""delete from `tabHas Role` where parent = %s From ab368994b05c1071c0d7460c73af36406c9ae27f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 13 Oct 2020 16:09:29 +0530 Subject: [PATCH 11/43] fix: Use get to avoid key error --- frappe/desk/query_report.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 922c50b036..4e7cbd9604 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -195,14 +195,7 @@ def get_script(report_name): @frappe.whitelist() @frappe.read_only() -def run( - report_name, - filters=None, - user=None, - ignore_prepared_report=False, - custom_columns=None, -): - +def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None): report = get_report_doc(report_name) if not user: user = frappe.session.user @@ -246,7 +239,8 @@ def add_custom_column_data(custom_columns, result): key = (column.get('doctype'), column.get('fieldname')) if key in custom_column_data: for row in result: - row_reference = row[frappe.scrub(column.get('link_field'))] + row_reference = row.get(scrub(column.get('link_field'))) + if not row_reference: continue row[column.get('fieldname')] = custom_column_data.get(key).get(row_reference) return result From f7e45d1acddd76fac171ad7ee290663f47efc8ef Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Tue, 13 Oct 2020 15:28:06 +0200 Subject: [PATCH 12/43] fix: show headline instead of save --- frappe/website/doctype/web_template/web_template.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_template/web_template.js b/frappe/website/doctype/web_template/web_template.js index 1488dbf1be..def4e2dab1 100644 --- a/frappe/website/doctype/web_template/web_template.js +++ b/frappe/website/doctype/web_template/web_template.js @@ -16,7 +16,7 @@ frappe.ui.form.on('Web Template', { // the next save. Changes will get overwritten from the backend // on save and should not be possible in the UI. frm.toggle_display('template', false); - frm.save(); + frm.dashboard.set_headline(__('Please save to edit the template.')); } } }); From 4f3ec03fd6aee53aabcd2fed1df065517852208b Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Thu, 15 Oct 2020 19:01:40 +0530 Subject: [PATCH 13/43] fix: follow single responsibility principle --- .../integration_request/integration_request.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index 7668ce683a..f1d59beb5a 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -23,19 +23,16 @@ class IntegrationRequest(Document): self.save(ignore_permissions=True) frappe.db.commit() - def handle_sucess(self, response): + def handle_success(self, response): """update the output field with the response along with the relevant status""" - self.process_response("output", response) + if isinstance(response, string_types): + response = json.loads(response) + self.db_set("status", "Completed") + self.db_set("output", json.dumps(response, default=json_handler)) def handle_failure(self, response): """update the error field with the response along with the relevant status""" - self.process_response("error", response) - - def process_response(self, ref_field, response): - """Update the response in integration request with status based on reference""" if isinstance(response, string_types): response = json.loads(response) - - status = "Completed" if ref_field == "output" else "Failed" - self.db_set(ref_field, json.dumps(response, default=json_handler)) - self.db_set("status", status) \ No newline at end of file + self.db_set("status", "Failed") + self.db_set("error", json.dumps(response, default=json_handler)) \ No newline at end of file From c0494aac5abc57e9a6cf4fde23518f4169a98777 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Oct 2020 10:46:56 +0530 Subject: [PATCH 14/43] fix: Handle empty row --- frappe/desk/query_report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 4e7cbd9604..c577426be5 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -239,8 +239,10 @@ def add_custom_column_data(custom_columns, result): key = (column.get('doctype'), column.get('fieldname')) if key in custom_column_data: for row in result: - row_reference = row.get(scrub(column.get('link_field'))) - if not row_reference: continue + row_reference = row.get(column.get('link_field')) + # possible if the row is empty + if not row_reference: + continue row[column.get('fieldname')] = custom_column_data.get(key).get(row_reference) return result From 5173b21bdbb061b24c07bba4a35f26911e61aaa1 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Oct 2020 13:40:32 +0530 Subject: [PATCH 15/43] refactor: Make export_query code more readable --- frappe/desk/query_report.py | 60 +++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index c577426be5..dd318c8de6 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -12,6 +12,7 @@ from frappe.modules import scrub, get_module_path from frappe.utils import ( flt, cint, + cstr, get_html_format, get_url_to_form, gzip_decompress, @@ -305,31 +306,27 @@ def get_prepared_report_result(report, filters, dn="", user=None): @frappe.whitelist() def export_query(): """export from query reports""" - data = frappe._dict(frappe.local.form_dict) - - del data["cmd"] - if "csrf_token" in data: - del data["csrf_token"] + data.pop("cmd", None) + data.pop("csrf_token", None) if isinstance(data.get("filters"), string_types): filters = json.loads(data["filters"]) - if isinstance(data.get("report_name"), string_types): + + if data.get("report_name"): report_name = data["report_name"] frappe.permissions.can_export( frappe.get_cached_value("Report", report_name, "ref_doctype"), raise_exception=True, ) - if isinstance(data.get("file_format_type"), string_types): - file_format_type = data["file_format_type"] - custom_columns = frappe.parse_json(data["custom_columns"]) + file_format_type = data.get("file_format_type") + custom_columns = frappe.parse_json(data.get("custom_columns", "[]")) + include_indentation = data.get("include_indentation") + visible_idx = data.get("visible_idx") - include_indentation = data["include_indentation"] - if isinstance(data.get("visible_idx"), string_types): - visible_idx = json.loads(data.get("visible_idx")) - else: - visible_idx = None + if isinstance(visible_idx, string_types): + visible_idx = json.loads(visible_idx) if file_format_type == "Excel": data = run(report_name, filters, custom_columns=custom_columns) @@ -384,28 +381,27 @@ def handle_duration_fieldtype_values(result, columns): def build_xlsx_data(columns, data, visible_idx, include_indentation): result = [[]] - # add column headings - for idx in range(len(data.columns)): - if not columns[idx].get("hidden"): - result[0].append(columns[idx]["label"]) + for column in data.columns: + if column.get("hidden"): + continue + result[0].append(column["label"]) # build table from result - for i, row in enumerate(data.result): + for row_idx, row in enumerate(data.result): # only pick up rows that are visible in the report - if i in visible_idx: + if row_idx in visible_idx: row_data = [] - - if isinstance(row, dict) and row: - for idx in range(len(data.columns)): - # check if column is not hidden - if not columns[idx].get("hidden"): - label = columns[idx]["label"] - fieldname = columns[idx]["fieldname"] - cell_value = row.get(fieldname, row.get(label, "")) - if cint(include_indentation) and "indent" in row and idx == 0: - cell_value = (" " * cint(row["indent"])) + cell_value - row_data.append(cell_value) - else: + if isinstance(row, dict): + for col_idx, column in enumerate(data.columns): + if column.get("hidden"): + continue + label = column.get("label") + fieldname = column.get("fieldname") + cell_value = row.get(fieldname, row.get(label, "")) + if cint(include_indentation) and "indent" in row and col_idx == 0: + cell_value = (" " * cint(row["indent"])) + cstr(cell_value) + row_data.append(cell_value) + elif row: row_data = row result.append(row_data) From f1cb8f4e5ceedbad9d14481f502e98096fe376d1 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Oct 2020 13:42:00 +0530 Subject: [PATCH 16/43] feat: Pass custom widths for excel columns --- frappe/desk/query_report.py | 11 ++++++++--- frappe/utils/xlsxutils.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index dd318c8de6..b9be0ad683 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -345,8 +345,8 @@ def export_query(): data["result"] = handle_duration_fieldtype_values( data.get("result"), data.get("columns") ) - xlsx_data = build_xlsx_data(columns, data, visible_idx, include_indentation) - xlsx_file = make_xlsx(xlsx_data, "Query Report") + xlsx_data, column_widths = build_xlsx_data(columns, data, visible_idx, include_indentation) + xlsx_file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths) frappe.response["filename"] = report_name + ".xlsx" frappe.response["filecontent"] = xlsx_file.getvalue() @@ -380,11 +380,16 @@ def handle_duration_fieldtype_values(result, columns): def build_xlsx_data(columns, data, visible_idx, include_indentation): result = [[]] + column_widths = [] for column in data.columns: if column.get("hidden"): continue result[0].append(column["label"]) + column_width = column.get('width', 0) + # to convert into scale accepted by openpyxl + column_width /= 10 + column_widths.append(column_width) # build table from result for row_idx, row in enumerate(data.result): @@ -406,7 +411,7 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation): result.append(row_data) - return result + return result, column_widths def add_total_row(result, columns, meta=None): diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 2814c5ff40..3c7b027470 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -9,19 +9,24 @@ import xlrd import re from openpyxl.styles import Font from openpyxl import load_workbook +from openpyxl.utils import get_column_letter from six import BytesIO, string_types ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]') # return xlsx file object -def make_xlsx(data, sheet_name, wb=None): - +def make_xlsx(data, sheet_name, wb=None, column_widths=None): + column_widths = column_widths or [] if wb is None: wb = openpyxl.Workbook(write_only=True) ws = wb.create_sheet(sheet_name, 0) + for i, column_width in enumerate(column_widths): + if column_width: + ws.column_dimensions[get_column_letter(i + 1)].width = column_width + row1 = ws.row_dimensions[1] - row1.font = Font(name='Calibri',bold=True) + row1.font = Font(name='Calibri', bold=True) for row in data: clean_row = [] From 44b1336a885483ed6e535b72080ffdc02e870f9b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Oct 2020 13:43:35 +0530 Subject: [PATCH 17/43] style: Remove unused import --- frappe/core/doctype/report/test_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 2d2290e448..fc47e3a8a8 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import frappe, json, os import unittest -from frappe.desk.query_report import run, save_report, get_report_doc +from frappe.desk.query_report import run, save_report test_records = frappe.get_test_records('Report') test_dependencies = ['User'] From b726a8769a665a20ae03a18549b30d805133d841 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 18 Oct 2020 11:20:04 +0530 Subject: [PATCH 18/43] test: Fix query report test --- frappe/tests/test_query_report.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_query_report.py b/frappe/tests/test_query_report.py index 8e62e03064..eaf4dc7070 100644 --- a/frappe/tests/test_query_report.py +++ b/frappe/tests/test_query_report.py @@ -23,7 +23,11 @@ class TestQueryReport(unittest.TestCase): # Create mock data data = frappe._dict() - data.columns = ["column_a", "column_b", "column_c"] + data.columns = [ + {"label": "Column A", "fieldname": "column_a"}, + {"label": "Column B", "fieldname": "column_b", "width": 150}, + {"label": "Column C", "fieldname": "column_c", "width": 100} + ] data.result = [ [1.0, 3.0, 5.5], {"column_a": 22.1, "column_b": 21.8, "column_c": 30.2}, @@ -35,10 +39,12 @@ class TestQueryReport(unittest.TestCase): visible_idx = [0, 2, 3] # Build the result - xlsx_data = build_xlsx_data(columns, data, visible_idx, include_indentation=0) + xlsx_data, column_widths = build_xlsx_data(columns, data, visible_idx, include_indentation=0) self.assertEqual(type(xlsx_data), list) self.assertEqual(len(xlsx_data), 4) # columns + data + # column widths are divided by 10 to match the scale that is supported by openpyxl + self.assertListEqual(column_widths, [0, 15, 10]) for row in xlsx_data: self.assertEqual(type(row), list) From 9e4f079877ac32b1ee01bd0b1affac4fa55dd77f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 18 Oct 2020 11:21:47 +0530 Subject: [PATCH 19/43] chore: Disable test verbosity --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2331217363..63895675ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --verbose --site test_site run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --verbose --site test_site run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Cypress" python: 3.7 From 57b493efb7f7bd33b06283520e476e8196f59293 Mon Sep 17 00:00:00 2001 From: Anupam Date: Mon, 19 Oct 2020 17:45:29 +0530 Subject: [PATCH 20/43] fix: deletion of standard report --- frappe/core/doctype/report/report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 8b7a03aa28..7e6b1d69ac 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -49,6 +49,8 @@ class Report(Document): self.export_doc() def on_trash(self): + if self.is_standard == 'Yes' and not cint(getattr(frappe.local.conf, 'developer_mode',0)): + frappe.throw(_("Can't Delete Standard Report")) delete_custom_role('report', self.name) def get_columns(self): From 89f212d4956c0f70d5d220d7fbd58c009f61e250 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Oct 2020 10:41:39 +0530 Subject: [PATCH 21/43] test: Clear any customizations on user doctype --- frappe/core/doctype/report/test_report.py | 5 ++++- .../custom/doctype/customize_form/customize_form.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index fc47e3a8a8..69e52e9be3 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import frappe, json, os import unittest from frappe.desk.query_report import run, save_report +from frappe.custom.doctype.customize_form.customize_form import reset_customization test_records = frappe.get_test_records('Report') test_dependencies = ['User'] @@ -57,6 +58,7 @@ class TestReport(unittest.TestCase): self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, result[0]) def test_report_with_custom_column(self): + reset_customization('User') response = run('Permitted Documents For User', filters={'user': 'Administrator', 'doctype': 'User'}, custom_columns=[{ @@ -74,7 +76,8 @@ class TestReport(unittest.TestCase): result = response.get('result') columns = response.get('columns') self.assertListEqual(['name', 'email', 'user_type'], [column.get('fieldname') for column in columns]) - self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, result[0]) + admin_dict = frappe.core.utils.find(result, lambda d: d['name'] == 'Administrator') + self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_permissions(self): frappe.set_user('test@example.com') diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index d4eeba3f93..c8c3182729 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -425,8 +425,13 @@ class CustomizeForm(Document): if not self.doc_type: return - frappe.db.sql("""DELETE FROM `tabProperty Setter` WHERE doc_type=%s - and `field_name`!='naming_series' - and `property`!='options'""", self.doc_type) - frappe.clear_cache(doctype=self.doc_type) + reset_customization(self.doctype) self.fetch_to_customize() + +def reset_customization(doctype): + frappe.db.sql(""" + DELETE FROM `tabProperty Setter` WHERE doc_type=%s + and `field_name`!='naming_series' + and `property`!='options' + """, doctype) + frappe.clear_cache(doctype=doctype) \ No newline at end of file From 6292b38692fcf9a17f97aaec185a5298ccd94d09 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Oct 2020 11:17:41 +0530 Subject: [PATCH 22/43] test: Clear user customization before test --- frappe/core/doctype/report/test_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 69e52e9be3..d76a1470e4 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -32,6 +32,7 @@ class TestReport(unittest.TestCase): self.assertTrue('User' in [d.get('name') for d in data]) def test_custom_report(self): + reset_customization('User') custom_report_name = save_report( 'Permitted Documents For User', 'Permitted Documents For User Custom', @@ -55,7 +56,8 @@ class TestReport(unittest.TestCase): }, user=frappe.session.user) self.assertListEqual(['email'], [column.get('fieldname') for column in columns]) - self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, result[0]) + admin_dict = frappe.core.utils.find(result, lambda d: d['name'] == 'Administrator') + self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_with_custom_column(self): reset_customization('User') From 5b97d66778b2a4026c3f29f3eb3f4ab6c2e8da30 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Oct 2020 11:20:56 +0530 Subject: [PATCH 23/43] fix: Typo --- frappe/custom/doctype/customize_form/customize_form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index c8c3182729..5c4e16fad7 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -425,7 +425,7 @@ class CustomizeForm(Document): if not self.doc_type: return - reset_customization(self.doctype) + reset_customization(self.doc_type) self.fetch_to_customize() def reset_customization(doctype): From 09dcf5ac6ebfa0f1b84d751644d1574d1851535c Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 20 Oct 2020 11:25:08 +0530 Subject: [PATCH 24/43] fix: Link Preview not working for Link fields with indicator formatters (#11729) --- frappe/public/js/frappe/form/form.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 9e6d3f0bdb..04b26875d1 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1517,11 +1517,12 @@ frappe.ui.form.Form = class FrappeForm { const escaped_name = encodeURIComponent(value); - return repl('%(label)s', { + return repl('%(label)s', { color: get_color(doc || {}), doctype: df.options, - name: escaped_name, - label: label + escaped_name: escaped_name, + label: label, + name: value }); } else { return ''; From 117d28ba3a59f121cc530037c8a3d0860731b196 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 13 Oct 2020 11:19:44 +0530 Subject: [PATCH 25/43] Enahancement: 1. Log Settings doctype to control log cleanup 2. Provision to show an alert message if unseen error log exists --- .../core/doctype/activity_log/activity_log.py | 10 ++- frappe/core/doctype/error_log/error_log.py | 3 - .../core/doctype/log_setting_user/__init__.py | 0 .../log_setting_user/log_setting_user.js | 8 ++ .../log_setting_user/log_setting_user.json | 34 ++++++++ .../log_setting_user/log_setting_user.py | 10 +++ .../log_setting_user/test_log_setting_user.py | 10 +++ frappe/core/doctype/log_settings/__init__.py | 0 .../core/doctype/log_settings/log_settings.js | 8 ++ .../doctype/log_settings/log_settings.json | 83 +++++++++++++++++++ .../core/doctype/log_settings/log_settings.py | 32 +++++++ .../doctype/log_settings/test_log_settings.py | 10 +++ frappe/hooks.py | 7 +- 13 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 frappe/core/doctype/log_setting_user/__init__.py create mode 100644 frappe/core/doctype/log_setting_user/log_setting_user.js create mode 100644 frappe/core/doctype/log_setting_user/log_setting_user.json create mode 100644 frappe/core/doctype/log_setting_user/log_setting_user.py create mode 100644 frappe/core/doctype/log_setting_user/test_log_setting_user.py create mode 100644 frappe/core/doctype/log_settings/__init__.py create mode 100644 frappe/core/doctype/log_settings/log_settings.js create mode 100644 frappe/core/doctype/log_settings/log_settings.json create mode 100644 frappe/core/doctype/log_settings/log_settings.py create mode 100644 frappe/core/doctype/log_settings/test_log_settings.py diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 27a2892ca8..98dc91806d 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -40,7 +40,11 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "operation": operation, }).insert(ignore_permissions=True, ignore_links=True) -def clear_authentication_logs(): - """clear 100 day old authentication logs""" +def clear_activity_logs(days=None): + """clear 90 day old authentication logs or configured in log settings""" + + if not days: + days = 90 + frappe.db.sql("""delete from `tabActivity Log` where \ - creation< (NOW() - INTERVAL '100' DAY)""") \ No newline at end of file + creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 6cc8275404..ec02aaf446 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -17,9 +17,6 @@ def set_old_logs_as_seen(): frappe.db.sql("""UPDATE `tabError Log` SET `seen`=1 WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""") - # clear old logs - frappe.db.sql("""DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '30' DAY)""") - @frappe.whitelist() def clear_error_logs(): '''Flush all Error Logs''' diff --git a/frappe/core/doctype/log_setting_user/__init__.py b/frappe/core/doctype/log_setting_user/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.js b/frappe/core/doctype/log_setting_user/log_setting_user.js new file mode 100644 index 0000000000..a1eb824e22 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Log Setting User', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.json b/frappe/core/doctype/log_setting_user/log_setting_user.json new file mode 100644 index 0000000000..7f4b0ef874 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.json @@ -0,0 +1,34 @@ +{ + "actions": [], + "autoname": "field:user", + "creation": "2020-10-08 13:09:36.034430", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user" + ], + "fields": [ + { + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User", + "reqd": 1, + "unique": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2020-10-08 17:22:04.690348", + "modified_by": "Administrator", + "module": "Core", + "name": "Log Setting User", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.py b/frappe/core/doctype/log_setting_user/log_setting_user.py new file mode 100644 index 0000000000..df6d55f0a9 --- /dev/null +++ b/frappe/core/doctype/log_setting_user/log_setting_user.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class LogSettingUser(Document): + pass diff --git a/frappe/core/doctype/log_setting_user/test_log_setting_user.py b/frappe/core/doctype/log_setting_user/test_log_setting_user.py new file mode 100644 index 0000000000..507c02d87d --- /dev/null +++ b/frappe/core/doctype/log_setting_user/test_log_setting_user.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestLogSettingUser(unittest.TestCase): + pass diff --git a/frappe/core/doctype/log_settings/__init__.py b/frappe/core/doctype/log_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js new file mode 100644 index 0000000000..09a2086a1d --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Log Settings', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json new file mode 100644 index 0000000000..d6fb4ada49 --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -0,0 +1,83 @@ +{ + "actions": [], + "creation": "2020-10-08 12:12:21.694424", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "error_log_notification_section", + "users_to_notify", + "log_cleanup_section", + "error_logs", + "activity_logs", + "column_break_4", + "email_queue" + ], + "fields": [ + { + "fieldname": "log_cleanup_section", + "fieldtype": "Section Break", + "label": "Log Cleanup" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "error_logs", + "fieldtype": "Int", + "label": "Error logs" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "activity_logs", + "fieldtype": "Int", + "label": "Activity Logs" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "email_queue", + "fieldtype": "Int", + "label": "Email Queue" + }, + { + "fieldname": "error_log_notification_section", + "fieldtype": "Section Break", + "label": "Error Log Notification" + }, + { + "fieldname": "users_to_notify", + "fieldtype": "Table MultiSelect", + "label": "Users To Notify", + "options": "Log Setting User" + } + ], + "index_web_pages_for_search": 1, + "issingle": 1, + "links": [], + "modified": "2020-10-13 10:56:41.793424", + "modified_by": "Administrator", + "module": "Core", + "name": "Log Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py new file mode 100644 index 0000000000..4b4825460a --- /dev/null +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe import _ +from frappe.model.document import Document +from frappe.core.doctype.activity_log.activity_log import clear_activity_logs + +class LogSettings(Document): + def clear_logs(self): + self.clear_error_logs() + self.clear_activity_logs() + + def clear_error_logs(self): + frappe.db.sql(""" DELETE FROM `tabError Log` + WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + """.format(self.error_log)) + + def clear_activity_logs(self): + clear_activity_logs(days=self.activity_log) + +def run_log_clean_up(): + doc = frappe.get_doc("DocType", "Log Settings") + doc.clear_logs() + +def show_error_log_reminder(): + print('here') + if frappe.db.count("Error Log", filters={'seen': 0}) > 0: + print("count") + frappe.msgprint(_("You have unseen {0}").format(' Error Log '), alert=1, indicator='red') \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py new file mode 100644 index 0000000000..2824c71c88 --- /dev/null +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestLogSettings(unittest.TestCase): + pass diff --git a/frappe/hooks.py b/frappe/hooks.py index c68c69a3bb..3d1fca564e 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -183,7 +183,8 @@ scheduler_events = { "0/15 * * * *": [ "frappe.oauth.delete_oauth2_data", "frappe.website.doctype.web_page.web_page.check_publish_status", - "frappe.twofactor.delete_all_barcodes_for_users" + "frappe.twofactor.delete_all_barcodes_for_users", + "frappe.core.doctype.log_settings.log_settings.show_error_log_reminder" ] }, "all": [ @@ -215,7 +216,6 @@ scheduler_events = { "frappe.realtime.remove_old_task_logs", "frappe.utils.scheduler.restrict_scheduler_events_if_dormant", "frappe.email.doctype.auto_email_report.auto_email_report.send_daily", - "frappe.core.doctype.activity_log.activity_log.clear_authentication_logs", "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record", "frappe.desk.form.document_follow.send_daily_updates", "frappe.social.doctype.energy_point_settings.energy_point_settings.allocate_review_points", @@ -223,7 +223,8 @@ scheduler_events = { "frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry", "frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed", "frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails", - "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports" + "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports", + "frappe.core.doctype.log_settings.log_settings.run_log_clean_up" ], "daily_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily", From ce97e7275ecbb53da05f41f9aa9a09a80915c611 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 13 Oct 2020 11:47:06 +0530 Subject: [PATCH 26/43] feat: provision to setup users for Erro Log notification on log settings --- .../core/doctype/log_settings/log_settings.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 4b4825460a..73d5a1feb6 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -26,7 +26,22 @@ def run_log_clean_up(): doc.clear_logs() def show_error_log_reminder(): - print('here') + users_to_notify = get_users_to_notify() + if frappe.db.count("Error Log", filters={'seen': 0}) > 0: - print("count") - frappe.msgprint(_("You have unseen {0}").format(' Error Log '), alert=1, indicator='red') \ No newline at end of file + for user in users_to_notify: + frappe.publish_realtime('msgprint', { + "message": _("You have unseen {0}").format(' Error Log '), + "alert":1, + "indicator" :"red" + }, user=user) + +def get_users_to_notify(): + from frappe.email import get_system_managers + log_settings = frappe.get_doc('Log Settings') + + if log_settings.users_to_notify: + return [u.user for u in log_settings.users_to_notify] + else: + return get_system_managers() + From 07e4885775b701e3ab70c4cea39fc3df6e78556a Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 13 Oct 2020 12:27:18 +0530 Subject: [PATCH 27/43] fix: renaming of fields and test fixes --- frappe/config/settings.py | 5 ++ .../doctype/log_settings/log_settings.json | 50 +++++++++---------- .../core/doctype/log_settings/log_settings.py | 13 +++-- frappe/email/queue.py | 14 ++++-- frappe/hooks.py | 2 +- 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/frappe/config/settings.py b/frappe/config/settings.py index e43abd9fcb..0112c7ccff 100644 --- a/frappe/config/settings.py +++ b/frappe/config/settings.py @@ -23,6 +23,11 @@ def get_data(): "description": _("Company, Fiscal Year and Currency defaults"), "hide_count": True }, + { + "type": "doctype", + "name": "Log Settings", + "description": _("Log cleanup and notification configuration") + }, { "type": "doctype", "name": "Error Log", diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json index d6fb4ada49..8a2596b35c 100644 --- a/frappe/core/doctype/log_settings/log_settings.json +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -8,10 +8,10 @@ "error_log_notification_section", "users_to_notify", "log_cleanup_section", - "error_logs", - "activity_logs", + "clear_error_log_after", + "clear_activity_log_after", "column_break_4", - "email_queue" + "clear_email_queue_after" ], "fields": [ { @@ -19,31 +19,10 @@ "fieldtype": "Section Break", "label": "Log Cleanup" }, - { - "default": "90", - "description": "In Days", - "fieldname": "error_logs", - "fieldtype": "Int", - "label": "Error logs" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "activity_logs", - "fieldtype": "Int", - "label": "Activity Logs" - }, { "fieldname": "column_break_4", "fieldtype": "Column Break" }, - { - "default": "90", - "description": "In Days", - "fieldname": "email_queue", - "fieldtype": "Int", - "label": "Email Queue" - }, { "fieldname": "error_log_notification_section", "fieldtype": "Section Break", @@ -54,12 +33,33 @@ "fieldtype": "Table MultiSelect", "label": "Users To Notify", "options": "Log Setting User" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_error_log_after", + "fieldtype": "Int", + "label": "Clear Error log After" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_activity_log_after", + "fieldtype": "Int", + "label": "Clear Activity Log After" + }, + { + "default": "90", + "description": "In Days", + "fieldname": "clear_email_queue_after", + "fieldtype": "Int", + "label": "Clear Email Queue After" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-10-13 10:56:41.793424", + "modified": "2020-10-13 12:18:48.649038", "modified_by": "Administrator", "module": "Core", "name": "Log Settings", diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 73d5a1feb6..e9d9d1f1d7 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -6,23 +6,28 @@ from __future__ import unicode_literals import frappe from frappe import _ from frappe.model.document import Document -from frappe.core.doctype.activity_log.activity_log import clear_activity_logs class LogSettings(Document): def clear_logs(self): self.clear_error_logs() self.clear_activity_logs() + self.clear_email_queue() def clear_error_logs(self): frappe.db.sql(""" DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) - """.format(self.error_log)) + """.format(self.clear_error_log_after)) def clear_activity_logs(self): - clear_activity_logs(days=self.activity_log) + from frappe.core.doctype.activity_log.activity_log import clear_activity_logs + clear_activity_logs(days=self.clear_activity_log_after) + + def clear_email_queue(self): + from frappe.email.queue import clear_outbox + clear_outbox(days=self.clear_email_queue_after) def run_log_clean_up(): - doc = frappe.get_doc("DocType", "Log Settings") + doc = frappe.get_doc("Log Settings") doc.clear_logs() def show_error_log_reminder(): diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 25b6f93f83..f780aebdc1 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -584,14 +584,15 @@ def prepare_message(email, recipient, recipients_list): return safe_encode(message.as_string()) -def clear_outbox(): - """Remove low priority older than 31 days in Outbox and expire mails not sent for 7 days. - Called daily via scheduler. +def clear_outbox(days=None): + """Remove low priority older than 31 days in Outbox or configured in Log Settings. Note: Used separate query to avoid deadlock """ + if not days: + days=31 email_queues = frappe.db.sql_list("""SELECT `name` FROM `tabEmail Queue` - WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '31' DAY)""") + WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( @@ -602,6 +603,11 @@ def clear_outbox(): ','.join(['%s']*len(email_queues) )), tuple(email_queues)) +def set_expiry_for_email_queue(): + ''' Mark emails as expire that has not sent for 7 days. + Called daily via scheduler. + ''' + frappe.db.sql(""" UPDATE `tabEmail Queue` SET `status`='Expired' diff --git a/frappe/hooks.py b/frappe/hooks.py index 3d1fca564e..a952b1d1d0 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -207,7 +207,7 @@ scheduler_events = { "frappe.utils.password.delete_password_reset_cache" ], "daily": [ - "frappe.email.queue.clear_outbox", + "frappe.email.queue.set_expiry_for_email_queue", "frappe.desk.notifications.clear_notifications", "frappe.core.doctype.error_log.error_log.set_old_logs_as_seen", "frappe.desk.doctype.event.event.send_event_digest", From 697a9e0cf2c179c1a5aa9bbe6196582d42e452fa Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 13 Oct 2020 14:12:14 +0530 Subject: [PATCH 28/43] fix: run log notifier from client side --- .../core/doctype/log_settings/log_settings.py | 33 +++++++++---------- frappe/hooks.py | 3 +- frappe/public/js/frappe/desk.js | 20 +++++++++++ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index e9d9d1f1d7..6d59cdeb29 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -30,23 +30,22 @@ def run_log_clean_up(): doc = frappe.get_doc("Log Settings") doc.clear_logs() -def show_error_log_reminder(): - users_to_notify = get_users_to_notify() +@frappe.whitelist() +def has_unseen_error_log(user): - if frappe.db.count("Error Log", filters={'seen': 0}) > 0: - for user in users_to_notify: - frappe.publish_realtime('msgprint', { - "message": _("You have unseen {0}").format(' Error Log '), - "alert":1, - "indicator" :"red" - }, user=user) + def _get_response(show_alert=True): + return { + 'show_alert': True, + 'message': _("You have unseen {0}").format(' Error Logs ') + } -def get_users_to_notify(): - from frappe.email import get_system_managers - log_settings = frappe.get_doc('Log Settings') - - if log_settings.users_to_notify: - return [u.user for u in log_settings.users_to_notify] - else: - return get_system_managers() + if frappe.db.sql_list("select name from `tabError Log` where seen = 0 limit 1"): + log_settings = frappe.get_cached_doc('Log Settings') + if log_settings.users_to_notify: + if user in [u.user for u in log_settings.users_to_notify]: + return _get_response() + else: + return _get_response(show_alert=False) + else: + return _get_response() \ No newline at end of file diff --git a/frappe/hooks.py b/frappe/hooks.py index a952b1d1d0..ff90cf0af6 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -183,8 +183,7 @@ scheduler_events = { "0/15 * * * *": [ "frappe.oauth.delete_oauth2_data", "frappe.website.doctype.web_page.web_page.check_publish_status", - "frappe.twofactor.delete_all_barcodes_for_users", - "frappe.core.doctype.log_settings.log_settings.show_error_log_reminder" + "frappe.twofactor.delete_all_barcodes_for_users" ] }, "all": [ diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 97d5239a72..99c1647455 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -139,6 +139,26 @@ frappe.Application = Class.extend({ } }); }, 300000); // check every 5 minutes + + if(frappe.user.has_role("System Manager")){ + setInterval(function() { + frappe.call({ + method: 'frappe.core.doctype.log_settings.log_settings.has_unseen_error_log', + args: { + user: frappe.session.user + }, + callback: function(r) { + console.log(r); + if(r.message.show_alert){ + frappe.show_alert({ + indicator: 'red', + message: r.message.message + }); + } + } + }); + }, 600000); // check every 10 minutes + } } this.fetch_tags(); From 94112b4f37899e6d3ee9836080e38ea033b54b08 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 20 Oct 2020 13:54:54 +0530 Subject: [PATCH 29/43] fix: Warning message --- frappe/core/doctype/report/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 7e6b1d69ac..8634ad1084 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -50,7 +50,7 @@ class Report(Document): def on_trash(self): if self.is_standard == 'Yes' and not cint(getattr(frappe.local.conf, 'developer_mode',0)): - frappe.throw(_("Can't Delete Standard Report")) + frappe.throw(_("You are not allowed to delete Standard Report")) delete_custom_role('report', self.name) def get_columns(self): From 7d57d8030b332eee6cbb3b5767fbb1bd80f540ae Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 20 Oct 2020 15:07:10 +0530 Subject: [PATCH 30/43] fix: minor changes Clear headline to avoid duplicate headlines. Dont show headline for new forms --- frappe/website/doctype/web_template/web_template.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/doctype/web_template/web_template.js b/frappe/website/doctype/web_template/web_template.js index def4e2dab1..df5b3cb2c1 100644 --- a/frappe/website/doctype/web_template/web_template.js +++ b/frappe/website/doctype/web_template/web_template.js @@ -8,14 +8,14 @@ frappe.ui.form.on('Web Template', { } frm.toggle_display('standard', frappe.boot.developer_mode); - frm.toggle_display('template', !frm.doc.standard); }, standard: function(frm) { - if (!frm.doc.standard) { + if (!frm.doc.standard && !frm.is_new()) { // If standard changes from true to false, hide template until // the next save. Changes will get overwritten from the backend // on save and should not be possible in the UI. frm.toggle_display('template', false); + frm.dashboard.clear_headline(); frm.dashboard.set_headline(__('Please save to edit the template.')); } } From b86e1e905d646e3eb5d8fb8e8f6a55d06655eb7b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 20 Oct 2020 15:09:40 +0530 Subject: [PATCH 31/43] fix: Add mandatory depends on and remove python validation --- frappe/website/doctype/web_template/web_template.json | 3 ++- frappe/website/doctype/web_template/web_template.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/website/doctype/web_template/web_template.json b/frappe/website/doctype/web_template/web_template.json index fed7008cdb..d50266b7b1 100644 --- a/frappe/website/doctype/web_template/web_template.json +++ b/frappe/website/doctype/web_template/web_template.json @@ -48,6 +48,7 @@ "fieldname": "module", "fieldtype": "Link", "label": "Module", + "mandatory_depends_on": "standard", "options": "Module Def" } ], @@ -58,7 +59,7 @@ "link_fieldname": "web_template" } ], - "modified": "2020-09-25 00:48:57.902292", + "modified": "2020-10-20 15:08:57.027962", "modified_by": "Administrator", "module": "Website", "name": "Web Template", diff --git a/frappe/website/doctype/web_template/web_template.py b/frappe/website/doctype/web_template/web_template.py index 811e4f5ad9..30db38eb74 100644 --- a/frappe/website/doctype/web_template/web_template.py +++ b/frappe/website/doctype/web_template/web_template.py @@ -26,9 +26,6 @@ class WebTemplate(Document): if not field.fieldname: field.fieldname = frappe.scrub(field.label) - if self.standard and not self.module: - frappe.throw(_("Please select which module this Web Template belongs to.")) - def on_update(self): if frappe.conf.developer_mode: # custom to standard From 8a442f4185c87dedc59be6612b48b60e6c1cb63d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 20 Oct 2020 15:34:44 +0530 Subject: [PATCH 32/43] feat: set fieldname before insert (#11675) --- frappe/custom/doctype/custom_field/custom_field.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index bc325b654e..ee6e3b9c61 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -32,6 +32,7 @@ class CustomField(Document): self.fieldname = self.fieldname.lower() def before_insert(self): + self.set_fieldname() meta = frappe.get_meta(self.dt, cached=False) fieldnames = [df.fieldname for df in meta.get("fields")] From af5f535d1fd1a73529c8c24232941f8b3f79076b Mon Sep 17 00:00:00 2001 From: Saurabh Date: Tue, 20 Oct 2020 13:22:18 +0530 Subject: [PATCH 33/43] fix: test case --- .../doctype/scheduled_job_type/test_scheduled_job_type.py | 4 ++-- frappe/tests/test_email.py | 6 ++++-- frappe/tests/test_scheduler.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index e7db6f9045..d0a65defa4 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -22,7 +22,7 @@ class TestScheduledJobType(unittest.TestCase): self.assertEqual(all_job.frequency, 'All') daily_job = frappe.get_doc('Scheduled Job Type', - dict(method='frappe.email.queue.clear_outbox')) + dict(method='frappe.email.queue.set_expiry_for_email_queue')) self.assertEqual(daily_job.frequency, 'Daily') # check if cron jobs are synced @@ -38,7 +38,7 @@ class TestScheduledJobType(unittest.TestCase): self.assertEqual(updated_scheduled_job.frequency, "Hourly") def test_daily_job(self): - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox')) + job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.set_expiry_for_email_queue')) job.db_set('last_execution', '2019-01-01 00:00:00') self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06'))) self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06'))) diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 0efcb47948..ff7e6d534c 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -130,8 +130,10 @@ class TestEmail(unittest.TestCase): def test_expired(self): self.test_email_queue() frappe.db.sql("UPDATE `tabEmail Queue` SET `modified`=(NOW() - INTERVAL '8' day)") - from frappe.email.queue import clear_outbox - clear_outbox() + + from frappe.email.queue import set_expiry_for_email_queue + set_expiry_for_email_queue() + email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Expired'""", as_dict=1) self.assertEqual(len(email_queue), 1) queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index 30818e6f17..df3d735488 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -31,7 +31,7 @@ class TestScheduler(TestCase): enqueue_events(site = frappe.local.site) frappe.flags.execute_job = False - self.assertTrue('frappe.email.queue.clear_outbox', frappe.flags.enqueued_jobs) + self.assertTrue('frappe.email.queue.set_expiry_for_email_queue', frappe.flags.enqueued_jobs) self.assertTrue('frappe.utils.change_log.check_for_update', frappe.flags.enqueued_jobs) self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs) From 2f890c1391facac992421c778dbdf83af6626696 Mon Sep 17 00:00:00 2001 From: prssanna Date: Wed, 16 Sep 2020 22:48:12 +0530 Subject: [PATCH 34/43] chore: update datatable to 1.15.3 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index da5fcaf081..8b6959403d 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "express": "^4.17.1", "fast-deep-equal": "^2.0.1", "frappe-charts": "^1.5.1", - "frappe-datatable": "^1.15.1", + "frappe-datatable": "^1.15.3", "frappe-gantt": "^0.5.0", "fuse.js": "^3.4.6", "highlight.js": "^9.18.1", diff --git a/yarn.lock b/yarn.lock index d2f918a049..82a9b7dabd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2251,10 +2251,10 @@ frappe-charts@^1.5.1: resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-1.5.1.tgz#77b9e61400b1657d4ca2eb2202053e3a4d18d54b" integrity sha512-Cvj6IyDkiH6LKw558A8syJUmkQSdNVnfC+WAzDaAtOfs+u2nST6HExA6JUZMaHU4+VJhC2PWwyRjRNw3B5FaUQ== -frappe-datatable@^1.15.1: - version "1.15.1" - resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.15.1.tgz#3895f6c42158c85a40a82ebd268b6427765facc2" - integrity sha512-bMWJnHwCjwLWSWlZswW66wPUF3oIz7sHeRf2I5rXUd/2m6HqfAAaali0qgDDiO/6VeZBDNttCbovLitjl0ouTA== +frappe-datatable@^1.15.3: + version "1.15.3" + resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.15.3.tgz#1737e9aebfd363ffadffced71a3534c40e350223" + integrity sha512-tUE3pNbxCMX0HPKvwurLBPRAOAdS0gNo1+MpoyFSqXI7b7sp6/TCBRht6qu1Luw+VyIzBtXkJdnnqU+Uoy8iow== dependencies: hyperlist "^1.0.0-beta" lodash "^4.17.5" From a0df4ed8629eb10e3ec38c9faf6a212207ba5af3 Mon Sep 17 00:00:00 2001 From: Abhishek Balam Date: Wed, 21 Oct 2020 15:13:31 +0530 Subject: [PATCH 35/43] fix: add noreferrer policy to footer links --- frappe/templates/includes/footer/footer_grouped_links.html | 2 +- frappe/templates/includes/footer/footer_links.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/templates/includes/footer/footer_grouped_links.html b/frappe/templates/includes/footer/footer_grouped_links.html index 0383409090..06fa777a40 100644 --- a/frappe/templates/includes/footer/footer_grouped_links.html +++ b/frappe/templates/includes/footer/footer_grouped_links.html @@ -15,7 +15,7 @@