From 0eae494dc3200c0c1a0b1e20d6dcdd5790d12f85 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Wed, 12 Aug 2020 18:28:40 +0530 Subject: [PATCH 01/52] fix: email formatting in communication and email queue Co-authored-by: Sahil Khan --- frappe/patches.txt | 1 + .../patches/v12_0/fix_email_id_formatting.py | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 frappe/patches/v12_0/fix_email_id_formatting.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 68b4e5d99c..2b729ad87c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -298,3 +298,4 @@ frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart frappe.patches.v13_0.generate_theme_files_in_public_folder frappe.patches.v13_0.increase_password_length +frappe.patches.v12_0.fix_email_id_formatting diff --git a/frappe/patches/v12_0/fix_email_id_formatting.py b/frappe/patches/v12_0/fix_email_id_formatting.py new file mode 100644 index 0000000000..03f606e0cc --- /dev/null +++ b/frappe/patches/v12_0/fix_email_id_formatting.py @@ -0,0 +1,44 @@ +import frappe + +def execute(): + fix_communications() + fix_show_as_cc_email_queue() + fix_email_queue_recipients() + +def fix_communications(): + for communication in frappe.db.sql('''select name, recipients, cc, bcc from tabCommunication + where creation > '2020-06-01' + and communication_medium='Email' + and communication_type='Communication' + and (cc like '%<%' or bcc like '%<%' or recipients like '%<%') + ''', as_dict=1): + + communication['recipients'] = format_email_id(communication.recipients) + communication['cc'] = format_email_id(communication.cc) + communication['bcc'] = format_email_id(communication.bcc) + + frappe.db.sql('''update `tabCommunication` set recipients=%s,cc=%s,bcc=%s + where name =%s ''', (communication['recipients'], communication['cc'], + communication['bcc'], communication['name'])) + +def fix_show_as_cc_email_queue(): + for queue in frappe.get_all("Email Queue", {'creation': ['>', '2020-06-01'], + 'status': 'Not Sent', 'show_as_cc': ['like', '%<%']}, + ['name', 'show_as_cc']): + + frappe.db.set_value('Email Queue', queue['name'], + 'show_as_cc', format_email_id(queue['show_as_cc'])) + +def fix_email_queue_recipients(): + for recipient in frappe.db.sql('''select recipient, name from + `tabEmail Queue Recipient` where recipient like '%<%' + and status='Not Sent' and creation > '2020-06-01' ''', as_dict=1): + + frappe.db.set_value('Email Queue Recipient', recipient['name'], + 'recipient', format_email_id(recipient['recipient'])) + +def format_email_id(email): + if email and ('<' in email and '>' in email): + return email.replace('>', '>').replace('<', '<') + + return email From ee0bb635deb8bd63e375c55085e421fb0ef55c2c Mon Sep 17 00:00:00 2001 From: Anupam K Date: Wed, 19 Aug 2020 17:53:09 +0530 Subject: [PATCH 02/52] fix: newsletter fix --- .../email/doctype/newsletter/newsletter.json | 39 ++++++++++++++++--- frappe/email/doctype/newsletter/newsletter.py | 34 ++++++++-------- frappe/patches.txt | 1 + .../v13_0/update_newsletter_content_type.py | 12 ++++++ 4 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 frappe/patches/v13_0/update_newsletter_content_type.py diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 01f75be954..6babf212aa 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -15,7 +15,10 @@ "email_sent", "newsletter_content", "subject", + "content_type", "message", + "message_md", + "message_html", "send_unsubscribe_link", "send_attachments", "published", @@ -50,7 +53,8 @@ }, { "fieldname": "newsletter_content", - "fieldtype": "Section Break" + "fieldtype": "Section Break", + "label": "Content" }, { "fieldname": "subject", @@ -61,11 +65,12 @@ "reqd": 1 }, { + "depends_on": "eval: doc.content_type === 'Rich Text'", "fieldname": "message", "fieldtype": "Text Editor", "in_list_view": 1, "label": "Message", - "reqd": 1 + "mandatory_depends_on": "eval: doc.content_type === 'Rich Text'" }, { "default": "1", @@ -87,16 +92,20 @@ "read_only": 1 }, { + "collapsible": 1, "fieldname": "test_the_newsletter", - "fieldtype": "Section Break" + "fieldtype": "Section Break", + "label": "Testing" }, { "description": "A Lead with this Email Address should exist", "fieldname": "test_email_id", "fieldtype": "Data", - "label": "Test Email Address" + "label": "Test Email Address", + "options": "Email" }, { + "depends_on": "eval: doc.test_email_id", "fieldname": "test_send", "fieldtype": "Button", "label": "Test", @@ -127,6 +136,26 @@ "fieldname": "send_attachments", "fieldtype": "Check", "label": "Send Attachments" + }, + { + "fieldname": "content_type", + "fieldtype": "Select", + "label": "Content Type", + "options": "Rich Text\nMarkdown\nHTML" + }, + { + "depends_on": "eval:doc.content_type === 'Markdown'", + "fieldname": "message_md", + "fieldtype": "Markdown Editor", + "label": "Message (Markdown)", + "mandatory_depends_on": "eval:doc.content_type === 'Markdown'" + }, + { + "depends_on": "eval:doc.content_type === 'HTML'", + "fieldname": "message_html", + "fieldtype": "HTML Editor", + "label": "Message (HTML)", + "mandatory_depends_on": "eval:doc.content_type === 'HTML'" } ], "has_web_view": 1, @@ -135,7 +164,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-05-12 18:09:40.137138", + "modified": "2020-08-19 17:54:13.290593", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 48688afdb6..08a0e3d247 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -11,7 +11,7 @@ from frappe.utils.verified_command import get_signed_params, verify_request from frappe.utils.background_jobs import enqueue from frappe.email.queue import send from frappe.email.doctype.email_group.email_group import add_subscribers -from frappe.utils import parse_addr, now_datetime +from frappe.utils import parse_addr, now_datetime, markdown from frappe.utils import validate_email_address @@ -29,8 +29,8 @@ class Newsletter(WebsiteGenerator): def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) - self.queue_all() - frappe.msgprint(_("Scheduled to send to {0}").format(self.test_email_id)) + self.queue_all(test_email=True) + frappe.msgprint(_("Test email is send to {0}").format(self.test_email_id)) def send_emails(self): """send emails to leads and customers""" @@ -41,20 +41,13 @@ class Newsletter(WebsiteGenerator): if self.recipients: if getattr(frappe.local, "is_ajax", False): - self.validate_send() - # using default queue with a longer timeout as this isn't a scheduled task - enqueue(send_newsletter, queue='default', timeout=6000, event='send_newsletter', - newsletter=self.name) - - else: self.queue_all() - - frappe.msgprint(_("Scheduled to send to {0} recipients").format(len(self.recipients))) + frappe.msgprint(_("Email queued to {0} recipients").format(len(self.recipients))) else: frappe.msgprint(_("Newsletter should have atleast one recipient")) - def queue_all(self): + def queue_all(self, test_email=False): if not self.get("recipients"): # in case it is called via worker self.recipients = self.get_recipients() @@ -80,7 +73,7 @@ class Newsletter(WebsiteGenerator): frappe.throw(_("Unable to find attachment {0}").format(file.name)) send(recipients=self.recipients, sender=sender, - subject=self.subject, message=self.message, + subject=self.subject, message=self.get_message(), reference_doctype=self.doctype, reference_name=self.name, add_unsubscribe_link=self.send_unsubscribe_link, attachments=attachments, unsubscribe_method="/unsubscribe", @@ -90,9 +83,18 @@ class Newsletter(WebsiteGenerator): if not frappe.flags.in_test: frappe.db.auto_commit_on_many_writes = False - self.db_set("email_sent", 1) - self.db_set("schedule_send", now_datetime()) - self.db_set("scheduled_to_send", len(self.recipients)) + if not test_email: + self.db_set("email_sent", 1) + self.db_set("schedule_send", now_datetime()) + self.db_set("scheduled_to_send", len(self.recipients)) + + def get_message(self): + if self.content_type == 'Rich Text': + return self.message + elif self.content_type == 'Markdown': + return markdown(self.message_md) + elif self.content_type == 'HTML': + return self.message_html def get_recipients(self): """Get recipients from Email Group""" diff --git a/frappe/patches.txt b/frappe/patches.txt index 75750ab59c..bf92486cbf 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -297,3 +297,4 @@ frappe.patches.v13_0.replace_old_data_import # 2020-06-24 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart frappe.patches.v13_0.generate_theme_files_in_public_folder +frappe.patches.v13_0.update_newsletter_content_type \ No newline at end of file diff --git a/frappe/patches/v13_0/update_newsletter_content_type.py b/frappe/patches/v13_0/update_newsletter_content_type.py new file mode 100644 index 0000000000..0b32fb49ed --- /dev/null +++ b/frappe/patches/v13_0/update_newsletter_content_type.py @@ -0,0 +1,12 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + +def execute(): + frappe.reload_doc('email', 'doctype', 'Newsletter') + frappe.db.sql(""" + UPDATE tabNewsletter + SET content_type = 'Rich Text' + """) \ No newline at end of file From 613b91735ef79449232b38874d6b562cfb084478 Mon Sep 17 00:00:00 2001 From: KanchanChauhan Date: Fri, 21 Aug 2020 17:58:37 +0530 Subject: [PATCH 03/52] fix: Image shown as broken in comment if private --- frappe/utils/file_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index b1eb2b9ab3..e165a4e338 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -406,6 +406,10 @@ def extract_images_from_html(doc, content): doctype = doc.parenttype if doc.parent else doc.doctype name = doc.parent or doc.name + if doc.doctype == "Comment": + doctype = doc.reference_doctype + name = doc.reference_name + # TODO fix this file_url = save_file(filename, content, doctype, name, decode=True).get("file_url") if not frappe.flags.has_dataurl: From acabfd0b1663fc81d3780b2c6f8bcba7274ec51d Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 24 Aug 2020 00:41:24 +0530 Subject: [PATCH 04/52] fix: group dashboard cards based on label --- frappe/desk/desktop.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 148ae87249..94a38a5304 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -203,7 +203,7 @@ class Workspace: cards = cards + get_custom_reports_and_doctypes(self.doc.module) if len(self.extended_cards): - cards = cards + self.extended_cards + cards = merge_cards_based_on_label(cards + self.extended_cards) default_country = frappe.db.get_default("country") def _doctype_contains_a_record(name): @@ -579,3 +579,16 @@ def update_onboarding_step(name, field, value): """ frappe.db.set_value("Onboarding Step", name, field, value) + +def merge_cards_based_on_label(cards): + """Merge cards with common label.""" + cards_dict = {} + for card in cards: + if card.label in cards_dict: + links = loads(cards_dict[card.label].links) + loads(card.links) + cards_dict[card.label].update(dict(links=dumps(links))) + cards_dict[card.label] = cards_dict.pop(card.label) + else: + cards_dict[card.label] = card + + return list(cards_dict.values()) \ No newline at end of file From 178a576b6a39a252415c9ef5b49a57a7cef2c034 Mon Sep 17 00:00:00 2001 From: Anupam K Date: Mon, 24 Aug 2020 20:11:25 +0530 Subject: [PATCH 05/52] fix: review changes --- frappe/email/doctype/newsletter/newsletter.js | 4 ---- frappe/email/doctype/newsletter/newsletter.json | 8 +++++--- frappe/email/doctype/newsletter/newsletter.py | 14 +++++++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index 3d69c0cfad..3277d8e9ee 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -14,10 +14,6 @@ frappe.ui.form.on('Newsletter', { }); }, "fa fa-play", "btn-success"); } - if (!doc.__islocal && cint(doc.email_sent)) { - frm.set_df_property('schedule_send', "read_only", 1); - frm.set_df_property('schedule_sending', "read_only", 1); - } frm.events.setup_dashboard(frm); diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index fac211ab49..1dd6115b43 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -125,7 +125,8 @@ "depends_on": "eval: doc.schedule_sending", "fieldname": "schedule_send", "fieldtype": "Datetime", - "label": "Schedule Send" + "label": "Schedule Send", + "read_only_depends_on": "eval: doc.email_sent" }, { "default": "0", @@ -157,7 +158,8 @@ "default": "0", "fieldname": "schedule_sending", "fieldtype": "Check", - "label": "Schedule Sending" + "label": "Schedule Sending", + "read_only_depends_on": "eval: doc.email_sent" } ], "has_web_view": 1, @@ -167,7 +169,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-08-20 10:22:24.560288", + "modified": "2020-08-24 19:59:37.262500", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index afc0fa638b..929855ea30 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -27,7 +27,7 @@ class Newsletter(WebsiteGenerator): def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) self.queue_all(test_email=True) - frappe.msgprint(_("Test email is send to {0}").format(self.test_email_id)) + frappe.msgprint(_("Test email sent to {0}").format(self.test_email_id)) def send_emails(self): """send emails to leads and customers""" @@ -86,12 +86,12 @@ class Newsletter(WebsiteGenerator): self.db_set("scheduled_to_send", len(self.recipients)) def get_message(self): - if self.content_type == 'Rich Text': - return self.message - elif self.content_type == 'Markdown': - return markdown(self.message_md) - elif self.content_type == 'HTML': - return self.message_html + + return { + 'Rich Text': self.message, + 'Markdown': markdown(self.message_md), + 'HTML': self.message_html + }[self.content_type] def get_recipients(self): """Get recipients from Email Group""" From 1c56732de57f4846e88065623a728108eafe24c6 Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 24 Aug 2020 22:06:12 +0530 Subject: [PATCH 06/52] feat: add option to disable custom script --- .../doctype/custom_script/custom_script.json | 252 ++++++------------ frappe/desk/form/meta.py | 2 +- frappe/patches.txt | 1 + frappe/patches/v13_0/enable_custom_script.py | 13 + 4 files changed, 93 insertions(+), 175 deletions(-) create mode 100644 frappe/patches/v13_0/enable_custom_script.py diff --git a/frappe/custom/doctype/custom_script/custom_script.json b/frappe/custom/doctype/custom_script/custom_script.json index fc086e4b0b..328b247c49 100644 --- a/frappe/custom/doctype/custom_script/custom_script.json +++ b/frappe/custom/doctype/custom_script/custom_script.json @@ -1,187 +1,91 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 1, - "allow_rename": 0, - "autoname": "", - "beta": 0, - "creation": "2013-01-10 16:34:01", - "custom": 0, - "description": "Adds a client custom script to a DocType", - "docstatus": 0, - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_import": 1, + "creation": "2013-01-10 16:34:01", + "description": "Adds a client custom script to a DocType", + "doctype": "DocType", + "document_type": "Document", + "engine": "InnoDB", + "field_order": [ + "dt", + "enabled", + "script", + "sample" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "dt", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "DocType", - "length": 0, - "no_copy": 0, - "oldfieldname": "dt", - "oldfieldtype": "Link", - "options": "DocType", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "dt", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "DocType", + "oldfieldname": "dt", + "oldfieldtype": "Link", + "options": "DocType", + "reqd": 1, + "show_days": 1, + "show_seconds": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "script", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Script", - "length": 0, - "no_copy": 0, - "oldfieldname": "script", - "oldfieldtype": "Code", - "options": "JS", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "script", + "fieldtype": "Code", + "label": "Script", + "oldfieldname": "script", + "oldfieldtype": "Code", + "options": "JS", + "show_days": 1, + "show_seconds": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, - "fieldname": "sample", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Sample", - "length": 0, - "no_copy": 0, - "options": "

Custom Script Help

\n

Custom Scripts are executed only on the client-side (i.e. in Forms). Here are some examples to get you started

\n
\n\n// fetch local_tax_no on selection of customer \n// cur_frm.add_fetch(link_field,  source_fieldname,  target_fieldname); \ncur_frm.add_fetch('customer',  'local_tax_no',  'local_tax_no');\n\n// additional validation on dates \nfrappe.ui.form.on('Task',  'validate',  function(frm) {\n    if (frm.doc.from_date < get_today()) {\n        msgprint('You can not select past date in From Date');\n        validated = false;\n    } \n});\n\n// make a field read-only after saving \nfrappe.ui.form.on('Task',  {\n    refresh: function(frm) {\n        // use the __islocal value of doc,  to check if the doc is saved or not\n        frm.set_df_property('myfield',  'read_only',  frm.doc.__islocal ? 0 : 1);\n    } \n});\n\n// additional permission check\nfrappe.ui.form.on('Task',  {\n    validate: function(frm) {\n        if(user=='user1@example.com' && frm.doc.purpose!='Material Receipt') {\n            msgprint('You are only allowed Material Receipt');\n            validated = false;\n        }\n    } \n});\n\n// calculate sales incentive\nfrappe.ui.form.on('Sales Invoice',  {\n    validate: function(frm) {\n        // calculate incentives for each person on the deal\n        total_incentive = 0\n        $.each(frm.doc.sales_team,  function(i,  d) {\n            // calculate incentive\n            var incentive_percent = 2;\n            if(frm.doc.base_grand_total > 400) incentive_percent = 4;\n            // actual incentive\n            d.incentives = flt(frm.doc.base_grand_total) * incentive_percent / 100;\n            total_incentive += flt(d.incentives)\n        });\n        frm.doc.total_incentive = total_incentive;\n    } \n})\n\n
", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "sample", + "fieldtype": "HTML", + "label": "Sample", + "options": "

Custom Script Help

\n

Custom Scripts are executed only on the client-side (i.e. in Forms). Here are some examples to get you started

\n
\n\n// fetch local_tax_no on selection of customer \n// cur_frm.add_fetch(link_field,  source_fieldname,  target_fieldname); \ncur_frm.add_fetch('customer',  'local_tax_no',  'local_tax_no');\n\n// additional validation on dates \nfrappe.ui.form.on('Task',  'validate',  function(frm) {\n    if (frm.doc.from_date < get_today()) {\n        msgprint('You can not select past date in From Date');\n        validated = false;\n    } \n});\n\n// make a field read-only after saving \nfrappe.ui.form.on('Task',  {\n    refresh: function(frm) {\n        // use the __islocal value of doc,  to check if the doc is saved or not\n        frm.set_df_property('myfield',  'read_only',  frm.doc.__islocal ? 0 : 1);\n    } \n});\n\n// additional permission check\nfrappe.ui.form.on('Task',  {\n    validate: function(frm) {\n        if(user=='user1@example.com' && frm.doc.purpose!='Material Receipt') {\n            msgprint('You are only allowed Material Receipt');\n            validated = false;\n        }\n    } \n});\n\n// calculate sales incentive\nfrappe.ui.form.on('Sales Invoice',  {\n    validate: function(frm) {\n        // calculate incentives for each person on the deal\n        total_incentive = 0\n        $.each(frm.doc.sales_team,  function(i,  d) {\n            // calculate incentive\n            var incentive_percent = 2;\n            if(frm.doc.base_grand_total > 400) incentive_percent = 4;\n            // actual incentive\n            d.incentives = flt(frm.doc.base_grand_total) * incentive_percent / 100;\n            total_incentive += flt(d.incentives)\n        });\n        frm.doc.total_incentive = total_incentive;\n    } \n})\n\n
", + "show_days": 1, + "show_seconds": 1 + }, + { + "default": "0", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled", + "show_days": 1, + "show_seconds": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-glass", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "menu_index": 0, - "modified": "2019-03-21 14:26:57.402994", - "modified_by": "Administrator", - "module": "Custom", - "name": "Custom Script", - "owner": "Administrator", + ], + "icon": "fa fa-glass", + "idx": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2020-08-24 21:56:07.719579", + "modified_by": "Administrator", + "module": "Custom", + "name": "Custom Script", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + ], + "sort_order": "ASC", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index ba0e5c2216..c28a40657f 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -130,7 +130,7 @@ class FormMeta(Meta): def add_custom_script(self): """embed all require files""" # custom script - custom = frappe.db.get_value("Custom Script", {"dt": self.name}, "script") or "" + custom = frappe.db.get_value("Custom Script", {"dt": self.name, "enabled": 1}, "script") or "" self.set("__custom_js", custom) diff --git a/frappe/patches.txt b/frappe/patches.txt index afbd007abc..a00d93fdcd 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -302,3 +302,4 @@ frappe.patches.v13_0.increase_password_length frappe.patches.v13_0.add_toggle_width_in_navbar_settings frappe.patches.v13_0.rename_notification_fields frappe.patches.v13_0.remove_duplicate_navbar_items +frappe.patches.v13_0.enable_custom_script diff --git a/frappe/patches/v13_0/enable_custom_script.py b/frappe/patches/v13_0/enable_custom_script.py new file mode 100644 index 0000000000..92284e6dcc --- /dev/null +++ b/frappe/patches/v13_0/enable_custom_script.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + +def execute(): + """Enable all the existing custom script""" + frappe.reload_doc("Custom", "doctype", "Custom Script") + + frappe.db.sql(""" + UPDATE `tabCustom Script` SET enabled=1 + """) \ No newline at end of file From eab928a7110be295a81b1945fb5d8993fbbea137 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 25 Aug 2020 17:40:18 +0530 Subject: [PATCH 07/52] fix: translation --- frappe/translations/fa.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/translations/fa.csv b/frappe/translations/fa.csv index e93fcc4a21..0698897880 100644 --- a/frappe/translations/fa.csv +++ b/frappe/translations/fa.csv @@ -3304,7 +3304,7 @@ Daily Long,روزانه طولانی, Data Import Beta,واردات داده بتا, Default Role on Creation,نقش پیش فرض در آفرینش, Default Theme,موضوع پیش فرض, -Default {0},پیش فرض {0, +Default {0},پیش فرض {0}, Delete All,حذف همه, "Determines the order of the slide in the wizard. If the slide is not to be displayed, priority should be set to 0.",ترتیب اسلاید در جادوگر را تعیین می کند. اگر اسلاید نمایش داده نمی شود ، اولویت باید بر روی 0 تنظیم شود., Do you want to cancel all linked documents?,آیا می خواهید کلیه اسناد مرتبط را لغو کنید؟, From cbaa795c200b82414b109fd3824582e8a16bb522 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 26 Aug 2020 11:57:03 +0530 Subject: [PATCH 08/52] fix(backups): Allow naming conf file via BackupGenerator --- frappe/utils/backups.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index a64ba8c3a7..abf2a5fc81 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -27,7 +27,7 @@ class BackupGenerator: To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost") If specifying db_file_name, also append ".sql.gz" """ - def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None, + def __init__(self, db_name, user, password, backup_path_conf=None, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None, verbose=False, db_type='mariadb'): global _verbose @@ -37,8 +37,9 @@ class BackupGenerator: self.db_type = db_type self.user = user self.password = password - self.backup_path_files = backup_path_files + self.backup_path_conf = backup_path_conf self.backup_path_db = backup_path_db + self.backup_path_files = backup_path_files self.backup_path_private_files = backup_path_private_files if not self.db_port and self.db_type == 'mariadb': @@ -83,11 +84,14 @@ class BackupGenerator: def set_backup_file_name(self): #Generate a random name using today's date and a 8 digit random number + for_conf = self.todays_date + "-" + self.site_slug + "-site_config_backup.json" for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz" for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar" for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar" backup_path = get_backup_path() + if not self.backup_path_conf: + self.backup_path_conf = os.path.join(backup_path, for_conf) if not self.backup_path_db: self.backup_path_db = os.path.join(backup_path, for_db) if not self.backup_path_files: @@ -150,19 +154,11 @@ class BackupGenerator: print('Backed up files', os.path.abspath(backup_path)) def copy_site_config(self): - site_config_backup_path = os.path.join( - get_backup_path(), - "{time_stamp}-{site_slug}-site_config_backup.json".format( - time_stamp=self.todays_date, - site_slug=self.site_slug)) + site_config_backup_path = self.backup_path_conf site_config_path = os.path.join(frappe.get_site_path(), "site_config.json") - site_config = {} - if os.path.exists(site_config_path): - site_config.update(frappe.get_file_json(site_config_path)) + with open(site_config_backup_path, "w") as f: - f.write(json.dumps(site_config, indent=2)) - f.flush() - self.site_config_backup_path = site_config_backup_path + json.dump(frappe.get_file_json(site_config_path), f, indent=2) def take_dump(self): import frappe.utils From ae8201cc1c8a8f430919d3832476479ce92c0414 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 26 Aug 2020 12:08:51 +0530 Subject: [PATCH 09/52] fix: Just copy the contents of the files instead of parsing it --- frappe/utils/backups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index abf2a5fc81..3d683f6c0a 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -157,8 +157,8 @@ class BackupGenerator: site_config_backup_path = self.backup_path_conf site_config_path = os.path.join(frappe.get_site_path(), "site_config.json") - with open(site_config_backup_path, "w") as f: - json.dump(frappe.get_file_json(site_config_path), f, indent=2) + with open(site_config_backup_path, "w") as n, open(site_config_path) as c: + n.write(c.read()) def take_dump(self): import frappe.utils From 2ae5ce7d0e157a3ca5e3b04cc71f4b6ccb997d6f Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 26 Aug 2020 20:48:29 +0530 Subject: [PATCH 10/52] fix: Subtitle in Section With Tabs web template --- .../web_template/section_with_tabs/section_with_tabs.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/website/web_template/section_with_tabs/section_with_tabs.html b/frappe/website/web_template/section_with_tabs/section_with_tabs.html index 0c206b1c48..bb90611b4c 100644 --- a/frappe/website/web_template/section_with_tabs/section_with_tabs.html +++ b/frappe/website/web_template/section_with_tabs/section_with_tabs.html @@ -1,5 +1,8 @@

{{ title }}

+ +{%- if subtitle-%}

{{ subtitle }}

+{%- endif -%}
{% set ns = namespace(tabs=[]) %} From 2414dd386c24dcd0334a8c491eeb21d65d21bb14 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 27 Aug 2020 11:36:43 +0530 Subject: [PATCH 11/52] fix: formatting --- .../web_template/section_with_tabs/section_with_tabs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/web_template/section_with_tabs/section_with_tabs.html b/frappe/website/web_template/section_with_tabs/section_with_tabs.html index bb90611b4c..9a5bb20e0f 100644 --- a/frappe/website/web_template/section_with_tabs/section_with_tabs.html +++ b/frappe/website/web_template/section_with_tabs/section_with_tabs.html @@ -1,6 +1,6 @@

{{ title }}

-{%- if subtitle-%} +{%- if subtitle -%}

{{ subtitle }}

{%- endif -%} From 8398b8b6d84def1a48bd48c00c309df0e83c797d Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Thu, 27 Aug 2020 18:11:29 +0200 Subject: [PATCH 12/52] feat: add autoprefixer --- package.json | 1 + rollup/config.js | 1 + yarn.lock | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/package.json b/package.json index f893d03ad3..fb3e507bd5 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "dependencies": { "ace-builds": "^1.4.8", "air-datepicker": "http://github.com/frappe/air-datepicker", + "autoprefixer": "^9.8.6", "awesomplete": "^1.1.5", "bootstrap": "^4.4.1", "cookie": "^0.4.0", diff --git a/rollup/config.js b/rollup/config.js index 460780bc1b..b1816cb4c6 100644 --- a/rollup/config.js +++ b/rollup/config.js @@ -117,6 +117,7 @@ function get_rollup_options_for_css(output_file, input_files) { // less -> css postcss({ plugins: [ + starts_with_css ? require('autoprefixer')() : null, starts_with_css && production ? require('cssnano')({ preset: 'default' }) : null ].filter(Boolean), extract: output_path, diff --git a/yarn.lock b/yarn.lock index c3808f680a..41fd1926c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -836,6 +836,19 @@ atob@^2.1.1: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +autoprefixer@^9.8.6: + version "9.8.6" + resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.6.tgz#3b73594ca1bf9266320c5acf1588d74dea74210f" + integrity sha512-XrvP4VVHdRBCdX1S3WXVD8+RyG9qeb1D5Sn1DeLiG2xfSpzellk5k54xbUERJ3M5DggQxes39UGOTP8CFrEGbg== + dependencies: + browserslist "^4.12.0" + caniuse-lite "^1.0.30001109" + colorette "^1.2.1" + normalize-range "^0.1.2" + num2fraction "^1.2.2" + postcss "^7.0.32" + postcss-value-parser "^4.1.0" + available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz#6b098ca9d8039079ee3f77f7b783c4480ba513f5" @@ -1048,6 +1061,16 @@ browserslist@^4.0.0: electron-to-chromium "^1.3.113" node-releases "^1.1.8" +browserslist@^4.12.0: + version "4.14.0" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.0.tgz#2908951abfe4ec98737b72f34c3bcedc8d43b000" + integrity sha512-pUsXKAF2lVwhmtpeA3LJrZ76jXuusrNyhduuQs7CDFf9foT4Y38aQOserd2lMe5DSSrjf3fx34oHwryuvxAUgQ== + dependencies: + caniuse-lite "^1.0.30001111" + electron-to-chromium "^1.3.523" + escalade "^3.0.2" + node-releases "^1.1.60" + buble@^0.19.6: version "0.19.6" resolved "https://registry.yarnpkg.com/buble/-/buble-0.19.6.tgz#915909b6bd5b11ee03b1c885ec914a8b974d34d3" @@ -1203,6 +1226,11 @@ caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939: resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001116.tgz" integrity sha512-f2lcYnmAI5Mst9+g0nkMIznFGsArRmZ0qU+dnq8l91hymdc2J3SFbiPhOJEeDqC1vtE8nc1qNQyklzB8veJefQ== +caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001111: + version "1.0.30001118" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001118.tgz#116a9a670e5264aec895207f5e918129174c6f62" + integrity sha512-RNKPLojZo74a0cP7jFMidQI7nvLER40HgNfgKQEJ2PFm225L0ectUungNQoK3Xk3StQcFbpBPNEvoWD59436Hg== + caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" @@ -1427,6 +1455,11 @@ color@^3.0.0: color-convert "^1.9.1" color-string "^1.5.2" +colorette@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" + integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== + combined-stream@^1.0.6, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -2134,6 +2167,11 @@ electron-to-chromium@^1.3.113: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.113.tgz#b1ccf619df7295aea17bc6951dc689632629e4a9" integrity sha512-De+lPAxEcpxvqPTyZAXELNpRZXABRxf+uL/rSykstQhzj/B0l1150G/ExIIxKc16lI89Hgz81J0BHAcbTqK49g== +electron-to-chromium@^1.3.523: + version "1.3.551" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.551.tgz#a94d243a4ca90705189bd4a5eca4e0f56b745a4f" + integrity sha512-11qcm2xvf2kqeFO5EIejaBx5cKXsW1quAyv3VctCMYwofnyVZLs97y6LCekss3/ghQpr7PYkSO3uId5FmxZsdw== + elegant-spinner@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e" @@ -2309,6 +2347,11 @@ es6-promisify@^5.0.0: dependencies: es6-promise "^4.0.3" +escalade@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.0.2.tgz#6a580d70edb87880f22b4c91d0d56078df6962c4" + integrity sha512-gPYAU37hYCUhW5euPeR+Y74F7BL+IBsV93j5cvGriSaD1aG6MGsqsV1yamRdrWrb2j3aiZvb0X+UBOWpx3JWtQ== + escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -4816,6 +4859,11 @@ node-gyp@^3.8.0: tar "^2.0.0" which "1" +node-releases@^1.1.60: + version "1.1.60" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.60.tgz#6948bdfce8286f0b5d0e5a88e8384e954dfe7084" + integrity sha512-gsO4vjEdQaTusZAEebUWp2a5d7dF5DYoIpDG7WySnk7BuZDW+GPpHXoXXuYawRBr/9t5q54tirPz79kFIWg4dA== + node-releases@^1.1.8: version "1.1.9" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.9.tgz#70d0985ec4bf7de9f08fc481f5dae111889ca482" @@ -4878,6 +4926,11 @@ normalize-package-data@^2.3.2, normalize-package-data@^2.3.4: semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" +normalize-range@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942" + integrity sha1-LRDAa9/TEuqXd2laTShDlFa3WUI= + normalize-url@^3.0.0: version "3.3.0" resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559" @@ -4912,6 +4965,11 @@ nth-check@^1.0.2: dependencies: boolbase "~1.0.0" +num2fraction@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/num2fraction/-/num2fraction-1.2.2.tgz#6f682b6a027a4e9ddfa4564cd2589d1d4e669ede" + integrity sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4= + number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" @@ -5731,6 +5789,11 @@ postcss-value-parser@^3.0.0, postcss-value-parser@^3.3.1: resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz#9ff822547e2893213cf1c30efa51ac5fd1ba8281" integrity sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ== +postcss-value-parser@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb" + integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ== + postcss@6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/postcss/-/postcss-6.0.1.tgz#000dbd1f8eef217aa368b9a212c5fc40b2a8f3f2" @@ -5768,6 +5831,15 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.5: source-map "^0.6.1" supports-color "^6.1.0" +postcss@^7.0.32: + version "7.0.32" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.32.tgz#4310d6ee347053da3433db2be492883d62cec59d" + integrity sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw== + dependencies: + chalk "^2.4.2" + source-map "^0.6.1" + supports-color "^6.1.0" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" From 1b73602a1731f718f3a1cac86116292e18df40a4 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 19 Aug 2020 14:31:38 +0530 Subject: [PATCH 13/52] feat(system console): Added a System Console to help in debugging and Console Log --- .../doctype_action/doctype_action.json | 10 +-- frappe/database/mariadb/framework_mariadb.sql | 2 +- .../database/postgres/framework_postgres.sql | 2 +- frappe/desk/doctype/console_log/__init__.py | 0 .../desk/doctype/console_log/console_log.js | 8 +++ .../desk/doctype/console_log/console_log.json | 52 +++++++++++++++ .../desk/doctype/console_log/console_log.py | 10 +++ .../doctype/console_log/test_console_log.py | 10 +++ .../desk/doctype/system_console/__init__.py | 0 .../doctype/system_console/system_console.js | 8 +++ .../system_console/system_console.json | 66 +++++++++++++++++++ .../doctype/system_console/system_console.py | 36 ++++++++++ .../system_console/test_system_console.py | 10 +++ frappe/model/__init__.py | 3 +- frappe/model/naming.py | 2 + frappe/public/js/frappe/form/form.js | 12 +++- frappe/utils/safe_exec.py | 1 + 17 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 frappe/desk/doctype/console_log/__init__.py create mode 100644 frappe/desk/doctype/console_log/console_log.js create mode 100644 frappe/desk/doctype/console_log/console_log.json create mode 100644 frappe/desk/doctype/console_log/console_log.py create mode 100644 frappe/desk/doctype/console_log/test_console_log.py create mode 100644 frappe/desk/doctype/system_console/__init__.py create mode 100644 frappe/desk/doctype/system_console/system_console.js create mode 100644 frappe/desk/doctype/system_console/system_console.json create mode 100644 frappe/desk/doctype/system_console/system_console.py create mode 100644 frappe/desk/doctype/system_console/test_system_console.py diff --git a/frappe/core/doctype/doctype_action/doctype_action.json b/frappe/core/doctype/doctype_action/doctype_action.json index 7a1b845af3..87ee6fc12f 100644 --- a/frappe/core/doctype/doctype_action/doctype_action.json +++ b/frappe/core/doctype/doctype_action/doctype_action.json @@ -31,20 +31,22 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Action Type", - "options": "Server Action", + "options": "Server Action\nRoute", "reqd": 1 }, { "columns": 4, "fieldname": "action", - "fieldtype": "Data", + "fieldtype": "Small Text", "in_list_view": 1, - "label": "Action", + "label": "Action / Route", "reqd": 1 } ], + "index_web_pages_for_search": 1, "istable": 1, - "modified": "2019-09-24 09:11:39.860100", + "links": [], + "modified": "2020-08-18 20:03:42.479436", "modified_by": "Administrator", "module": "Core", "name": "DocType Action", diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index 1e3749e030..15b0bed699 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -128,7 +128,7 @@ CREATE TABLE `tabDocType Action` ( `label` varchar(140) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `group` varchar(140) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `action_type` varchar(140) COLLATE utf8mb4_unicode_ci DEFAULT NULL, - `action` varchar(140) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `action` text COLLATE utf8mb4_unicode_ci DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index a946a7ee5c..eeb0eecd3f 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -128,7 +128,7 @@ CREATE TABLE "tabDocType Action" ( "parenttype" varchar(255) DEFAULT NULL, "idx" bigint NOT NULL DEFAULT 0, "label" varchar(140) NOT NULL, - "group" varchar(140) DEFAULT NULL, + "group" text DEFAULT NULL, "action_type" varchar(140) NOT NULL, "action" varchar(140) NOT NULL, PRIMARY KEY ("name") diff --git a/frappe/desk/doctype/console_log/__init__.py b/frappe/desk/doctype/console_log/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/desk/doctype/console_log/console_log.js b/frappe/desk/doctype/console_log/console_log.js new file mode 100644 index 0000000000..1ef4fdce59 --- /dev/null +++ b/frappe/desk/doctype/console_log/console_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Console Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/desk/doctype/console_log/console_log.json b/frappe/desk/doctype/console_log/console_log.json new file mode 100644 index 0000000000..a9ae9717fd --- /dev/null +++ b/frappe/desk/doctype/console_log/console_log.json @@ -0,0 +1,52 @@ +{ + "actions": [], + "autoname": "format:Log on {timestamp}", + "creation": "2020-08-18 19:56:12.336427", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "script", + "output" + ], + "fields": [ + { + "fieldname": "script", + "fieldtype": "Code", + "in_list_view": 1, + "label": "Script", + "read_only": 1 + }, + { + "fieldname": "output", + "fieldtype": "Code", + "label": "Output", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2020-08-18 20:07:57.587344", + "modified_by": "Administrator", + "module": "Desk", + "name": "Console Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/desk/doctype/console_log/console_log.py b/frappe/desk/doctype/console_log/console_log.py new file mode 100644 index 0000000000..635c4c1ba7 --- /dev/null +++ b/frappe/desk/doctype/console_log/console_log.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class ConsoleLog(Document): + pass diff --git a/frappe/desk/doctype/console_log/test_console_log.py b/frappe/desk/doctype/console_log/test_console_log.py new file mode 100644 index 0000000000..04dc4f241f --- /dev/null +++ b/frappe/desk/doctype/console_log/test_console_log.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestConsoleLog(unittest.TestCase): + pass diff --git a/frappe/desk/doctype/system_console/__init__.py b/frappe/desk/doctype/system_console/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js new file mode 100644 index 0000000000..20c1257c10 --- /dev/null +++ b/frappe/desk/doctype/system_console/system_console.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('System Console', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json new file mode 100644 index 0000000000..8c56792abb --- /dev/null +++ b/frappe/desk/doctype/system_console/system_console.json @@ -0,0 +1,66 @@ +{ + "actions": [ + { + "action": "#List/Console Log/List", + "action_type": "Route", + "label": "Logs" + }, + { + "action": "frappe.desk.doctype.system_console.system_console.execute_code", + "action_type": "Server Action", + "label": "Execute" + } + ], + "creation": "2020-08-18 17:44:35.647815", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "console", + "commit", + "output" + ], + "fields": [ + { + "description": "To print output use log(text)", + "fieldname": "console", + "fieldtype": "Code", + "label": "Console" + }, + { + "fieldname": "output", + "fieldtype": "Code", + "label": "Output", + "read_only": 1 + }, + { + "default": "0", + "fieldname": "commit", + "fieldtype": "Check", + "label": "Commit" + } + ], + "hide_toolbar": 1, + "index_web_pages_for_search": 1, + "issingle": 1, + "links": [], + "modified": "2020-08-18 20:05:36.936664", + "modified_by": "Administrator", + "module": "Desk", + "name": "System Console", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py new file mode 100644 index 0000000000..3d7b3db2c2 --- /dev/null +++ b/frappe/desk/doctype/system_console/system_console.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals + +import json + +import frappe +from frappe.utils.safe_exec import safe_exec +from frappe.model.document import Document + +class SystemConsole(Document): + pass + +@frappe.whitelist() +def execute_code(doc): + doc = json.loads(doc) + frappe.only_for('System Manager') + try: + frappe.debug_log = [] + safe_exec(doc['console']) + doc['output'] = '\n'.join(frappe.debug_log) + except: + doc['output'] = frappe.get_traceback() + + if doc.get('commit'): + frappe.db.commit() + else: + frappe.db.rollback() + + frappe.get_doc(dict(doctype='Console Log', script=doc['console'], output=doc['output'])).insert() + frappe.db.commit() + + + return doc \ No newline at end of file diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py new file mode 100644 index 0000000000..2635b5a210 --- /dev/null +++ b/frappe/desk/doctype/system_console/test_system_console.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestSystemConsole(unittest.TestCase): + pass diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index e59d325c9a..c39a73ccd7 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -134,7 +134,8 @@ log_types = ( 'Notification Log', 'Email Queue', 'DocShare', - 'Document Follow' + 'Document Follow', + 'Console Log' ) def delete_fields(args_dict, delete=0): diff --git a/frappe/model/naming.py b/frappe/model/naming.py index ffaf84e2b3..f2c918113b 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -142,6 +142,8 @@ def parse_naming_series(parts, doctype='', doc=''): part = today.strftime("%d") elif e == 'YYYY': part = today.strftime('%Y') + elif e == 'timestamp': + part = str(today) elif e == 'FY': part = frappe.defaults.get_user_default("fiscal_year") elif e.startswith('{') and doc: diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index ff48ad2f60..d417d37c08 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -323,12 +323,22 @@ frappe.ui.form.Form = class FrappeForm { if (!this.is_new()) { this.add_custom_button(action.label, () => { if (action.action_type==='Server Action') { - frappe.xcall(action.action, {doc: this.doc}).then(() => { + frappe.xcall(action.action, {doc: this.doc}).then((doc) => { + if (doc.doctype) { + // document is returned by the method, + // apply the changes locally and refresh + frappe.model.sync(doc); + this.refresh(); + } + + // feedback frappe.msgprint({ message: __('{} Complete', [action.label]), alert: true }); }); + } else if (action.action_type==='Route') { + frappe.set_route(action.action); } }, action.group); } diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index c95b7e4699..548bd0baf7 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -48,6 +48,7 @@ def get_safe_globals(): # make available limited methods of frappe json=json, dict=dict, + log=frappe.log, _dict=frappe._dict, frappe=frappe._dict( flags=frappe.flags, From 12be3d26ff26fa798000504547325251e474820a Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 21 Aug 2020 15:03:57 +0530 Subject: [PATCH 14/52] fix(minor): primary button --- .../doctype_action/doctype_action.json | 11 +++- .../doctype/system_console/system_console.js | 9 ++- .../system_console/system_console.json | 3 +- frappe/public/js/frappe/form/form.js | 60 ++++++++++++------- 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/frappe/core/doctype/doctype_action/doctype_action.json b/frappe/core/doctype/doctype_action/doctype_action.json index 87ee6fc12f..0f9da802eb 100644 --- a/frappe/core/doctype/doctype_action/doctype_action.json +++ b/frappe/core/doctype/doctype_action/doctype_action.json @@ -8,7 +8,8 @@ "label", "action_type", "action", - "group" + "group", + "hidden" ], "fields": [ { @@ -41,12 +42,18 @@ "in_list_view": 1, "label": "Action / Route", "reqd": 1 + }, + { + "default": "0", + "fieldname": "hidden", + "fieldtype": "Check", + "label": "Hidden" } ], "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2020-08-18 20:03:42.479436", + "modified": "2020-08-21 14:44:03.845315", "modified_by": "Administrator", "module": "Core", "name": "DocType Action", diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index 20c1257c10..0aae8bd519 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -2,7 +2,10 @@ // For license information, please see license.txt frappe.ui.form.on('System Console', { - // refresh: function(frm) { - - // } + refresh: function(frm) { + frm.disable_save(); + frm.page.set_primary_action(__("Execute"), () => { + frm.execute_action('Execute'); + }); + } }); diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json index 8c56792abb..296647a17a 100644 --- a/frappe/desk/doctype/system_console/system_console.json +++ b/frappe/desk/doctype/system_console/system_console.json @@ -8,6 +8,7 @@ { "action": "frappe.desk.doctype.system_console.system_console.execute_code", "action_type": "Server Action", + "hidden": 1, "label": "Execute" } ], @@ -44,7 +45,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-08-18 20:05:36.936664", + "modified": "2020-08-21 14:44:35.296877", "modified_by": "Administrator", "module": "Desk", "name": "System Console", diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index d417d37c08..9e221f7131 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -321,32 +321,52 @@ frappe.ui.form.Form = class FrappeForm { for (let action of this.meta.actions) { frappe.ui.form.on(this.doctype, 'refresh', () => { if (!this.is_new()) { - this.add_custom_button(action.label, () => { - if (action.action_type==='Server Action') { - frappe.xcall(action.action, {doc: this.doc}).then((doc) => { - if (doc.doctype) { - // document is returned by the method, - // apply the changes locally and refresh - frappe.model.sync(doc); - this.refresh(); - } - - // feedback - frappe.msgprint({ - message: __('{} Complete', [action.label]), - alert: true - }); - }); - } else if (action.action_type==='Route') { - frappe.set_route(action.action); - } - }, action.group); + if (!action.hidden) { + this.add_custom_button(action.label, () => { + this.execute_action(action); + }, action.group); + } } }); } } } + execute_action(action) { + if (typeof action === 'string') { + // called by label - maybe via custom script + // frm.execute_action('Action') + for (let _action of this.meta.actions) { + if (_action.label === action) { + action = _action; + break; + } + } + + if (typeof action === 'string') { + frappe.throw(`Action ${action} not found`); + } + } + if (action.action_type==='Server Action') { + frappe.xcall(action.action, {doc: this.doc}).then((doc) => { + if (doc.doctype) { + // document is returned by the method, + // apply the changes locally and refresh + frappe.model.sync(doc); + this.refresh(); + } + + // feedback + frappe.msgprint({ + message: __('{} Complete', [action.label]), + alert: true + }); + }); + } else if (action.action_type==='Route') { + frappe.set_route(action.action); + } + } + switch_doc(docname) { // record switch if(this.docname != docname && (!this.meta.in_dialog || this.in_form) && !this.meta.istable) { From 5a61558820d1c05892c27320a6b455b112aefbd5 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Mon, 24 Aug 2020 17:45:20 +0530 Subject: [PATCH 15/52] fix(minor): return value in frappe.flags --- frappe/core/doctype/server_script/server_script.json | 7 ++++--- frappe/core/doctype/server_script/server_script.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 3ed4076430..cc3995ad1d 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -7,12 +7,12 @@ "engine": "InnoDB", "field_order": [ "script_type", - "disabled", - "column_break_3", "reference_doctype", "doctype_event", "api_method", "allow_guest", + "column_break_3", + "disabled", "section_break_8", "script", "help_section", @@ -85,8 +85,9 @@ "fieldtype": "HTML" } ], + "index_web_pages_for_search": 1, "links": [], - "modified": "2020-08-07 13:13:02.483963", + "modified": "2020-08-24 16:44:41.060350", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index 539ae8eb01..55d7a33b8c 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -25,6 +25,7 @@ class ServerScript(Document): if frappe.session.user == 'Guest' and not self.allow_guest: raise frappe.PermissionError safe_exec(self.script) + return frappe.flags # output can be stored in flags else: # wrong report type! raise frappe.DoesNotExistError From 079a0e4af0fb38d8773b4f0d6669cbd090cba26e Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 25 Aug 2020 11:14:03 +0530 Subject: [PATCH 16/52] wip: refactor System Console --- .../doctype/system_console/system_console.py | 42 ++++++++++--------- frappe/utils/safe_exec.py | 8 +++- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 3d7b3db2c2..2715cd1425 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -11,26 +11,28 @@ from frappe.utils.safe_exec import safe_exec from frappe.model.document import Document class SystemConsole(Document): - pass + def run(self): + frappe.only_for('System Manager') + try: + frappe.debug_log = [] + safe_exec(self.console) + self.output = '\n'.join(frappe.debug_log) + except: + self.output = frappe.get_traceback() + + if self.commit: + frappe.db.commit() + else: + frappe.db.rollback() + + frappe.get_doc(dict( + doctype='Console Log', + script=self.console, + output=self.output)).insert() + frappe.db.commit() @frappe.whitelist() def execute_code(doc): - doc = json.loads(doc) - frappe.only_for('System Manager') - try: - frappe.debug_log = [] - safe_exec(doc['console']) - doc['output'] = '\n'.join(frappe.debug_log) - except: - doc['output'] = frappe.get_traceback() - - if doc.get('commit'): - frappe.db.commit() - else: - frappe.db.rollback() - - frappe.get_doc(dict(doctype='Console Log', script=doc['console'], output=doc['output'])).insert() - frappe.db.commit() - - - return doc \ No newline at end of file + console = frappe.get_doc(json.loads(doc)) + console.run() + return console.as_dict() \ No newline at end of file diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 548bd0baf7..a070d287da 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -100,7 +100,8 @@ def get_safe_globals(): scrub=scrub, guess_mimetype=mimetypes.guess_type, html2text=html2text, - dev_server=1 if os.environ.get('DEV_SERVER', False) else 0 + dev_server=1 if os.environ.get('DEV_SERVER', False) else 0, + run_script=run_script ) add_module_properties(frappe.exceptions, out.frappe, lambda obj: inspect.isclass(obj) and issubclass(obj, Exception)) @@ -143,6 +144,11 @@ def read_sql(query, *args, **kwargs): else: raise frappe.PermissionError('Only SELECT SQL allowed in scripting') +def run_script(script): + '''run another server script''' + frappe.get_doc('Server Script', script).execute_method() + return frappe.flags + def _getitem(obj, key): # guard function for RestrictedPython # allow any key to be accessed as long as it does not start with underscore From 39e36e01cde80e37ba0e758f60b4492700dbd177 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 25 Aug 2020 15:09:33 +0530 Subject: [PATCH 17/52] fix(test): test for system console; --- .../doctype/system_console/test_system_console.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py index 2635b5a210..55ef199122 100644 --- a/frappe/desk/doctype/system_console/test_system_console.py +++ b/frappe/desk/doctype/system_console/test_system_console.py @@ -3,8 +3,18 @@ # See license.txt from __future__ import unicode_literals -# import frappe +import frappe import unittest class TestSystemConsole(unittest.TestCase): - pass + def test_system_console(self): + system_console = frappe.get_doc('System Console') + system_console.console = 'log("hello")' + system_console.run() + + self.assertEqual(system_console.output, 'hello') + + system_console.console = 'log(frappe.db.get_value("DocType", "DocType", "module"))' + system_console.run() + + self.assertEqual(system_console.output, 'Core') From 5e4ee1a6bcfd77d10ace5b810d43244c25394fc7 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Mon, 31 Aug 2020 11:29:06 +0530 Subject: [PATCH 18/52] fix(minor): quote the column names in alter table --- frappe/database/postgres/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index b5129b60bb..58153ca6ce 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -49,7 +49,7 @@ class PostgresTable(DBTable): elif col.fieldtype in ("Check"): using_clause = "USING {}::smallint".format(col.fieldname) - query.append("ALTER COLUMN {0} TYPE {1} {2}".format( + query.append("ALTER COLUMN `{0}` TYPE {1} {2}".format( col.fieldname, get_definition(col.fieldtype, precision=col.precision, length=col.length), using_clause) From a032e9fc3d0d87835f22d4a2990a57ad247244b5 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Mon, 31 Aug 2020 11:31:44 +0530 Subject: [PATCH 19/52] fix(flake8): ignore bare except --- frappe/desk/doctype/system_console/system_console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 2715cd1425..6c87ca8c36 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -17,7 +17,7 @@ class SystemConsole(Document): frappe.debug_log = [] safe_exec(self.console) self.output = '\n'.join(frappe.debug_log) - except: + except: # noqa: E722 self.output = frappe.get_traceback() if self.commit: From dcce45b558f1a7d602214540f81f53a1ad3b888d Mon Sep 17 00:00:00 2001 From: Anupam K Date: Mon, 31 Aug 2020 11:54:56 +0530 Subject: [PATCH 20/52] fix: test case --- frappe/email/doctype/newsletter/test_newsletter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index bb339165d3..ee7f123b7e 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -67,6 +67,7 @@ class TestNewsletter(unittest.TestCase): "doctype": "Newsletter", "subject": "_Test Newsletter", "send_from": "Test Sender ", + "content_type": "Rich Text", "message": "Testing my news.", "published": published, "schedule_sending": bool(schedule_send), From afe5959963d8277f97c93b37ffd15d55ef58b8c1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 31 Aug 2020 14:30:33 +0530 Subject: [PATCH 21/52] fix: Unbreaking change for kwarg backup_path_conf --- frappe/utils/backups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 3d683f6c0a..77d3e83dd9 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -27,9 +27,9 @@ class BackupGenerator: To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost") If specifying db_file_name, also append ".sql.gz" """ - def __init__(self, db_name, user, password, backup_path_conf=None, backup_path_db=None, backup_path_files=None, + def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None, backup_path_private_files=None, db_host="localhost", db_port=None, verbose=False, - db_type='mariadb'): + db_type='mariadb', backup_path_conf=None): global _verbose self.db_host = db_host self.db_port = db_port From 1eb1cba6664774aeb693b0ca53bc715eb806c141 Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 31 Aug 2020 19:54:19 +0530 Subject: [PATCH 22/52] feat: Show Status Section in Query Report --- frappe/public/js/frappe/views/reports/query_report.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 7958ff46cc..875be7d29e 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -592,6 +592,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { this.render_summary(data.report_summary); } + if(data.message && !data.prepared_report) this.show_status(data.message); + this.toggle_message(false); if (data.result && data.result.length) { this.prepare_report_data(data); From 4b2c82b9c90bb02432a76b34c2611cd1b51acd09 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 31 Aug 2020 20:14:34 +0530 Subject: [PATCH 23/52] fix: Set blog route if title and category are set Set blog route on client side as soon as title and category are selected. This behavious is in line with Web Page, where the route is set as soon as the title is entered. This will ensure blog routes are consistent and contain category as the part of the route. The user can change the route ofcourse, but this behaviour is the most common expectation. --- frappe/website/doctype/blog_post/blog_post.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/website/doctype/blog_post/blog_post.js b/frappe/website/doctype/blog_post/blog_post.js index 7aa83f536d..bfff947948 100644 --- a/frappe/website/doctype/blog_post/blog_post.js +++ b/frappe/website/doctype/blog_post/blog_post.js @@ -11,16 +11,29 @@ frappe.ui.form.on('Blog Post', { }, title: function(frm) { generate_google_search_preview(frm); + frm.trigger('set_route'); }, meta_description: function(frm) { generate_google_search_preview(frm); }, blog_intro: function(frm) { generate_google_search_preview(frm); + }, + blog_category(frm) { + frm.trigger('set_route'); + }, + set_route(frm) { + if (frm.doc.route) return; + if (frm.doc.title && frm.doc.blog_category) { + frm.call('make_route').then(r => { + frm.set_value('route', r.message); + }); + } } }); function generate_google_search_preview(frm) { + if (!frm.doc.title) return; let google_preview = frm.get_field("google_preview"); let seo_title = (frm.doc.title).slice(0, 60); let seo_description = (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160); From 04dbb4c2613f345007ab992cd5b610fb1bc31f3d Mon Sep 17 00:00:00 2001 From: Marica Date: Mon, 31 Aug 2020 20:16:02 +0530 Subject: [PATCH 24/52] fix: Spacing --- frappe/public/js/frappe/views/reports/query_report.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 875be7d29e..1bec65e460 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -592,7 +592,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { this.render_summary(data.report_summary); } - if(data.message && !data.prepared_report) this.show_status(data.message); + if (data.message && !data.prepared_report) this.show_status(data.message); this.toggle_message(false); if (data.result && data.result.length) { From 8f819da4ff7afdf9c218d0bf45417ee9fe167704 Mon Sep 17 00:00:00 2001 From: Anupam K Date: Tue, 1 Sep 2020 12:43:58 +0530 Subject: [PATCH 25/52] fix: failed test case --- frappe/email/doctype/newsletter/newsletter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 929855ea30..14bd7eb1dc 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -37,8 +37,8 @@ class Newsletter(WebsiteGenerator): self.recipients = self.get_recipients() if self.recipients: + self.queue_all() if getattr(frappe.local, "is_ajax", False): - self.queue_all() frappe.msgprint(_("Email queued to {0} recipients").format(len(self.recipients))) else: From f772aee2110b2383d42c2ee276c766baa19a7dca Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 1 Sep 2020 12:48:26 +0530 Subject: [PATCH 26/52] fix(minor): don't allow access to global flags --- frappe/utils/safe_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index c95b7e4699..b393939f92 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -50,7 +50,7 @@ def get_safe_globals(): dict=dict, _dict=frappe._dict, frappe=frappe._dict( - flags=frappe.flags, + flags=frappe._dict(), format=frappe.format_value, format_value=frappe.format_value, date_format=date_format, From c53813950fde81bd884f601690a9c5c15c070c1c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 1 Sep 2020 12:59:22 +0530 Subject: [PATCH 27/52] fix(minor): Server Script can return values in frappe.flags --- frappe/core/doctype/server_script/server_script.py | 4 ++-- .../core/doctype/server_script/test_server_script.py | 12 ++++++++++++ frappe/utils/safe_exec.py | 5 +++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index 55d7a33b8c..839b784651 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -24,8 +24,8 @@ class ServerScript(Document): # validate if guest is allowed if frappe.session.user == 'Guest' and not self.allow_guest: raise frappe.PermissionError - safe_exec(self.script) - return frappe.flags # output can be stored in flags + _globals, _locals = safe_exec(self.script) + return _globals.frappe.flags # output can be stored in flags else: # wrong report type! raise frappe.DoesNotExistError diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 5c12858e8a..3356e584af 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -36,6 +36,15 @@ if "validate" in doc.description: allow_guest = 1, script = ''' frappe.response['message'] = 'hello' +''' + ), + dict( + name='test_return_value', + script_type = 'API', + api_method = 'test_return_value', + allow_guest = 1, + script = ''' +frappe.flags = 'hello' ''' ) ] @@ -73,3 +82,6 @@ class TestServerScript(unittest.TestCase): response = requests.post(get_site_url(frappe.local.site) + "/api/method/test_server_script") self.assertEqual(response.status_code, 200) self.assertEqual("hello", response.json()["message"]) + + def test_api_return(self): + self.assertEqual(frappe.get_doc('Server Script', 'test_return_value').execute_method(), 'hello') diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index a070d287da..56c5fe90c8 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -28,6 +28,8 @@ def safe_exec(script, _globals=None, _locals=None): # execute script compiled by RestrictedPython exec(compile_restricted(script), exec_globals, _locals) # pylint: disable=exec-used + return exec_globals, _locals + def get_safe_globals(): datautils = frappe._dict() if frappe.db: @@ -146,8 +148,7 @@ def read_sql(query, *args, **kwargs): def run_script(script): '''run another server script''' - frappe.get_doc('Server Script', script).execute_method() - return frappe.flags + return frappe.get_doc('Server Script', script).execute_method() def _getitem(obj, key): # guard function for RestrictedPython From 7efe1db6287043f3d6d2f34c07ba447daf7d079d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 1 Sep 2020 14:07:16 +0530 Subject: [PATCH 28/52] Revert "fix: child table rendering with escape_html" --- frappe/public/js/frappe/form/grid_row.js | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 366f746dc5..733c1bea5f 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -276,32 +276,32 @@ export default class GridRow { make_column(df, colsize, txt, ci) { let me = this; - var add_class = ((["Text", "Small Text"].indexOf(df.fieldtype) !== -1) ? + var add_class = ((["Text", "Small Text"].indexOf(df.fieldtype)!==-1) ? " grid-overflow-no-ellipsis" : ""); - add_class += (["Int", "Currency", "Float", "Percent"].indexOf(df.fieldtype) !== -1) ? - " text-right" : ""; - add_class += (["Check"].indexOf(df.fieldtype) !== -1) ? - " text-center" : ""; + add_class += (["Int", "Currency", "Float", "Percent"].indexOf(df.fieldtype)!==-1) ? + " text-right": ""; + add_class += (["Check"].indexOf(df.fieldtype)!==-1) ? + " text-center": ""; - var $col = $('
') + var $col = $('
') .attr("data-fieldname", df.fieldname) .attr("data-fieldtype", df.fieldtype) .data("df", df) .appendTo(this.row) - .on('click', function () { - if (frappe.ui.form.editable_row === me) { + .on('click', function() { + if(frappe.ui.form.editable_row===me) { return; } var out = me.toggle_editable_row(); var col = this; - setTimeout(function () { + setTimeout(function() { $(col).find('input[type="Text"]:first').focus(); }, 500); return out; }); $col.field_area = $('
').appendTo($col).toggle(false); - $col.static_area = $('
').appendTo($col).html(frappe.utils.escape_html(txt)); + $col.static_area = $('
').appendTo($col).html(txt); $col.df = df; $col.column_index = ci; @@ -577,39 +577,39 @@ export default class GridRow { var df = this.grid.get_docfield(fieldname) || undefined; // format values if no frm - if (!df) { + if(!df) { df = this.grid.visible_columns.find((col) => { return col[0].fieldname === fieldname; }); - if (df && this.doc) { + if(df && this.doc) { var txt = frappe.format(this.doc[fieldname], df[0], null, this.doc); } } - if (txt === undefined && this.frm) { + if(txt===undefined && this.frm) { var txt = frappe.format(this.doc[fieldname], df, null, this.frm.doc); } // reset static value var column = this.columns[fieldname]; - if (column) { - column.static_area.html(frappe.utils.escape_html(txt) || ""); - if (df && df.reqd) { - column.toggleClass('error', !!(txt === null || txt === '')); + if(column) { + column.static_area.html(txt || ""); + if(df && df.reqd) { + column.toggleClass('error', !!(txt===null || txt==='')); } } // reset field value var field = this.on_grid_fields_dict[fieldname]; - if (field) { + if(field) { field.docname = this.doc.name; field.refresh(); } // in form - if (this.grid_form) { + if(this.grid_form) { this.grid_form.refresh_field(fieldname); } } From 428080bc56003427a113826c488f2d55459c1841 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Tue, 1 Sep 2020 21:04:57 +0530 Subject: [PATCH 29/52] fix: Send email notification to all document assignees --- .../email/doctype/notification/notification.json | 13 +++++++++++-- frappe/email/doctype/notification/notification.py | 15 +++++++++++++-- .../notification_recipient.json | 3 ++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index 95f218ad73..8918709fa0 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -34,6 +34,7 @@ "set_property_after_alert", "property_value", "column_break_5", + "send_to_all_assignees", "recipients", "message_sb", "message", @@ -216,7 +217,7 @@ "fieldname": "recipients", "fieldtype": "Table", "label": "Recipients", - "mandatory_depends_on": "eval:doc.channel!=='Slack'", + "mandatory_depends_on": "eval:doc.channel!=='Slack' && !doc.send_to_all_assignees", "options": "Notification Recipient" }, { @@ -277,11 +278,19 @@ "fieldname": "send_system_notification", "fieldtype": "Check", "label": "Send System Notification" + }, + { + "default": "0", + "depends_on": "eval:doc.channel == 'Email'", + "fieldname": "send_to_all_assignees", + "fieldtype": "Check", + "label": "Send To All Assignees" } ], "icon": "fa fa-envelope", + "index_web_pages_for_search": 1, "links": [], - "modified": "2020-08-11 19:24:35.479373", + "modified": "2020-09-01 18:36:22.550891", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 2ec208c89d..cfa3c43ff7 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -189,6 +189,7 @@ def get_context(context): recipients, cc, bcc = self.get_list_of_recipients(doc, context) if not (recipients or cc or bcc): return + sender = None if self.sender and self.sender_email: sender = formataddr((self.sender, self.sender_email)) @@ -239,8 +240,6 @@ def get_context(context): email_ids = email_ids_value.replace(",", "\n") recipients = recipients + email_ids.split("\n") - # else: - # print "invalid email" if recipient.cc and "{" in recipient.cc: recipient.cc = frappe.render_template(recipient.cc, context) @@ -262,6 +261,9 @@ def get_context(context): for email in emails: recipients = recipients + email.split("\n") + if self.send_to_all_assignees: + recipients = recipients + get_assignees(doc) + if not recipients and not cc and not bcc: return None, None, None return list(set(recipients)), list(set(cc)), list(set(bcc)) @@ -405,3 +407,12 @@ def evaluate_alert(doc, alert, event): def get_context(doc): return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=frappe.utils)} + +def get_assignees(doc): + assignees = [] + assignees = frappe.get_all('ToDo', filters={'status': 'Open', 'reference_name': doc.name, + 'reference_type': doc.doctype}, fields=['owner']) + + recipients = [d.owner for d in assignees] + + return recipients \ No newline at end of file diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.json b/frappe/email/doctype/notification_recipient/notification_recipient.json index 201899cd57..0670320a77 100644 --- a/frappe/email/doctype/notification_recipient/notification_recipient.json +++ b/frappe/email/doctype/notification_recipient/notification_recipient.json @@ -46,9 +46,10 @@ "options": "Role" } ], + "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2020-02-21 11:18:40.125233", + "modified": "2020-09-01 17:40:27.289105", "modified_by": "Administrator", "module": "Email", "name": "Notification Recipient", From 3eb0e5ce8a88f39bc79f61415f2af6bc0a7c45e2 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 2 Sep 2020 09:21:24 +0530 Subject: [PATCH 30/52] Update frappe/desk/doctype/system_console/system_console.js Co-authored-by: Aditya Hase --- frappe/desk/doctype/system_console/system_console.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js index 0aae8bd519..c7eac39490 100644 --- a/frappe/desk/doctype/system_console/system_console.js +++ b/frappe/desk/doctype/system_console/system_console.js @@ -2,6 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on('System Console', { + onload: function(frm) { + frappe.ui.keys.add_shortcut({ + shortcut: 'shift+enter', + action: () => frm.execute_action('Execute'), + page: frm.page, + description: __('Execute Console script'), + ignore_inputs: true, + }); + }, + refresh: function(frm) { frm.disable_save(); frm.page.set_primary_action(__("Execute"), () => { From ac76c551600408f2020c192ceb4dfa1016746ccf Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 2 Sep 2020 09:22:28 +0530 Subject: [PATCH 31/52] Update frappe/desk/doctype/system_console/system_console.json Co-authored-by: Aditya Hase --- frappe/desk/doctype/system_console/system_console.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json index 296647a17a..14e36e6fd3 100644 --- a/frappe/desk/doctype/system_console/system_console.json +++ b/frappe/desk/doctype/system_console/system_console.json @@ -26,7 +26,8 @@ "description": "To print output use log(text)", "fieldname": "console", "fieldtype": "Code", - "label": "Console" + "label": "Console", + "options": "Python" }, { "fieldname": "output", @@ -64,4 +65,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} \ No newline at end of file +} From aecb77ce9b5023dab233e55ee918cb87f169812e Mon Sep 17 00:00:00 2001 From: Anupam K Date: Wed, 2 Sep 2020 12:19:34 +0530 Subject: [PATCH 32/52] fix: removing condition --- frappe/email/doctype/newsletter/newsletter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 14bd7eb1dc..0a0a13a6ce 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -38,8 +38,7 @@ class Newsletter(WebsiteGenerator): if self.recipients: self.queue_all() - if getattr(frappe.local, "is_ajax", False): - frappe.msgprint(_("Email queued to {0} recipients").format(len(self.recipients))) + frappe.msgprint(_("Email queued to {0} recipients").format(len(self.recipients))) else: frappe.msgprint(_("Newsletter should have atleast one recipient")) From 0b8ac586c0f98c8d69777ac1b6b52dd402cd7348 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Wed, 2 Sep 2020 12:51:15 +0530 Subject: [PATCH 33/52] fix: newsletter issues (#11321) * fix: newsletter fix * fix: review changes * fix: test case * fix: failed test case * fix: removing condition Co-authored-by: Prssanna Desai --- frappe/email/doctype/newsletter/newsletter.js | 3 -- .../email/doctype/newsletter/newsletter.json | 48 +++++++++++++++---- frappe/email/doctype/newsletter/newsletter.py | 40 ++++++++-------- .../doctype/newsletter/newsletter_list.js | 6 ++- .../doctype/newsletter/test_newsletter.py | 1 + frappe/patches.txt | 1 + .../v13_0/update_newsletter_content_type.py | 12 +++++ 7 files changed, 76 insertions(+), 35 deletions(-) create mode 100644 frappe/patches/v13_0/update_newsletter_content_type.py diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index 0f1e8dc57c..3277d8e9ee 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -14,9 +14,6 @@ frappe.ui.form.on('Newsletter', { }); }, "fa fa-play", "btn-success"); } - if (!doc.__islocal && cint(doc.email_sent)) { - frm.set_df_property('schedule_send', "read_only", 1); - } frm.events.setup_dashboard(frm); diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 4804b3d6fa..1dd6115b43 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -15,7 +15,10 @@ "email_sent", "newsletter_content", "subject", + "content_type", "message", + "message_md", + "message_html", "send_unsubscribe_link", "send_attachments", "published", @@ -37,8 +40,7 @@ "fieldname": "send_from", "fieldtype": "Data", "ignore_xss_filter": 1, - "label": "Sender", - "no_copy": 1 + "label": "Sender" }, { "default": "0", @@ -50,7 +52,8 @@ }, { "fieldname": "newsletter_content", - "fieldtype": "Section Break" + "fieldtype": "Section Break", + "label": "Content" }, { "fieldname": "subject", @@ -61,11 +64,12 @@ "reqd": 1 }, { + "depends_on": "eval: doc.content_type === 'Rich Text'", "fieldname": "message", "fieldtype": "Text Editor", "in_list_view": 1, "label": "Message", - "reqd": 1 + "mandatory_depends_on": "eval: doc.content_type === 'Rich Text'" }, { "default": "1", @@ -87,16 +91,20 @@ "read_only": 1 }, { + "collapsible": 1, "fieldname": "test_the_newsletter", - "fieldtype": "Section Break" + "fieldtype": "Section Break", + "label": "Testing" }, { "description": "A Lead with this Email Address should exist", "fieldname": "test_email_id", "fieldtype": "Data", - "label": "Test Email Address" + "label": "Test Email Address", + "options": "Email" }, { + "depends_on": "eval: doc.test_email_id", "fieldname": "test_send", "fieldtype": "Button", "label": "Test", @@ -117,7 +125,8 @@ "depends_on": "eval: doc.schedule_sending", "fieldname": "schedule_send", "fieldtype": "Datetime", - "label": "Schedule Send" + "label": "Schedule Send", + "read_only_depends_on": "eval: doc.email_sent" }, { "default": "0", @@ -125,11 +134,32 @@ "fieldtype": "Check", "label": "Send Attachments" }, + { + "fieldname": "content_type", + "fieldtype": "Select", + "label": "Content Type", + "options": "Rich Text\nMarkdown\nHTML" + }, + { + "depends_on": "eval:doc.content_type === 'Markdown'", + "fieldname": "message_md", + "fieldtype": "Markdown Editor", + "label": "Message (Markdown)", + "mandatory_depends_on": "eval:doc.content_type === 'Markdown'" + }, + { + "depends_on": "eval:doc.content_type === 'HTML'", + "fieldname": "message_html", + "fieldtype": "HTML Editor", + "label": "Message (HTML)", + "mandatory_depends_on": "eval:doc.content_type === 'HTML'" + }, { "default": "0", "fieldname": "schedule_sending", "fieldtype": "Check", - "label": "Schedule Sending" + "label": "Schedule Sending", + "read_only_depends_on": "eval: doc.email_sent" } ], "has_web_view": 1, @@ -139,7 +169,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-08-17 18:11:59.541686", + "modified": "2020-08-24 19:59:37.262500", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 849c21f768..0a0a13a6ce 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -8,12 +8,9 @@ import frappe.utils from frappe import throw, _ from frappe.website.website_generator import WebsiteGenerator from frappe.utils.verified_command import get_signed_params, verify_request -from frappe.utils.background_jobs import enqueue from frappe.email.queue import send from frappe.email.doctype.email_group.email_group import add_subscribers -from frappe.utils import parse_addr, now_datetime -from frappe.utils import validate_email_address - +from frappe.utils import parse_addr, now_datetime, markdown, validate_email_address class Newsletter(WebsiteGenerator): def onload(self): @@ -29,8 +26,8 @@ class Newsletter(WebsiteGenerator): def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) - self.queue_all() - frappe.msgprint(_("Scheduled to send to {0}").format(self.test_email_id)) + self.queue_all(test_email=True) + frappe.msgprint(_("Test email sent to {0}").format(self.test_email_id)) def send_emails(self): """send emails to leads and customers""" @@ -40,21 +37,13 @@ class Newsletter(WebsiteGenerator): self.recipients = self.get_recipients() if self.recipients: - if getattr(frappe.local, "is_ajax", False): - self.validate_send() - # using default queue with a longer timeout as this isn't a scheduled task - enqueue(send_newsletter, queue='default', timeout=6000, event='send_newsletter', - newsletter=self.name) - - else: - self.queue_all() - - frappe.msgprint(_("Scheduled to send to {0} recipients").format(len(self.recipients))) + self.queue_all() + frappe.msgprint(_("Email queued to {0} recipients").format(len(self.recipients))) else: frappe.msgprint(_("Newsletter should have atleast one recipient")) - def queue_all(self): + def queue_all(self, test_email=False): if not self.get("recipients"): # in case it is called via worker self.recipients = self.get_recipients() @@ -80,7 +69,7 @@ class Newsletter(WebsiteGenerator): frappe.throw(_("Unable to find attachment {0}").format(file.name)) send(recipients=self.recipients, sender=sender, - subject=self.subject, message=self.message, + subject=self.subject, message=self.get_message(), reference_doctype=self.doctype, reference_name=self.name, add_unsubscribe_link=self.send_unsubscribe_link, attachments=attachments, unsubscribe_method="/unsubscribe", @@ -90,9 +79,18 @@ class Newsletter(WebsiteGenerator): if not frappe.flags.in_test: frappe.db.auto_commit_on_many_writes = False - self.db_set("email_sent", 1) - self.db_set("schedule_send", now_datetime()) - self.db_set("scheduled_to_send", len(self.recipients)) + if not test_email: + self.db_set("email_sent", 1) + self.db_set("schedule_send", now_datetime()) + self.db_set("scheduled_to_send", len(self.recipients)) + + def get_message(self): + + return { + 'Rich Text': self.message, + 'Markdown': markdown(self.message_md), + 'HTML': self.message_html + }[self.content_type] def get_recipients(self): """Get recipients from Email Group""" diff --git a/frappe/email/doctype/newsletter/newsletter_list.js b/frappe/email/doctype/newsletter/newsletter_list.js index e95d29545d..9ded6148e0 100644 --- a/frappe/email/doctype/newsletter/newsletter_list.js +++ b/frappe/email/doctype/newsletter/newsletter_list.js @@ -1,8 +1,10 @@ frappe.listview_settings['Newsletter'] = { - add_fields: ["subject", "email_sent"], + add_fields: ["subject", "email_sent", "schedule_sending"], get_indicator: function(doc) { - if(doc.email_sent) { + if (doc.email_sent) { return [__("Sent"), "green", "email_sent,=,Yes"]; + } else if (doc.schedule_sending) { + return [__("Scheduled"), "orange", "email_sent,=,No|schedule_sending,=,Yes"]; } else { return [__("Not Sent"), "orange", "email_sent,=,No"]; } diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index bb339165d3..ee7f123b7e 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -67,6 +67,7 @@ class TestNewsletter(unittest.TestCase): "doctype": "Newsletter", "subject": "_Test Newsletter", "send_from": "Test Sender ", + "content_type": "Rich Text", "message": "Testing my news.", "published": published, "schedule_sending": bool(schedule_send), diff --git a/frappe/patches.txt b/frappe/patches.txt index 9c9fd71661..35389eee43 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -306,3 +306,4 @@ frappe.patches.v13_0.add_toggle_width_in_navbar_settings frappe.patches.v13_0.rename_notification_fields frappe.patches.v13_0.remove_duplicate_navbar_items frappe.patches.v13_0.enable_custom_script +frappe.patches.v13_0.update_newsletter_content_type diff --git a/frappe/patches/v13_0/update_newsletter_content_type.py b/frappe/patches/v13_0/update_newsletter_content_type.py new file mode 100644 index 0000000000..0b32fb49ed --- /dev/null +++ b/frappe/patches/v13_0/update_newsletter_content_type.py @@ -0,0 +1,12 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + +def execute(): + frappe.reload_doc('email', 'doctype', 'Newsletter') + frappe.db.sql(""" + UPDATE tabNewsletter + SET content_type = 'Rich Text' + """) \ No newline at end of file From fda7999bdd6b67561e7ded71f12e9264f417dd27 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 15:48:49 +0530 Subject: [PATCH 34/52] fix: Send email notification by child table fields --- .../doctype/notification/notification.js | 22 ++++++++++++++----- .../doctype/notification/notification.py | 15 +++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index 454514f922..5eae016573 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -19,9 +19,11 @@ frappe.notification = { } frappe.model.with_doctype(frm.doc.document_type, function() { - let get_select_options = function(df) { + let get_select_options = function(df, parent_field) { + let select_value = parent_field ? df.fieldname + ',' + parent_field : df.fieldname; + return { - value: df.fieldname, + value: select_value, label: df.fieldname + ' (' + __(df.label) + ')' }; }; @@ -59,9 +61,19 @@ frappe.notification = { let receiver_fields = []; if (frm.doc.channel === 'Email') { receiver_fields = $.map(fields, function(d) { - return d.options == 'Email' || - (d.options == 'User' && d.fieldtype == 'Link') - ? get_select_options(d) : null; + + if (d.fieldtype == 'Table') { + let child_fields = frappe.get_doc('DocType', d.options).fields; + return $.map(child_fields, function(df) { + return df.options == 'Email' || + (df.options == 'User' && df.fieldtype == 'Link') + ? get_select_options(df, d.fieldname) : null; + }); + } else { + return d.options == 'Email' || + (d.options == 'User' && d.fieldtype == 'Link') + ? get_select_options(d) : null; + } }); } else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) { receiver_fields = $.map(fields, function(d) { diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index cfa3c43ff7..a51afbf7aa 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -235,10 +235,17 @@ def get_context(context): if not frappe.safe_eval(recipient.condition, None, context): continue if recipient.receiver_by_document_field: - email_ids_value = doc.get(recipient.receiver_by_document_field) - if validate_email_address(email_ids_value): - email_ids = email_ids_value.replace(",", "\n") - recipients = recipients + email_ids.split("\n") + fields = recipient.receiver_by_document_field.split(',') + if len(fields) > 1: + for d in doc.get(fields[1]): + email_id = d.get(fields[0]) + if validate_email_address(email_id): + recipients.append(email_id) + else: + email_ids_value = doc.get(fields[0]) + if validate_email_address(email_ids_value): + email_ids = email_ids_value.replace(",", "\n") + recipients = recipients + email_ids.split("\n") if recipient.cc and "{" in recipient.cc: recipient.cc = frappe.render_template(recipient.cc, context) From 07728df16af98858accc34cb339bce6e92ebc26f Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 15:49:55 +0530 Subject: [PATCH 35/52] fix: Add test cases --- .../doctype/notification/test_notification.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 9bdf09375d..71d46cbdfa 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import frappe, frappe.utils, frappe.utils.scheduler +from frappe.desk.form import assign_to import unittest test_records = frappe.get_test_records('Notification') @@ -177,3 +178,92 @@ class TestNotification(unittest.TestCase): frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""") frappe.db.sql("""delete from `tabEmail Queue`""") frappe.db.sql("""delete from `tabEmail Queue Recipient`""") + + def test_notification_to_assignee(self): + frappe.set_user("Administrator") + + todo = frappe.new_doc('ToDo') + todo.description = 'Test Notification' + todo.save() + + assign_to.add({ + "assign_to": ["test2@example.com"], + "doctype": todo.doctype, + "name": todo.name, + "description": "Close this Todo" + }) + + assign_to.add({ + "assign_to": ["test1@example.com"], + "doctype": todo.doctype, + "name": todo.name, + "description": "Close this Todo" + }) + + if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'): + notification = frappe.new_doc('Notification') + notification.name = 'ToDo Status Update' + notification.subject = 'ToDo Status Update' + notification.document_type = 'ToDo' + notification.event = 'Value Change' + notification.value_changed = 'status' + notification.send_to_all_assignees = 1 + notification.save() + + #change status of todo + todo.status = 'Closed' + todo.save() + + email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'ToDo', + 'reference_name': todo.name}) + + self.assertTrue(email_queue) + + recipients = [d.recipient for d in email_queue.recipients] + self.assertTrue('test2@example.com' in recipients) + self.assertTrue('test1@example.com' in recipients) + + def test_notification_by_child_table_field(self): + frappe.set_user("Administrator") + + if not frappe.db.exists('Notification', {'name': 'Contact Status Update'}, 'name'): + notification = frappe.new_doc('Notification') + notification.name = 'Contact Status Update' + notification.subject = 'Contact Status Update' + notification.document_type = 'Contact' + notification.event = 'Value Change' + notification.value_changed = 'status' + notification.message = 'Test Contact Update' + notification.append('recipients', { + 'receiver_by_document_field': 'email_id,email_ids' + }) + notification.save() + + contact = frappe.new_doc('Contact') + contact.first_name = 'John Doe' + contact.status = 'Open' + contact.append('email_ids', { + 'email_id': 'test2@example.com', + 'is_primary': 1 + }) + + contact.append('email_ids', { + 'email_id': 'test1@example.com' + }) + + contact.save() + + #change status of todo + contact.status = 'Replied' + contact.save() + + email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'Contact', + 'reference_name': contact.name}) + + self.assertTrue(email_queue) + + recipients = [d.recipient for d in email_queue.recipients] + self.assertTrue('test2@example.com' in recipients) + self.assertTrue('test1@example.com' in recipients) + + From 6ef4536c697885d1fdd617edafd30ab21733b578 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 16:20:32 +0530 Subject: [PATCH 36/52] fix: Update comment --- frappe/email/doctype/notification/test_notification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 71d46cbdfa..e370bac9e5 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -253,7 +253,7 @@ class TestNotification(unittest.TestCase): contact.save() - #change status of todo + #change status of contact contact.status = 'Replied' contact.save() From 87c06ef5a49a86150680f9102f81f0883a567285 Mon Sep 17 00:00:00 2001 From: Anupam K Date: Wed, 2 Sep 2020 17:19:00 +0530 Subject: [PATCH 37/52] fix: tabs --- frappe/patches/v13_0/update_newsletter_content_type.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/patches/v13_0/update_newsletter_content_type.py b/frappe/patches/v13_0/update_newsletter_content_type.py index 0b32fb49ed..01f4bacd16 100644 --- a/frappe/patches/v13_0/update_newsletter_content_type.py +++ b/frappe/patches/v13_0/update_newsletter_content_type.py @@ -5,8 +5,8 @@ from __future__ import unicode_literals import frappe def execute(): - frappe.reload_doc('email', 'doctype', 'Newsletter') - frappe.db.sql(""" - UPDATE tabNewsletter - SET content_type = 'Rich Text' - """) \ No newline at end of file + frappe.reload_doc('email', 'doctype', 'Newsletter') + frappe.db.sql(""" + UPDATE tabNewsletter + SET content_type = 'Rich Text' + """) \ No newline at end of file From b373f10432cc5be1df2f5922eeae1eaecd227890 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 20:00:36 +0530 Subject: [PATCH 38/52] fix: Add sleep to fix test cases --- frappe/email/doctype/notification/test_notification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index e370bac9e5..5b8ceb03d4 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals import frappe, frappe.utils, frappe.utils.scheduler from frappe.desk.form import assign_to import unittest +import time test_records = frappe.get_test_records('Notification') @@ -214,6 +215,8 @@ class TestNotification(unittest.TestCase): todo.status = 'Closed' todo.save() + # adding sleep so that email queue is fetched once its created + time.sleep(10) email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'ToDo', 'reference_name': todo.name}) @@ -257,6 +260,8 @@ class TestNotification(unittest.TestCase): contact.status = 'Replied' contact.save() + # adding sleep so that email queue is fetched once its created + time.sleep(10) email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'Contact', 'reference_name': contact.name}) From b65e6588282eb29843b91cb6a868ccb945751f93 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 20:42:43 +0530 Subject: [PATCH 39/52] fix: Move notification creation in setup --- .../doctype/notification/test_notification.py | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 5b8ceb03d4..45a1587c1a 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -6,7 +6,6 @@ from __future__ import unicode_literals import frappe, frappe.utils, frappe.utils.scheduler from frappe.desk.form import assign_to import unittest -import time test_records = frappe.get_test_records('Notification') @@ -15,7 +14,31 @@ test_dependencies = ["User"] class TestNotification(unittest.TestCase): def setUp(self): frappe.db.sql("""delete from `tabEmail Queue`""") - frappe.set_user("test1@example.com") + frappe.set_user("test@example.com") + + if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'): + notification = frappe.new_doc('Notification') + notification.name = 'ToDo Status Update' + notification.subject = 'ToDo Status Update' + notification.document_type = 'ToDo' + notification.event = 'Value Change' + notification.value_changed = 'status' + notification.send_to_all_assignees = 1 + notification.save() + + if not frappe.db.exists('Notification', {'name': 'Contact Status Update'}, 'name'): + notification = frappe.new_doc('Notification') + notification.name = 'Contact Status Update' + notification.subject = 'Contact Status Update' + notification.document_type = 'Contact' + notification.event = 'Value Change' + notification.value_changed = 'status' + notification.message = 'Test Contact Update' + notification.append('recipients', { + 'receiver_by_document_field': 'email_id,email_ids' + }) + notification.save() + def tearDown(self): frappe.set_user("Administrator") @@ -181,8 +204,6 @@ class TestNotification(unittest.TestCase): frappe.db.sql("""delete from `tabEmail Queue Recipient`""") def test_notification_to_assignee(self): - frappe.set_user("Administrator") - todo = frappe.new_doc('ToDo') todo.description = 'Test Notification' todo.save() @@ -201,22 +222,10 @@ class TestNotification(unittest.TestCase): "description": "Close this Todo" }) - if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'): - notification = frappe.new_doc('Notification') - notification.name = 'ToDo Status Update' - notification.subject = 'ToDo Status Update' - notification.document_type = 'ToDo' - notification.event = 'Value Change' - notification.value_changed = 'status' - notification.send_to_all_assignees = 1 - notification.save() - #change status of todo todo.status = 'Closed' todo.save() - # adding sleep so that email queue is fetched once its created - time.sleep(10) email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'ToDo', 'reference_name': todo.name}) @@ -227,21 +236,6 @@ class TestNotification(unittest.TestCase): self.assertTrue('test1@example.com' in recipients) def test_notification_by_child_table_field(self): - frappe.set_user("Administrator") - - if not frappe.db.exists('Notification', {'name': 'Contact Status Update'}, 'name'): - notification = frappe.new_doc('Notification') - notification.name = 'Contact Status Update' - notification.subject = 'Contact Status Update' - notification.document_type = 'Contact' - notification.event = 'Value Change' - notification.value_changed = 'status' - notification.message = 'Test Contact Update' - notification.append('recipients', { - 'receiver_by_document_field': 'email_id,email_ids' - }) - notification.save() - contact = frappe.new_doc('Contact') contact.first_name = 'John Doe' contact.status = 'Open' @@ -260,8 +254,6 @@ class TestNotification(unittest.TestCase): contact.status = 'Replied' contact.save() - # adding sleep so that email queue is fetched once its created - time.sleep(10) email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'Contact', 'reference_name': contact.name}) From 0bdb4c8b4e00e37a5353561419dff82337fe3f97 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Wed, 2 Sep 2020 21:12:26 +0530 Subject: [PATCH 40/52] fix: Linting issues --- frappe/email/doctype/notification/notification.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index 5eae016573..cb49232906 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -65,7 +65,7 @@ frappe.notification = { if (d.fieldtype == 'Table') { let child_fields = frappe.get_doc('DocType', d.options).fields; return $.map(child_fields, function(df) { - return df.options == 'Email' || + return df.options == 'Email' || (df.options == 'User' && df.fieldtype == 'Link') ? get_select_options(df, d.fieldname) : null; }); From 66934b311fd9a0275252b71f7b43c61f7cc1ae22 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 22:18:05 +0530 Subject: [PATCH 41/52] chore(deps): [security] bump bl from 3.0.0 to 3.0.1 (#11403) Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- yarn.lock | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index 41fd1926c2..e5c975c357 100644 --- a/yarn.lock +++ b/yarn.lock @@ -937,9 +937,9 @@ big.js@^3.1.3: integrity sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q== bl@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.0.tgz#3611ec00579fd18561754360b21e9f784500ff88" - integrity sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A== + version "3.0.1" + resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.1.tgz#1cbb439299609e419b5a74d7fce2f8b37d8e5c6f" + integrity sha512-jrCW5ZhfQ/Vt07WX1Ngs+yn9BDqPL/gw28S7s9H6QK/gupnizNzJAss5akW20ISgOrbLTlXOOCTJeNUQqruAWQ== dependencies: readable-stream "^3.0.1" @@ -6157,7 +6157,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@~2.3.6: +readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@^2.3.5, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -6170,19 +6170,6 @@ readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stre string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.3.5: - version "2.3.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" - integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - readable-stream@^3.0.1, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" @@ -6612,11 +6599,16 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@^5.0.1, safe-buffer@^5.1.2: version "5.2.0" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== +safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + safe-regex@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" From 9eb903038144e6823f7857d180788a01e33b0da8 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 3 Sep 2020 08:42:42 +0530 Subject: [PATCH 42/52] fix(minor): move get_source_value to data_migration_mapper --- .../data_migration_mapping/data_migration_mapping.py | 8 +++++++- .../doctype/data_migration_run/data_migration_run.py | 3 ++- frappe/utils/data.py | 7 ------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py index e89282885f..b346864f02 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document -from frappe.utils import get_source_value class DataMigrationMapping(Document): def get_filters(self): @@ -70,3 +69,10 @@ def get_value_from_fieldname(field_map, fieldname_field, doc): else: value = get_source_value(doc, field_name) return value + +def get_source_value(source, key): + '''Get value from source (object or dict) based on key''' + if isinstance(source, dict): + return source.get(key) + else: + return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index b2ce4606f8..473acfb3d0 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -6,7 +6,8 @@ from __future__ import unicode_literals import frappe, json, math from frappe.model.document import Document from frappe import _ -from frappe.utils import get_source_value, cstr +from frappe.utils import cstr +from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import get_source_value class DataMigrationRun(Document): def run(self): diff --git a/frappe/utils/data.py b/frappe/utils/data.py index fd5c838b57..8dad5f3e92 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1226,13 +1226,6 @@ def md_to_html(markdown_text): return html -def get_source_value(source, key): - '''Get value from source (object or dict) based on key''' - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) - def is_subset(list_a, list_b): '''Returns whether list_a is a subset of list_b''' return len(list(set(list_a) & set(list_b))) == len(list_a) From 4d91d72d1ace10725cb74a146af911e6b6275d45 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 3 Sep 2020 09:00:58 +0530 Subject: [PATCH 43/52] refactor: Client-side get_role_permissions --- frappe/public/js/frappe/form/form.js | 11 +++---- frappe/public/js/frappe/model/perm.js | 43 ++++++++++++--------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index ff48ad2f60..cae0cb3c4a 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1405,19 +1405,16 @@ frappe.ui.form.Form = class FrappeForm { } set_read_only() { - var perm = []; - var docperms = frappe.perm.get_perm(this.doc.doctype); - for (var i=0, l=docperms.length; i { + return { read: p.read, cancel: p.cancel, share: p.share, print: p.print, email: p.email }; - } - this.perm = perm; + }); } trigger(event, doctype, docname) { diff --git a/frappe/public/js/frappe/model/perm.js b/frappe/public/js/frappe/model/perm.js index f30368bbc3..2a1f52fbfb 100644 --- a/frappe/public/js/frappe/model/perm.js +++ b/frappe/public/js/frappe/model/perm.js @@ -42,7 +42,7 @@ $.extend(frappe.perm, { }, get_perm: (doctype, doc) => { - let perm = [{ read: 0 }]; + let perm = [{ read: 0, permlevel: 0 }]; let meta = frappe.get_doc("DocType", doctype); const user = frappe.session.user; @@ -53,7 +53,7 @@ $.extend(frappe.perm, { if (!meta) return perm; - frappe.perm.build_role_permissions(perm, meta); + perm = frappe.perm.get_role_permissions(meta); if (doc) { // apply user permissions via docinfo (which is processed server-side) @@ -107,35 +107,30 @@ $.extend(frappe.perm, { return perm; }, - build_role_permissions: (perm, meta) => { + get_role_permissions: (meta) => { + let perm = [{ read: 0, permlevel: 0 }]; // Returns a `dict` of evaluated Role Permissions - $.each(meta.permissions || [], (i, p) => { + (meta.permissions || []).forEach(p => { // if user has this role - if (frappe.user_roles.includes(p.role)) { - let permlevel = cint(p.permlevel); - if (!perm[permlevel]) { - perm[permlevel] = {}; - perm[permlevel]["permlevel"] = permlevel - } + let permlevel = cint(p.permlevel); + if (!perm[permlevel]) { + perm[permlevel] = {}; + perm[permlevel]["permlevel"] = permlevel; + } - $.each(frappe.perm.rights, (i, key) => { - perm[permlevel][key] = perm[permlevel][key] || (p[key] || 0); + if (frappe.user_roles.includes(p.role)) { + frappe.perm.rights.forEach(right => { + let value = perm[permlevel][right] || (p[right] || 0); + if (value) { + perm[permlevel][right] = value; + } }); } }); - // remove values with 0 - $.each(perm[0], (key, val) => { - if (!val) { - delete perm[0][key]; - } - }); - - $.each(perm, (i, v) => { - if (v === undefined) { - perm[i] = {}; - } - }); + // fill gaps with empty object + perm = perm.map(p => p || {}); + return perm; }, get_match_rules: (doctype, ptype) => { From ba9c7bdc09b658238391c913b45922b06340a57c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 3 Sep 2020 13:55:55 +0530 Subject: [PATCH 44/52] fix: Also check if name is set, since empty object is truthy --- frappe/public/js/frappe/form/controls/data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/data.js b/frappe/public/js/frappe/form/controls/data.js index 355f35891a..bbf9a89072 100644 --- a/frappe/public/js/frappe/form/controls/data.js +++ b/frappe/public/js/frappe/form/controls/data.js @@ -61,7 +61,7 @@ frappe.ui.form.ControlData = frappe.ui.form.ControlInput.extend({ // check if name exists frappe.db.get_value(this.doctype, this.$input.val(), 'name', (val) => { - if (val) { + if (val && val.name) { this.set_description(__('{0} already exists. Select another name', [val.name])); } }, From 5b449d1084abba1016940bd49975b38225ed9e9e Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Thu, 3 Sep 2020 14:24:19 +0530 Subject: [PATCH 45/52] chore: Add comments --- frappe/email/doctype/notification/notification.js | 3 +++ frappe/email/doctype/notification/notification.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index cb49232906..059f5518fc 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -20,6 +20,7 @@ frappe.notification = { frappe.model.with_doctype(frm.doc.document_type, function() { let get_select_options = function(df, parent_field) { + // Append parent_field name along with fieldname for child table fields let select_value = parent_field ? df.fieldname + ',' + parent_field : df.fieldname; return { @@ -62,6 +63,7 @@ frappe.notification = { if (frm.doc.channel === 'Email') { receiver_fields = $.map(fields, function(d) { + // Add User and Email fields from child into select dropdown if (d.fieldtype == 'Table') { let child_fields = frappe.get_doc('DocType', d.options).fields; return $.map(child_fields, function(df) { @@ -69,6 +71,7 @@ frappe.notification = { (df.options == 'User' && df.fieldtype == 'Link') ? get_select_options(df, d.fieldname) : null; }); + // Add User and Email fields from parent into select dropdown } else { return d.options == 'Email' || (d.options == 'User' && d.fieldtype == 'Link') diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index a51afbf7aa..cd5aacabbd 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -236,11 +236,13 @@ def get_context(context): continue if recipient.receiver_by_document_field: fields = recipient.receiver_by_document_field.split(',') + # fields from child table if len(fields) > 1: for d in doc.get(fields[1]): email_id = d.get(fields[0]) if validate_email_address(email_id): recipients.append(email_id) + # field from parent doc else: email_ids_value = doc.get(fields[0]) if validate_email_address(email_ids_value): From 8c8f7313f41d9d6e350295403d0a21c59537e6b4 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 3 Sep 2020 14:26:01 +0530 Subject: [PATCH 46/52] fix(minor): make utils explicit in safe_globals (#11408) * fix(minor): make utils explicit in safe_globals * fix(minor): import subprocess * fix(minor): fix globals in safe_eval; * fix(minor): import subprocess * fix(minor): add test * fix(minor): webhook.py * fix(minor): document_type_mapping.py --- .../data_migration_mapping.py | 3 +- .../document_type_mapping.py | 2 +- .../integrations/doctype/webhook/webhook.py | 4 +- frappe/tests/test_safe_exec.py | 11 +- frappe/utils/data.py | 15 --- frappe/utils/pdf.py | 17 ++- frappe/utils/safe_exec.py | 110 +++++++++++++++++- 7 files changed, 139 insertions(+), 23 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py index b346864f02..097652fb88 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py @@ -5,11 +5,12 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document +from frappe.utils.safe_exec import get_safe_globals class DataMigrationMapping(Document): def get_filters(self): if self.condition: - return frappe.safe_eval(self.condition, dict(frappe=frappe)) + return frappe.safe_eval(self.condition, get_safe_globals()) def get_fields(self): fields = [] diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index 8ace4f57d3..bf96e4e27b 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -102,7 +102,7 @@ class DocumentTypeMapping(Document): filters = json.loads(mapping.remote_value_filters) for key, value in iteritems(filters): if value.startswith('eval:'): - val = frappe.safe_eval(value[5:], dict(frappe=frappe)) + val = frappe.safe_eval(value[5:], None, dict(doc=doc)) filters[key] = val if doc.get(value): filters[key] = doc.get(value) diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index 8e6c8d58e4..f1556aa661 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -18,6 +18,7 @@ import frappe from frappe import _ from frappe.model.document import Document from frappe.utils.jinja import validate_template +from frappe.utils.safe_exec import get_safe_globals WEBHOOK_SECRET_HEADER = "X-Frappe-Webhook-Signature" @@ -75,8 +76,7 @@ class Webhook(Document): def get_context(doc): - return {"doc": doc, "utils": frappe.utils} - + return {'doc': doc, 'utils': get_safe_globals().get('frappe').get('utils')} def enqueue_webhook(doc, webhook): webhook = frappe.get_doc("Webhook", webhook.get("name")) diff --git a/frappe/tests/test_safe_exec.py b/frappe/tests/test_safe_exec.py index 95bbae6746..d7b25b8194 100644 --- a/frappe/tests/test_safe_exec.py +++ b/frappe/tests/test_safe_exec.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals import unittest, frappe -from frappe.utils.safe_exec import safe_exec +from frappe.utils.safe_exec import safe_exec, get_safe_globals class TestSafeExec(unittest.TestCase): def test_import_fails(self): @@ -9,6 +9,15 @@ class TestSafeExec(unittest.TestCase): def test_internal_attributes(self): self.assertRaises(SyntaxError, safe_exec, '().__class__.__call__') + def test_utils(self): + _locals = dict(out=None) + safe_exec('''out = frappe.utils.cint("1")''', None, _locals) + self.assertEqual(_locals['out'], 1) + + def test_safe_eval(self): + self.assertEqual(frappe.safe_eval('1+1'), 2) + self.assertRaises(AttributeError, frappe.safe_eval, 'frappe.utils.os.path', get_safe_globals()) + def test_sql(self): _locals = dict(out=None) safe_exec('''out = frappe.db.sql("select name from tabDocType where name='DocType'")''', None, _locals) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8dad5f3e92..8a25c84276 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -3,10 +3,8 @@ from __future__ import unicode_literals -# IMPORTANT: only import safe functions as this module will be included in jinja environment import frappe from dateutil.parser._parser import ParserError -import subprocess import operator import json import re, datetime, math, time @@ -427,19 +425,6 @@ def flt(s, precision=None): return num -def get_wkhtmltopdf_version(): - wkhtmltopdf_version = frappe.cache().hget("wkhtmltopdf_version", None) - - if not wkhtmltopdf_version: - try: - res = subprocess.check_output(["wkhtmltopdf", "--version"]) - wkhtmltopdf_version = res.decode('utf-8').split(" ")[1] - frappe.cache().hset("wkhtmltopdf_version", None, wkhtmltopdf_version) - except Exception: - pass - - return (wkhtmltopdf_version or '0') - def cint(s): """Convert to integer""" try: num = int(float(s)) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index f3d2e75c2b..51febb5f72 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -6,6 +6,7 @@ import io import os import re from distutils.version import LooseVersion +import subprocess import pdfkit import six @@ -14,7 +15,7 @@ from PyPDF2 import PdfFileReader, PdfFileWriter import frappe from frappe import _ -from frappe.utils import get_wkhtmltopdf_version, scrub_urls +from frappe.utils import scrub_urls PDF_CONTENT_ERRORS = ["ContentNotFoundError", "ContentOperationNotPermittedError", @@ -191,7 +192,6 @@ def cleanup(fname, options): if options.get(key) and os.path.exists(options[key]): os.remove(options[key]) - def toggle_visible_pdf(soup): for tag in soup.find_all(attrs={"class": "visible-pdf"}): # remove visible-pdf class to unhide @@ -200,3 +200,16 @@ def toggle_visible_pdf(soup): for tag in soup.find_all(attrs={"class": "hidden-pdf"}): # remove tag from html tag.extract() + +def get_wkhtmltopdf_version(): + wkhtmltopdf_version = frappe.cache().hget("wkhtmltopdf_version", None) + + if not wkhtmltopdf_version: + try: + res = subprocess.check_output(["wkhtmltopdf", "--version"]) + wkhtmltopdf_version = res.decode('utf-8').split(" ")[1] + frappe.cache().hset("wkhtmltopdf_version", None, wkhtmltopdf_version) + except Exception: + pass + + return (wkhtmltopdf_version or '0') diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 738ce2ba0f..12382e85cd 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -39,7 +39,7 @@ def get_safe_globals(): date_format = "yyyy-mm-dd" time_format = "HH:mm:ss" - add_module_properties(frappe.utils.data, datautils, lambda obj: hasattr(obj, "__call__")) + add_data_utils(datautils) if "_" in getattr(frappe.local, 'form_dict', {}): del frappe.local.form_dict["_"] @@ -162,6 +162,11 @@ def _write(obj): # allow writing to any object return obj +def add_data_utils(data): + for key, obj in frappe.utils.data.__dict__.items(): + if key in VALID_UTILS: + data[key] = obj + def add_module_properties(module, data, filter_method): for key, obj in module.__dict__.items(): if key.startswith("_"): @@ -171,3 +176,106 @@ def add_module_properties(module, data, filter_method): if filter_method(obj): # only allow functions data[key] = obj + +VALID_UTILS = ( +"DATE_FORMAT", +"TIME_FORMAT", +"DATETIME_FORMAT", +"is_invalid_date_string", +"getdate", +"get_datetime", +"to_timedelta", +"add_to_date", +"add_days", +"add_months", +"add_years", +"date_diff", +"month_diff", +"time_diff", +"time_diff_in_seconds", +"time_diff_in_hours", +"now_datetime", +"get_timestamp", +"get_eta", +"get_time_zone", +"convert_utc_to_user_timezone", +"now", +"nowdate", +"today", +"nowtime", +"get_first_day", +"get_quarter_start", +"get_first_day_of_week", +"get_year_start", +"get_last_day_of_week", +"get_last_day", +"get_time", +"get_datetime_str", +"get_date_str", +"get_time_str", +"get_user_date_format", +"get_user_time_format", +"format_date", +"format_time", +"format_datetime", +"format_duration", +"get_weekdays", +"get_weekday", +"get_timespan_date_range", +"global_date_format", +"has_common", +"flt", +"cint", +"floor", +"ceil", +"cstr", +"rounded", +"remainder", +"safe_div", +"round_based_on_smallest_currency_fraction", +"encode", +"parse_val", +"fmt_money", +"get_number_format_info", +"money_in_words", +"in_words", +"is_html", +"is_image", +"get_thumbnail_base64_for_image", +"image_to_base64", +"strip_html", +"escape_html", +"pretty_date", +"comma_or", +"comma_and", +"comma_sep", +"new_line_sep", +"filter_strip_join", +"get_url", +"get_host_name_from_request", +"url_contains_port", +"get_host_name", +"get_link_to_form", +"get_link_to_report", +"get_absolute_url", +"get_url_to_form", +"get_url_to_list", +"get_url_to_report", +"get_url_to_report_with_filters", +"evaluate_filters", +"compare", +"get_filter", +"make_filter_tuple", +"make_filter_dict", +"sanitize_column", +"scrub_urls", +"expand_relative_urls", +"quoted", +"quote_urls", +"unique", +"strip", +"to_markdown", +"md_to_html", +"is_subset", +"generate_hash" +) \ No newline at end of file From dffc0ca5b7fc230fbb9925efbc0e85838b036007 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 4 Sep 2020 13:15:12 +0530 Subject: [PATCH 47/52] fix(minor): Use get_safe_globals --- .../doctype/data_migration_mapping/data_migration_mapping.py | 2 +- frappe/email/doctype/notification/notification.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py index 097652fb88..1cc54a0d1a 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py @@ -64,7 +64,7 @@ def get_value_from_fieldname(field_map, fieldname_field, doc): field_name = get_source_value(field_map, fieldname_field) if field_name.startswith('eval:'): - value = frappe.safe_eval(field_name[5:], dict(frappe=frappe)) + value = frappe.safe_eval(field_name[5:], get_safe_globals()) elif field_name[0] in ('"', "'"): value = field_name[1:-1] else: diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index cd5aacabbd..5355d56b35 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -10,6 +10,7 @@ from frappe.model.document import Document from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date from frappe.utils.jinja import validate_template +from frappe.utils.safe_exec import get_safe_globals from frappe.modules.utils import export_module_json, get_doc_module from six import string_types from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message @@ -415,7 +416,7 @@ def evaluate_alert(doc, alert, event): frappe.utils.get_link_to_form('Error Log', error_log.name))) def get_context(doc): - return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=frappe.utils)} + return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=get_safe_globals().get("frappe").get("utils"))} def get_assignees(doc): assignees = [] From 3349c9f9426b9ad6f346abea28b894baa1c78cf9 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 4 Sep 2020 16:28:50 +0530 Subject: [PATCH 48/52] fix(minor): Prevent path traversal --- frappe/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8a25c84276..e9593ff89e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -738,7 +738,7 @@ def get_thumbnail_base64_for_image(src): if not src: frappe.throw('Invalid source for image: {0}'.format(src)) - if not src.startswith('/files'): + if not src.startswith('/files') or '..' in src: return def _get_base64(): From d94a1313f068b48a1f76759822aa07400cff572d Mon Sep 17 00:00:00 2001 From: prssanna Date: Sat, 5 Sep 2020 13:14:12 +0530 Subject: [PATCH 49/52] fix: catch exception while setting In-Reply-To in email header --- frappe/email/email_body.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index d545190c47..6d0c6f8f33 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -207,7 +207,11 @@ class EMail: def set_in_reply_to(self, in_reply_to): """Used to send the Message-Id of a received email back as In-Reply-To""" - self.msg_root["In-Reply-To"] = in_reply_to + try: + self.msg_root["In-Reply-To"] = in_reply_to + except ValueError: + # in_reply_to may contain line feed characters, so ignore in that case + pass def make(self): """build into msg_root""" From e48ed816c3b2802b389c231f1fb8d902f22048e8 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Sun, 6 Sep 2020 16:35:28 +0530 Subject: [PATCH 50/52] fix(Event Streaming): remove in_test flag from Event Consumer --- .../doctype/event_consumer/event_consumer.json | 13 ++----------- .../doctype/event_consumer/event_consumer.py | 7 ++++--- .../doctype/event_producer/event_producer.py | 5 +---- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.json b/frappe/event_streaming/doctype/event_consumer/event_consumer.json index d863677e03..85970dc277 100644 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.json +++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.json @@ -13,8 +13,7 @@ "api_secret", "column_break_6", "user", - "incoming_change", - "in_test" + "incoming_change" ], "fields": [ { @@ -60,14 +59,6 @@ "label": "Incoming Change", "read_only": 1 }, - { - "default": "0", - "fieldname": "in_test", - "fieldtype": "Check", - "hidden": 1, - "label": "In Test", - "read_only": 1 - }, { "fieldname": "consumer_doctypes", "fieldtype": "Table", @@ -78,7 +69,7 @@ ], "in_create": 1, "links": [], - "modified": "2019-12-30 11:52:16.276047", + "modified": "2020-09-06 15:42:00.746493", "modified_by": "Administrator", "module": "Event Streaming", "name": "Event Consumer", diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py index a53d046be5..2e10c71d0d 100644 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals import frappe import json import requests +import os from frappe.model.document import Document from frappe.frappeclient import FrappeClient from frappe.utils.data import get_url @@ -14,10 +15,11 @@ from frappe.utils.background_jobs import get_jobs class EventConsumer(Document): def validate(self): - if self.in_test: + # approve subscribed doctypes for tests + # frappe.flags.in_test won't work here as tests are running on the consumer site + if os.environ.get('CI'): for entry in self.consumer_doctypes: entry.status = 'Approved' - self.in_test = False def on_update(self): if not self.incoming_change: @@ -80,7 +82,6 @@ def register_consumer(data): api_secret = frappe.generate_hash(length=10) consumer.api_key = api_key consumer.api_secret = api_secret - consumer.in_test = data['in_test'] consumer.insert(ignore_permissions=True) frappe.db.commit() diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index 73aea114ab..555b71f851 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -75,8 +75,7 @@ class EventProducer(Document): return { 'event_consumer': get_url(), 'consumer_doctypes': json.dumps(consumer_doctypes), - 'user': self.user, - 'in_test': frappe.flags.in_test + 'user': self.user } def create_custom_fields(self): @@ -110,8 +109,6 @@ class EventProducer(Document): 'status': get_approval_status(config, ref_doctype), 'unsubscribed': entry.unsubscribe }) - if frappe.flags.in_test: - event_consumer.in_test = True event_consumer.user = self.user event_consumer.incoming_change = True producer_site.update(event_consumer) From 3f5a50d677e7a3bc3a58e32377956689904e1a00 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 6 Sep 2020 18:37:22 +0530 Subject: [PATCH 51/52] fix: Restrict ui test helper method to dev server mode --- frappe/__init__.py | 1 + frappe/tests/ui_test_helpers.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/frappe/__init__.py b/frappe/__init__.py index 36a8b48ecd..46792e82a8 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -182,6 +182,7 @@ def init(site, sites_path=None, new_site=False): local.meta_cache = {} local.form_dict = _dict() local.session = _dict() + local.dev_server = os.environ.get('DEV_SERVER', False) setup_module_map() diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 40ebc8ea6e..ef572c6971 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -9,6 +9,9 @@ def create_if_not_exists(doc): :param doc: dict of field value pairs. can be a list of dict for multiple records. ''' + if not frappe.local.dev_server: + frappe.throw('This method can only be accessed in development', frappe.PermissionError) + doc = frappe.parse_json(doc) if not isinstance(doc, list): From 4f5acf3d9b31609d60e12e0220790ff07f7b29e9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 09:21:07 +0530 Subject: [PATCH 52/52] chore(deps): [security] bump js-yaml from 3.12.2 to 3.14.0 (#11413) Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 3.12.2 to 3.14.0. **This update includes security fixes.** - [Release notes](https://github.com/nodeca/js-yaml/releases) - [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/3.12.2...3.14.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- yarn.lock | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/yarn.lock b/yarn.lock index e5c975c357..23bfe25255 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3965,7 +3965,7 @@ js-tokens@^4.0.0: resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@^3.10.0: +js-yaml@^3.10.0, js-yaml@^3.12.0, js-yaml@^3.13.1, js-yaml@^3.9.0: version "3.14.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482" integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== @@ -3973,22 +3973,6 @@ js-yaml@^3.10.0: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^3.12.0, js-yaml@^3.9.0: - version "3.12.2" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.2.tgz#ef1d067c5a9d9cb65bd72f285b5d8105c77f14fc" - integrity sha512-QHn/Lh/7HhZ/Twc7vJYQTkjuCa0kaCcDcjK5Zlk2rvnUpy7DxMJ23+Jc2dcyvltwQVg1nygAVlB2oRDFHoRS5Q== - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - -js-yaml@^3.13.1: - version "3.13.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.13.1.tgz#aff151b30bfdfa8e49e05da22e7415e9dfa37847" - integrity sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw== - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - jsbarcode@^3.9.0: version "3.11.0" resolved "https://registry.yarnpkg.com/jsbarcode/-/jsbarcode-3.11.0.tgz#20623e008b101ef45d0cce9c8022cdf49be28547"