diff --git a/frappe/__init__.py b/frappe/__init__.py index a68c32fe03..6b50f8ab28 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -763,7 +763,7 @@ def get_doc(*args, **kwargs): # insert a new document todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}) - tood.insert() + todo.insert() # open an existing document todo = frappe.get_doc("ToDo", "TD0001") diff --git a/frappe/auth.py b/frappe/auth.py index 998e97fe24..7902469c14 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -340,6 +340,11 @@ class CookieManager: def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): if not secure and hasattr(frappe.local, 'request'): secure = frappe.local.request.scheme == "https" + + # Cordova does not work with Lax + if frappe.local.session.data.device == "mobile": + samesite = None + self.cookies[key] = { "value": value, "expires": expires, diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 26eb455338..b72d98c433 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -274,8 +274,9 @@ def disable_user(context, email): @click.command('migrate') @click.option('--rebuild-website', help="Rebuild webpages after migration") @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run") +@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents") @pass_context -def migrate(context, rebuild_website=False, skip_failing=False): +def migrate(context, rebuild_website=False, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" from frappe.migrate import migrate @@ -284,7 +285,12 @@ def migrate(context, rebuild_website=False, skip_failing=False): frappe.init(site=site) frappe.connect() try: - migrate(context.verbose, rebuild_website=rebuild_website, skip_failing=skip_failing) + migrate( + context.verbose, + rebuild_website=rebuild_website, + skip_failing=skip_failing, + skip_search_index=skip_search_index + ) finally: frappe.destroy() if not context.sites: @@ -655,6 +661,22 @@ def start_ngrok(context): frappe.destroy() ngrok.kill() +@click.command('build-search-index') +@pass_context +def build_search_index(context): + from frappe.search.website_search import build_index_for_all_routes + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + + print('Building search index for {}'.format(site)) + frappe.init(site=site) + frappe.connect() + try: + build_index_for_all_routes() + finally: + frappe.destroy() + commands = [ add_system_manager, backup, @@ -680,5 +702,6 @@ commands = [ start_recording, stop_recording, add_to_hosts, - start_ngrok + start_ngrok, + build_search_index ] diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index bc2962ab3f..116fc5caf5 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -3,17 +3,26 @@ # For license information, please see license.txt from __future__ import unicode_literals -import frappe, json +import frappe +import json +from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document from frappe import _ + class DeletedDocument(Document): pass + @frappe.whitelist() -def restore(name): +def restore(name, alert=True): deleted = frappe.get_doc('Deleted Document', name) + + if deleted.restored: + frappe.throw(_("Document {0} Already Restored").format(name), exc=frappe.DocumentAlreadyRestored) + doc = frappe.get_doc(json.loads(deleted.data)) + try: doc.insert() except frappe.DocstatusTransitionError: @@ -27,4 +36,34 @@ def restore(name): deleted.restored = 1 deleted.db_update() - frappe.msgprint(_('Document Restored')) + if alert: + frappe.msgprint(_('Document Restored')) + + +@frappe.whitelist() +def bulk_restore(docnames): + docnames = frappe.parse_json(docnames) + message = _('Restoring Deleted Document') + restored, invalid, failed = [], [], [] + + for i, d in enumerate(docnames): + try: + show_progress(docnames, message, i + 1, d) + restore(d, alert=False) + frappe.db.commit() + restored.append(d) + + except frappe.DocumentAlreadyRestored: + frappe.message_log.pop() + invalid.append(d) + + except Exception: + frappe.message_log.pop() + failed.append(d) + frappe.db.rollback() + + return { + "restored": restored, + "invalid": invalid, + "failed": failed + } diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js new file mode 100644 index 0000000000..f5e1147dfb --- /dev/null +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -0,0 +1,40 @@ +frappe.listview_settings["Deleted Document"] = { + onload: function (doclist) { + const action = () => { + const selected_docs = doclist.get_checked_items(); + if (selected_docs.length > 0) { + let docnames = selected_docs.map(doc => doc.name); + frappe.call({ + method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", + args: { docnames }, + callback: function (r) { + if (r.message) { + function body(docnames) { + const html = docnames.map(docname => { + return `
  • ${docname}
    • `; + }); + return "
    ": ""; + } + + const { restored, invalid, failed } = r.message; + const restored_summary = message(__("Documents restored successfully"), restored); + const invalid_summary = message(__("Documents that were already restored"), invalid); + const failed_summary = message(__("Documents that failed to restore"), failed); + const summary = restored_summary + invalid_summary + failed_summary; + + frappe.msgprint(summary, __("Document Restoration Summary"), true); + + if (restored.length > 0) { + doclist.refresh(); + } + } + }, + }); + } + }; + doclist.page.add_actions_menu_item(__("Restore"), action, false); + }, +}; diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 379ea227cb..698289140e 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -70,6 +70,7 @@ "web_view", "has_web_view", "allow_guest_to_view", + "index_web_pages_for_search", "route", "is_published_field", "advanced", @@ -517,12 +518,18 @@ "fieldname": "email_settings_sb", "fieldtype": "Section Break", "label": "Email Settings" + }, + { + "default": "1", + "fieldname": "index_web_pages_for_search", + "fieldtype": "Check", + "label": "Index Web Pages for Search" } ], "icon": "fa fa-bolt", "idx": 6, "links": [], - "modified": "2020-03-27 14:51:44.581128", + "modified": "2020-07-21 16:20:57.028802", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index fe9f88b9b3..00e80ce4e7 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -376,3 +376,96 @@ class TestDocType(unittest.TestCase): link_doc.delete() doc.delete() frappe.db.commit() + + def test_ignore_cancelation_of_linked_doctype_during_cancell(self): + import json + from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs + + #create linked doctype + link_doc = self.new_doctype('Test Linked Doctype 1') + link_doc.is_submittable = 1 + for data in link_doc.get('permissions'): + data.submit = 1 + data.cancel = 1 + link_doc.insert() + + #create first parent doctype + test_doc_1 = self.new_doctype('Test Doctype 1') + test_doc_1.is_submittable = 1 + + field_2 = test_doc_1.append('fields', {}) + field_2.label = 'Test Linked Doctype 1' + field_2.fieldname = 'test_linked_doctype_a' + field_2.fieldtype = 'Link' + field_2.options = 'Test Linked Doctype 1' + + for data in test_doc_1.get('permissions'): + data.submit = 1 + data.cancel = 1 + test_doc_1.insert() + + #crete second parent doctype + doc = self.new_doctype('Test Doctype 2') + doc.is_submittable = 1 + + field_2 = doc.append('fields', {}) + field_2.label = 'Test Linked Doctype 1' + field_2.fieldname = 'test_linked_doctype_a' + field_2.fieldtype = 'Link' + field_2.options = 'Test Linked Doctype 1' + + for data in link_doc.get('permissions'): + data.submit = 1 + data.cancel = 1 + doc.insert() + + # create doctype data + data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') + data_link_doc_1.some_fieldname = 'Data1' + data_link_doc_1.insert() + data_link_doc_1.save() + data_link_doc_1.submit() + + data_doc_2 = frappe.new_doc('Test Doctype 1') + data_doc_2.some_fieldname = 'Data1' + data_doc_2.test_linked_doctype_a = data_link_doc_1.name + data_doc_2.insert() + data_doc_2.save() + data_doc_2.submit() + + data_doc = frappe.new_doc('Test Doctype 2') + data_doc.some_fieldname = 'Data1' + data_doc.test_linked_doctype_a = data_link_doc_1.name + data_doc.insert() + data_doc.save() + data_doc.submit() + + docs = get_submitted_linked_docs(link_doc.name, data_link_doc_1.name) + dump_docs = json.dumps(docs.get('docs')) + + cancel_all_linked_docs(dump_docs, ignore_doctypes_on_cancel_all=["Test Doctype 2"]) + + # checking that doc for Test Doctype 2 is not canceled + self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) + + data_doc.load_from_db() + data_doc_2.load_from_db() + self.assertEqual(data_link_doc_1.docstatus, 2) + + #linked doc is canceled + self.assertEqual(data_doc_2.docstatus, 2) + + #ignored doctype 2 during cancel + self.assertEqual(data_doc.docstatus, 1) + + # delete doctype record + data_doc.cancel() + data_doc.delete() + data_doc_2.delete() + data_link_doc_1.delete() + + # delete doctype + link_doc.delete() + doc.delete() + test_doc_1.delete() + frappe.db.commit() diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 657340ec24..e458b401e4 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -33,16 +33,22 @@ class Role(Document): if user_type != user.user_type: user.save() -# Get email addresses of all users that have been assigned this role -def get_emails_from_role(role): - emails = [] - for user in get_users(role): - user_email, enabled = frappe.db.get_value("User", user, ["email", "enabled"]) - if enabled and user_email not in ["admin@example.com", "guest@example.com"]: - emails.append(user_email) +def get_info_based_on_role(role, field='email'): + ''' Get information of all users that have been assigned this role ''' + users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"}, + fields=["parent"]) - return emails + return get_user_info(users, field) + +def get_user_info(users, field='email'): + ''' Fetch details about users for the specified field ''' + info_list = [] + for user in users: + user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"]) + if enabled and user_info not in ["admin@example.com", "guest@example.com"]: + info_list.append(user_info) + return info_list def get_users(role): return [d.parent for d in frappe.get_all("Has Role", filters={"role": role, "parenttype": "User"}, diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index f6134e045a..ac835108c1 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -18,6 +18,9 @@ class SMSSettings(Document): def validate_receiver_nos(receiver_list): validated_receiver_list = [] for d in receiver_list: + if not d: + break + # remove invalid character for x in [' ','-', '(', ')']: d = d.replace(x, '') diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index 5bae49ea95..733ee1774c 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -11,7 +11,6 @@ from frappe import _ from frappe.model.meta import is_single from frappe.modules import load_doctype_module - @frappe.whitelist() def get_submitted_linked_docs(doctype, name, docs=None, visited=None): """ @@ -78,7 +77,7 @@ def get_submitted_linked_docs(doctype, name, docs=None, visited=None): @frappe.whitelist() -def cancel_all_linked_docs(docs): +def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]): """ Cancel all linked doctype @@ -87,14 +86,16 @@ def cancel_all_linked_docs(docs): """ docs = json.loads(docs) + if isinstance(ignore_doctypes_on_cancel_all, string_types): + ignore_doctypes_on_cancel_all = json.loads(ignore_doctypes_on_cancel_all) for i, doc in enumerate(docs, 1): - if validate_linked_doc(doc) is True: - frappe.publish_progress(percent=i * 100 / len(docs), title=_("Cancelling documents")) + if validate_linked_doc(doc, ignore_doctypes_on_cancel_all) is True: + frappe.publish_progress(percent=i * 100 / ((len(docs) - len(ignore_doctypes_on_cancel_all))), title=_("Cancelling documents")) linked_doc = frappe.get_doc(doc.get("doctype"), doc.get("name")) linked_doc.cancel() -def validate_linked_doc(docinfo): +def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]): """ Validate a document to be submitted and non-exempted from auto-cancel. @@ -105,6 +106,10 @@ def validate_linked_doc(docinfo): bool: True if linked document passes all validations, else False """ + #ignore doctype to cancel + if docinfo.get("doctype") in ignore_doctypes_on_cancel_all: + return False + # skip non-submittable doctypes since they don't need to be cancelled if not frappe.get_meta(docinfo.get('doctype')).is_submittable: return False diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 01f75be954..1ec64826da 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -132,10 +132,11 @@ "has_web_view": 1, "icon": "fa fa-envelope", "idx": 1, + "index_web_pages_for_search": 1, "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2020-05-12 18:09:40.137138", + "modified": "2020-07-21 16:25:17.687476", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index 02fc8512ca..454514f922 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -3,104 +3,175 @@ this.frm.add_fetch('sender', 'email_id', 'sender_email'); -this.frm.fields_dict.sender.get_query = function(){ +this.frm.fields_dict.sender.get_query = function() { return { filters: { - 'enable_outgoing': 1 + enable_outgoing: 1 } - } + }; }; frappe.notification = { setup_fieldname_select: function(frm) { // get the doctype to update fields - if(!frm.doc.document_type) { + if (!frm.doc.document_type) { return; } frappe.model.with_doctype(frm.doc.document_type, function() { let get_select_options = function(df) { - return {value: df.fieldname, label: df.fieldname + " (" + __(df.label) + ")"}; - } + return { + value: df.fieldname, + label: df.fieldname + ' (' + __(df.label) + ')' + }; + }; let get_date_change_options = function() { let date_options = $.map(fields, function(d) { - return (d.fieldtype=="Date" || d.fieldtype=="Datetime")? - get_select_options(d) : null; + return d.fieldtype == 'Date' || d.fieldtype == 'Datetime' + ? get_select_options(d) + : null; }); // append creation and modified date to Date Change field return date_options.concat([ - { value: "creation", label: `creation (${__('Created On')})` }, - { value: "modified", label: `modified (${__('Last Modified Date')})` } + { value: 'creation', label: `creation (${__('Created On')})` }, + { value: 'modified', label: `modified (${__('Last Modified Date')})` } ]); - } + }; - let fields = frappe.get_doc("DocType", frm.doc.document_type).fields; - let options = $.map(fields, - function(d) { return in_list(frappe.model.no_value_type, d.fieldtype) ? - null : get_select_options(d); }); + let fields = frappe.get_doc('DocType', frm.doc.document_type).fields; + let options = $.map(fields, function(d) { + return in_list(frappe.model.no_value_type, d.fieldtype) + ? null : get_select_options(d); + }); // set value changed options - frm.set_df_property("value_changed", "options", [""].concat(options)); - frm.set_df_property("set_property_after_alert", "options", [""].concat(options)); + frm.set_df_property('value_changed', 'options', [''].concat(options)); + frm.set_df_property( + 'set_property_after_alert', + 'options', + [''].concat(options) + ); // set date changed options - frm.set_df_property("date_changed", "options", get_date_change_options()); + frm.set_df_property('date_changed', 'options', get_date_change_options()); - let email_fields = $.map(fields, - function(d) { return (d.options == "Email" || - (d.options=='User' && d.fieldtype=='Link')) ? - get_select_options(d) : null; }); + let receiver_fields = []; + if (frm.doc.channel === 'Email') { + receiver_fields = $.map(fields, function(d) { + return d.options == 'Email' || + (d.options == 'User' && d.fieldtype == 'Link') + ? get_select_options(d) : null; + }); + } else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) { + receiver_fields = $.map(fields, function(d) { + return d.options == 'Phone' ? get_select_options(d) : null; + }); + } // set email recipient options - frappe.meta.get_docfield("Notification Recipient", "email_by_document_field", + frappe.meta.get_docfield( + 'Notification Recipient', + 'receiver_by_document_field', // set first option as blank to allow notification not to be defaulted to the owner - frm.doc.name).options = [""].concat(["owner"].concat(email_fields)); + frm.doc.name + ).options = [''].concat(["owner"]).concat(receiver_fields); frm.fields_dict.recipients.grid.refresh(); }); - } -} + }, + setup_example_message: function(frm) { + let template = ''; + if (frm.doc.channel === 'WhatsApp') { + template = `
    Warning:
    Only Use Pre-Approved WhatsApp for Business Template +
    Message Example
    -frappe.ui.form.on("Notification", { +
    +Your {{ doc.name }} order of {{ doc.total }} has shipped and should be delivered on {{ doc.date }}. Details : {{doc.customer}}
+
    `; + } else if (frm.doc.channel === 'Email') { + template = `
    Message Example
    + +
    <h3>Order Overdue</h3>
+
+<p>Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.</p>
+
+<!-- show last comment -->
+{% if comments %}
+Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
+{% endif %}
+
+<h4>Details</h4>
+
+<ul>
+<li>Customer: {{ doc.customer }}
+<li>Amount: {{ doc.grand_total }}
+</ul>
+
    + `; + } else { + template = `
    Message Example
    + +
    *Order Overdue*
+
+Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.
+
+
+{% if comments %}
+Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
+{% endif %}
+
+*Details*
+
+• Customer: {{ doc.customer }}
+• Amount: {{ doc.grand_total }}
+
    `; + } + frm.set_df_property('message_examples', 'options', template); + + } +}; + +frappe.ui.form.on('Notification', { onload: function(frm) { - frm.set_query("document_type", function() { + frm.set_query('document_type', function() { return { - "filters": { - "istable": 0 + filters: { + istable: 0 } - } + }; }); - frm.set_query("print_format", function() { + frm.set_query('print_format', function() { return { - "filters": { - "doc_type": frm.doc.document_type + filters: { + doc_type: frm.doc.document_type } - } + }; }); }, refresh: function(frm) { frappe.notification.setup_fieldname_select(frm); - frm.get_field("is_standard").toggle(frappe.boot.developer_mode); + frm.get_field('is_standard').toggle(frappe.boot.developer_mode); frm.trigger('event'); }, document_type: function(frm) { frappe.notification.setup_fieldname_select(frm); }, view_properties: function(frm) { - frappe.route_options = {doc_type:frm.doc.document_type}; - frappe.set_route("Form", "Customize Form"); + frappe.route_options = { doc_type: frm.doc.document_type }; + frappe.set_route('Form', 'Customize Form'); }, event: function(frm) { - if(in_list(['Days Before', 'Days After'], frm.doc.event)) { + if (in_list(['Days Before', 'Days After'], frm.doc.event)) { frm.add_custom_button(__('Get Alerts for Today'), function() { frappe.call({ - method: 'frappe.email.doctype.notification.notification.get_documents_for_today', + method: + 'frappe.email.doctype.notification.notification.get_documents_for_today', args: { notification: frm.doc.name }, callback: function(r) { - if(r.message) { + if (r.message) { frappe.msgprint(r.message); } else { frappe.msgprint(__('No alerts for today')); @@ -111,6 +182,14 @@ frappe.ui.form.on("Notification", { } }, channel: function(frm) { - frm.toggle_reqd("recipients", frm.doc.channel=="Email"); + frm.toggle_reqd('recipients', frm.doc.channel == 'Email'); + frappe.notification.setup_fieldname_select(frm); + frappe.notification.setup_example_message(frm); + if (frm.doc.channel === 'SMS' && frm.doc.__islocal) { + frm.set_df_property('channel', + 'description', `To use SMS Channel, initialize SMS Settings.`); + } else { + frm.set_df_property('channel', 'description', ` `); + } } }); diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index 932f0491a9..95f218ad73 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -10,6 +10,7 @@ "enabled", "column_break_2", "channel", + "twilio_number", "slack_webhook_url", "filters", "subject", @@ -37,7 +38,6 @@ "message_sb", "message", "message_examples", - "slack_message_examples", "view_properties", "column_break_25", "attach_print", @@ -60,11 +60,13 @@ "fieldname": "channel", "fieldtype": "Select", "label": "Channel", - "options": "Email\nSlack\nSystem Notification", - "reqd": 1 + "options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS", + "reqd": 1, + "set_only_once": 1 }, { "depends_on": "eval:doc.channel=='Slack'", + "description": "To use Slack Channel, add a Slack Webhook URL.", "fieldname": "slack_webhook_url", "fieldtype": "Link", "label": "Slack Channel", @@ -77,13 +79,14 @@ "label": "Filters" }, { + "depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)", "description": "To add dynamic subject, use jinja tags like\n\n
    {{ doc.name }} Delivered
    ", "fieldname": "subject", "fieldtype": "Data", "ignore_xss_filter": 1, "in_list_view": 1, "label": "Subject", - "reqd": 1 + "mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)" }, { "fieldname": "document_type", @@ -153,6 +156,7 @@ "label": "Value Changed" }, { + "depends_on": "eval: doc.channel == 'Email'", "fieldname": "sender", "fieldtype": "Link", "label": "Sender", @@ -203,7 +207,7 @@ "label": "Value To Be Set" }, { - "depends_on": "eval:doc.channel!=='Slack'", + "depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)", "fieldname": "column_break_5", "fieldtype": "Section Break", "label": "Recipients" @@ -228,19 +232,11 @@ "label": "Message" }, { - "depends_on": "eval:doc.channel=='Email'", "fieldname": "message_examples", "fieldtype": "HTML", "label": "Message Examples", "options": "
    Message Example
    \n\n
    <h3>Order Overdue</h3>\n\n<p>Transaction {{ doc.name }} has exceeded Due Date. Please take necessary action.</p>\n\n<!-- show last comment -->\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n<h4>Details</h4>\n\n<ul>\n<li>Customer: {{ doc.customer }}\n<li>Amount: {{ doc.grand_total }}\n</ul>\n
    " }, - { - "depends_on": "eval:doc.channel=='Slack'", - "fieldname": "slack_message_examples", - "fieldtype": "HTML", - "label": "Message Examples", - "options": "
    Message Example
    \n\n
    *Order Overdue*\n\nTransaction {{ doc.name }} has exceeded Due Date. Please take necessary action.\n\n\n{% if comments %}\nLast comment: {{ comments[-1].comment }} by {{ comments[-1].by }}\n{% endif %}\n\n*Details*\n\n\u2022 Customer: {{ doc.customer }}\n\u2022 Amount: {{ doc.grand_total }}\n
    " - }, { "fieldname": "view_properties", "fieldtype": "Button", @@ -266,6 +262,14 @@ "label": "Print Format", "options": "Print Format" }, + { + "depends_on": "eval: doc.channel==='WhatsApp'", + "description": "To use WhatsApp for Business, initialize Twilio Settings.", + "fieldname": "twilio_number", + "fieldtype": "Link", + "label": "Twilio Number", + "options": "Twilio Number Group" + }, { "default": "0", "depends_on": "eval: doc.channel !== 'System Notification'", @@ -277,7 +281,7 @@ ], "icon": "fa fa-envelope", "links": [], - "modified": "2020-06-23 14:01:25.462544", + "modified": "2020-08-11 19:24:35.479373", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 81670756f6..2ec208c89d 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -7,12 +7,14 @@ import frappe import json, os from frappe import _ from frappe.model.document import Document -from frappe.core.doctype.role.role import get_emails_from_role +from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date from frappe.utils.jinja import validate_template from frappe.modules.utils import export_module_json, get_doc_module from six import string_types from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message +from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message +from frappe.core.doctype.sms_settings.sms_settings import send_sms from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification class Notification(Document): @@ -26,7 +28,9 @@ class Notification(Document): self.name = self.subject def validate(self): - validate_template(self.subject) + if self.channel not in ('WhatsApp', 'SMS'): + validate_template(self.subject) + validate_template(self.message) if self.event in ("Days Before", "Days After") and not self.date_changed: @@ -126,8 +130,15 @@ def get_context(context): if self.channel == 'Slack': self.send_a_slack_msg(doc, context) + if self.channel == 'WhatsApp': + self.send_whatsapp_msg(doc, context) + + if self.channel == 'SMS': + self.send_sms(doc, context) + if self.channel == 'System Notification' or self.send_system_notification: self.create_system_notification(doc, context) + except: frappe.log_error(title='Failed to send notification', message=frappe.get_traceback()) @@ -195,11 +206,24 @@ def get_context(context): and attachments[0].get('print_letterhead')) or False)) def send_a_slack_msg(self, doc, context): - send_slack_message( - webhook_url=self.slack_webhook_url, - message=frappe.render_template(self.message, context), - reference_doctype = doc.doctype, - reference_name = doc.name) + send_slack_message( + webhook_url=self.slack_webhook_url, + message=frappe.render_template(self.message, context), + reference_doctype=doc.doctype, + reference_name=doc.name) + + def send_whatsapp_msg(self, doc, context): + send_whatsapp_message( + sender=self.twilio_number, + receiver_list=self.get_receiver_list(doc, context), + message=frappe.render_template(self.message, context), + ) + + def send_sms(self, doc, context): + send_sms( + receiver_list=self.get_receiver_list(doc, context), + msg=frappe.render_template(self.message, context) + ) def get_list_of_recipients(self, doc, context): recipients = [] @@ -209,8 +233,8 @@ def get_context(context): if recipient.condition: if not frappe.safe_eval(recipient.condition, None, context): continue - if recipient.email_by_document_field: - email_ids_value = doc.get(recipient.email_by_document_field) + if recipient.receiver_by_document_field: + email_ids_value = doc.get(recipient.receiver_by_document_field) if validate_email_address(email_ids_value): email_ids = email_ids_value.replace(",", "\n") recipients = recipients + email_ids.split("\n") @@ -232,8 +256,8 @@ def get_context(context): bcc = bcc + recipient.bcc.split("\n") #For sending emails to specified role - if recipient.email_by_role: - emails = get_emails_from_role(recipient.email_by_role) + if recipient.receiver_by_role: + emails = get_info_based_on_role(recipient.receiver_by_role, 'email') for email in emails: recipients = recipients + email.split("\n") @@ -242,6 +266,27 @@ def get_context(context): return None, None, None return list(set(recipients)), list(set(cc)), list(set(bcc)) + def get_receiver_list(self, doc, context): + ''' return receiver list based on the doc field and role specified ''' + receiver_list = [] + for recipient in self.recipients: + if recipient.condition: + if not frappe.safe_eval(recipient.condition, None, context): + continue + + # For sending messages to the owner's mobile phone number + if recipient.receiver_by_document_field == 'owner': + receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no')) + # For sending messages to the number specified in the receiver field + elif recipient.receiver_by_document_field: + receiver_list.append(doc.get(recipient.receiver_by_document_field)) + + #For sending messages to specified role + if recipient.receiver_by_role: + receiver_list += get_info_based_on_role(recipient.receiver_by_role, 'mobile_no') + + return receiver_list + def get_attachment(self, doc): """ check print settings are attach the pdf """ if not self.attach_print: diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index b9bbde172d..9bdf09375d 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -63,7 +63,7 @@ class TestNotification(unittest.TestCase): notification.message = "test" recipent = frappe.new_doc("Notification Recipient") - recipent.email_by_document_field = "owner" + recipent.receiver_by_document_field = "owner" notification.recipents = recipent notification.condition = "test" @@ -105,7 +105,7 @@ class TestNotification(unittest.TestCase): "value_changed": "description1", "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }).insert() frappe.db.commit() diff --git a/frappe/email/doctype/notification/test_records.json b/frappe/email/doctype/notification/test_records.json index 865b2ac021..665f800c0f 100644 --- a/frappe/email/doctype/notification/test_records.json +++ b/frappe/email/doctype/notification/test_records.json @@ -8,7 +8,7 @@ "message": "New comment {{ doc.content }} created", "condition": "doc.communication_type=='Comment'", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -20,7 +20,7 @@ "message": "New comment {{ doc.content }} saved", "condition": "doc.communication_type=='Comment'", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ], "set_property_after_alert": "subject", "property_value": "__testing__" @@ -34,7 +34,7 @@ "condition": "doc.event_type=='Public'", "message": "A new public event {{ doc.subject }} on {{ doc.starts_on }} is created", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -46,7 +46,7 @@ "value_changed": "description", "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -59,7 +59,7 @@ "days_in_advance": 2, "message": "Description changed", "recipients": [ - { "email_by_document_field": "owner" } + { "receiver_by_document_field": "owner" } ] }, { @@ -70,7 +70,7 @@ "attach_print": 0, "message": "New user {{ doc.name }} created", "recipients": [ - { "email_by_document_field": "owner", "cc": "{{ doc.email }}" } + { "receiver_by_document_field": "owner", "cc": "{{ doc.email }}" } ] } ] diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.json b/frappe/email/doctype/notification_recipient/notification_recipient.json index ec35dccc63..201899cd57 100644 --- a/frappe/email/doctype/notification_recipient/notification_recipient.json +++ b/frappe/email/doctype/notification_recipient/notification_recipient.json @@ -1,204 +1,60 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2014-07-11 17:19:37.037109", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2014-07-11 17:19:37.037109", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "receiver_by_document_field", + "receiver_by_role", + "cc", + "bcc", + "condition" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "", - "fieldname": "email_by_document_field", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Email By Document Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:parent.channel=='Email'", + "description": "Optional: Always send to these ids. Each Email Address on a new row", + "fieldname": "cc", + "fieldtype": "Code", + "label": "CC" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "email_by_role", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Email By Role", - "length": 0, - "no_copy": 0, - "options": "Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:parent.channel=='Email'", + "fieldname": "bcc", + "fieldtype": "Code", + "label": "BCC" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "Optional: Always send to these ids. Each Email Address on a new row", - "fieldname": "cc", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "CC", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "description": "Expression, Optional", + "fieldname": "condition", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Condition" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "bcc", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "BCC", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "receiver_by_document_field", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Receiver By Document Field" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "Expression, Optional", - "fieldname": "condition", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "receiver_by_role", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Receiver By Role", + "options": "Role" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2018-09-03 18:37:57.043251", - "modified_by": "Administrator", - "module": "Email", - "name": "Notification Recipient", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0, - "track_views": 0 + ], + "istable": 1, + "links": [], + "modified": "2020-02-21 11:18:40.125233", + "modified_by": "Administrator", + "module": "Email", + "name": "Notification Recipient", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8ebda9c7b8..88428b875c 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -104,6 +104,7 @@ class IncompatibleApp(ValidationError): pass class InvalidDates(ValidationError): pass class DataTooLongException(ValidationError): pass class FileAlreadyAttachedException(Exception): pass +class DocumentAlreadyRestored(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass diff --git a/frappe/hooks.py b/frappe/hooks.py index 1f209f00a2..894e72a121 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -272,9 +272,6 @@ setup_wizard_exception = [ ] before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute'] -after_migrate = [ - 'frappe.modules.full_text_search.build_index_for_all_routes' -] otp_methods = ['OTP App','Email','SMS'] user_privacy_documents = [ diff --git a/frappe/integrations/doctype/twilio_number_group/__init__.py b/frappe/integrations/doctype/twilio_number_group/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json new file mode 100644 index 0000000000..1790581ca7 --- /dev/null +++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.json @@ -0,0 +1,32 @@ +{ + "actions": [], + "autoname": "field:phone_number", + "creation": "2020-02-24 13:58:58.036914", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "phone_number" + ], + "fields": [ + { + "fieldname": "phone_number", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Phone Number", + "unique": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2020-03-02 14:54:34.396254", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Twilio Number Group", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py new file mode 100644 index 0000000000..04cb9ae146 --- /dev/null +++ b/frappe/integrations/doctype/twilio_number_group/twilio_number_group.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class TwilioNumberGroup(Document): + pass diff --git a/frappe/integrations/doctype/twilio_settings/__init__.py b/frappe/integrations/doctype/twilio_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py new file mode 100644 index 0000000000..bcb1368d68 --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/test_twilio_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestTwilioSettings(unittest.TestCase): + pass diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.js b/frappe/integrations/doctype/twilio_settings/twilio_settings.js new file mode 100644 index 0000000000..59ebcf2e7d --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Twilio Settings', { + refresh: function(frm) { + frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`])); + } +}); diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.json b/frappe/integrations/doctype/twilio_settings/twilio_settings.json new file mode 100644 index 0000000000..e54500fd5d --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.json @@ -0,0 +1,57 @@ +{ + "actions": [], + "creation": "2020-01-28 15:21:44.457163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "account_sid", + "auth_token", + "column_break_2", + "twilio_number" + ], + "fields": [ + { + "fieldname": "account_sid", + "fieldtype": "Data", + "label": "Account SID" + }, + { + "fieldname": "auth_token", + "fieldtype": "Password", + "label": "Auth Token" + }, + { + "fieldname": "column_break_2", + "fieldtype": "Column Break" + }, + { + "fieldname": "twilio_number", + "fieldtype": "Table", + "label": "Twilio Number", + "options": "Twilio Number Group" + } + ], + "issingle": 1, + "links": [], + "modified": "2020-08-11 15:28:57.860554", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Twilio Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/twilio_settings/twilio_settings.py b/frappe/integrations/doctype/twilio_settings/twilio_settings.py new file mode 100644 index 0000000000..ba0565b3af --- /dev/null +++ b/frappe/integrations/doctype/twilio_settings/twilio_settings.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.document import Document +from twilio.rest import Client +from frappe import _ +from frappe.utils.password import get_decrypted_password +from six import string_types + +class TwilioSettings(Document): + pass + +def send_whatsapp_message(sender, receiver_list, message): + import json + if isinstance(receiver_list, string_types): + receiver_list = json.loads(receiver_list) + if not isinstance(receiver_list, list): + receiver_list = [receiver_list] + + + twilio_settings = frappe.get_doc("Twilio Settings") + auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token') + client = Client(twilio_settings.account_sid, auth_token) + args = { + "from_": 'whatsapp:+{}'.format(sender), + "body": message + } + + failed_delivery = [] + + for rec in receiver_list: + args.update({"to": 'whatsapp:{}'.format(rec)}) + resp = _send_whatsapp(args, client) + if not resp or resp.error_message: + failed_delivery.append(rec) + + if failed_delivery: + frappe.log_error(_("The message wasn't correctly delivered to: {}".format(", ".join(failed_delivery))), _('Delivery Failed')) + + +def _send_whatsapp(message_dict, client): + response = frappe._dict() + try: + response = client.messages.create(**message_dict) + except Exception as e: + frappe.log_error(e, title = _('Twilio WhatsApp Message Error')) + + return response \ No newline at end of file diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 9de176b2d0..a551d8edf1 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -12,8 +12,10 @@ from frappe.utils import split_emails, get_backups_path def send_email(success, service_name, doctype, email_field, error_status=None): recipients = get_recipients(doctype, email_field) if not recipients: - frappe.log_error("No Email Recipient found for {0}".format(service_name), - "{0}: Failed to send backup status email".format(service_name)) + frappe.log_error( + "No Email Recipient found for {0}".format(service_name), + "{0}: Failed to send backup status email".format(service_name), + ) return if success: @@ -23,7 +25,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None): subject = "Backup Upload Successful" message = """

    Backup Uploaded Successfully!

    -

    Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!

    """.format(service_name) +

    Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!

    """.format( + service_name + ) else: subject = "[Warning] Backup Upload Failed" @@ -31,7 +35,9 @@ def send_email(success, service_name, doctype, email_field, error_status=None):

    Backup Upload Failed!

    Oops, your automated backup to {0} failed.

    Error message: {1}

    -

    Please contact your system manager for more information.

    """.format(service_name, error_status) +

    Please contact your system manager for more information.

    """.format( + service_name, error_status + ) frappe.sendmail(recipients=recipients, subject=subject, message=message) @@ -44,29 +50,31 @@ def get_recipients(doctype, email_field): def get_latest_backup_file(with_files=False): + from frappe.utils.backups import BackupGenerator - def get_latest(file_ext): - file_list = glob.glob(os.path.join(get_backups_path(), file_ext)) - return max(file_list, key=os.path.getctime) if file_list else None - - latest_file = get_latest('*.sql.gz') - latest_site_config = get_latest('*.json') + odb = BackupGenerator( + frappe.conf.db_name, + frappe.conf.db_name, + frappe.conf.db_password, + db_host=frappe.db.host, + db_type=frappe.conf.db_type, + db_port=frappe.conf.db_port, + ) + database, public, private, config = odb.get_recent_backup(older_than=24 * 30) if with_files: - latest_public_file_bak = get_latest('*-files.tar') - latest_private_file_bak = get_latest('*-private-files.tar') - return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak + return database, config, public, private - return latest_file, latest_site_config + return database, config def get_file_size(file_path, unit): if not unit: - unit = 'MB' + unit = "MB" file_size = os.path.getsize(file_path) - memory_size_unit_mapper = {'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4} + memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4} i = 0 while i < memory_size_unit_mapper[unit]: file_size = file_size / 1000.0 @@ -78,7 +86,7 @@ def get_file_size(file_path, unit): def validate_file_size(): frappe.flags.create_new_backup = True latest_file, site_config = get_latest_backup_file() - file_size = get_file_size(latest_file, unit='GB') + file_size = get_file_size(latest_file, unit="GB") if file_size > 1: frappe.flags.create_new_backup = False diff --git a/frappe/migrate.py b/frappe/migrate.py index 9ec23d8ae7..6d64799fdd 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -19,10 +19,10 @@ from frappe.website import render from frappe.core.doctype.language.language import sync_languages from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs -from frappe.utils import global_search +from frappe.search.website_search import build_index_for_all_routes -def migrate(verbose=True, rebuild_website=False, skip_failing=False): +def migrate(verbose=True, rebuild_website=False, skip_failing=False, skip_search_index=False): '''Migrate all apps to the latest version, will: - run before migrate hooks - run patches @@ -80,9 +80,6 @@ Otherwise, check the server logs and ensure that all the required services are r # syncs statics render.clear_cache() - # add static pages to global search - global_search.update_global_search_for_all_web_pages() - # updating installed applications data frappe.get_single('Installed Applications').update_versions() @@ -91,6 +88,12 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Building search index for {}'.format(frappe.local.site)) + build_index_for_all_routes() + frappe.db.commit() clear_notifications() diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py deleted file mode 100644 index fce9983907..0000000000 --- a/frappe/modules/full_text_search.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals -import frappe -from whoosh.index import create_in, open_dir -from whoosh.fields import TEXT, ID, Schema -from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin -from whoosh.query import Prefix -from bs4 import BeautifulSoup -from frappe.website.render import render_page -from frappe.utils import set_request, cint -from frappe.utils.global_search import get_routes_to_index - - -def build_index_for_all_routes(): - print("Building search index for all web routes...") - routes = get_routes_to_index() - documents = [get_document_to_index(route) for route in routes] - build_index("web_routes", documents) - - -@frappe.whitelist(allow_guest=True) -def web_search(index_name, query, scope=None, limit=20): - limit = cint(limit) - return search(index_name, query, scope, limit) - - -def get_document_to_index(route): - frappe.set_user("Guest") - frappe.local.no_cache = True - - try: - set_request(method="GET", path=route) - content = render_page(route) - soup = BeautifulSoup(content, "html.parser") - page_content = soup.find(class_="page_content") - text_content = page_content.text if page_content else "" - title = soup.title.text.strip() if soup.title else route - - frappe.set_user("Administrator") - - return frappe._dict(title=title, content=text_content, path=route) - except ( - frappe.PermissionError, - frappe.DoesNotExistError, - frappe.ValidationError, - Exception, - ): - pass - - -def build_index(index_name, documents): - schema = Schema( - title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True) - ) - - index_dir = get_index_path(index_name) - frappe.create_folder(index_dir) - - ix = create_in(index_dir, schema) - writer = ix.writer() - - for document in documents: - if document: - writer.add_document( - title=document.title, path=document.path, content=document.content - ) - - writer.commit() - - -def search(index_name, text, scope=None, limit=20): - index_dir = get_index_path(index_name) - ix = open_dir(index_dir) - - results = None - out = [] - with ix.searcher() as searcher: - parser = MultifieldParser(["title", "content"], ix.schema) - parser.remove_plugin_class(FieldsPlugin) - parser.remove_plugin_class(WildcardPlugin) - query = parser.parse(text) - - filter_scoped = None - if scope: - filter_scoped = Prefix("path", scope) - results = searcher.search(query, limit=limit, filter=filter_scoped) - - for r in results: - title_highlights = r.highlights("title") - content_highlights = r.highlights("content") - out.append( - frappe._dict( - title=r["title"], - path=r["path"], - title_highlights=title_highlights, - content_highlights=content_highlights, - ) - ) - - return out - - -def get_index_path(index_name): - return frappe.get_site_path("indexes", index_name) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 27649b8da9..5970eae5ca 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -9,7 +9,7 @@ from frappe.utils import get_datetime_str from frappe.model.base_document import get_controller ignore_values = { - "Report": ["disabled", "prepared_report"], + "Report": ["disabled", "prepared_report", "add_total_row"], "Print Format": ["disabled"], "Notification": ["enabled"], "Print Style": ["disabled"], diff --git a/frappe/patches.txt b/frappe/patches.txt index e3a8180357..8657be1fc5 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -300,3 +300,4 @@ frappe.patches.v13_0.add_standard_navbar_items frappe.patches.v13_0.generate_theme_files_in_public_folder frappe.patches.v13_0.increase_password_length frappe.patches.v13_0.add_toggle_width_in_navbar_settings +frappe.patches.v13_0.rename_notification_fields diff --git a/frappe/patches/v13_0/rename_notification_fields.py b/frappe/patches/v13_0/rename_notification_fields.py new file mode 100644 index 0000000000..2984e6503c --- /dev/null +++ b/frappe/patches/v13_0/rename_notification_fields.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from frappe.model.utils.rename_field import rename_field + +def execute(): + """ + Change notification recipient fields from email to receiver fields + """ + frappe.reload_doc("Email", "doctype", "Notification Recipient") + frappe.reload_doc("Email", "doctype", "Notification") + + rename_field("Notification Recipient", "email_by_document_field", "receiver_by_document_field") + rename_field("Notification Recipient", "email_by_role", "receiver_by_role") \ No newline at end of file diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js index 726a83db72..a547cfcf32 100644 --- a/frappe/public/js/frappe/form/controls/code.js +++ b/frappe/public/js/frappe/form/controls/code.js @@ -46,6 +46,8 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({ const language_map = { 'Javascript': 'ace/mode/javascript', 'JS': 'ace/mode/javascript', + 'Python': 'ace/mode/python', + 'Py': 'ace/mode/python', 'HTML': 'ace/mode/html', 'CSS': 'ace/mode/css', 'Markdown': 'ace/mode/markdown', @@ -57,7 +59,7 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({ const language = this.df.options; const valid_languages = Object.keys(language_map); - if (!valid_languages.includes(language)) { + if (language && !valid_languages.includes(language)) { // eslint-disable-next-line console.warn(`Invalid language option provided for field "${this.df.label}". Valid options are ${valid_languages.join(', ')}.`); } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index c0b76ee94d..b4bc758399 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -667,22 +667,29 @@ frappe.ui.form.Form = class FrappeForm { savecancel(btn, callback, on_error) { const me = this; this.validate_form_action('Cancel'); - + me.ignore_doctypes_on_cancel_all = me.ignore_doctypes_on_cancel_all || []; frappe.call({ method: "frappe.desk.form.linked_with.get_submitted_linked_docs", args: { doctype: me.doc.doctype, name: me.doc.name }, - freeze: true, - callback: (r) => { - if (!r.exc && r.message.count > 0) { - me._cancel_all(r, btn, callback, on_error); - } else { - me._cancel(btn, callback, on_error, false); + freeze: true + }).then(r => { + if (!r.exc) { + let doctypes_to_cancel = (r.message.docs || []).map(value => { + return value.doctype; + }).filter(value => { + return !me.ignore_doctypes_on_cancel_all.includes(value); + }); + + if (doctypes_to_cancel.length) { + return me._cancel_all(r, btn, callback, on_error); } } - }); + return me._cancel(btn, callback, on_error, false); + } + ); } _cancel_all(r, btn, callback, on_error) { @@ -693,12 +700,16 @@ frappe.ui.form.Form = class FrappeForm { let links = r.message.docs; const doctypes = Array.from(new Set(links.map(link => link.doctype))); + me.ignore_doctypes_on_cancel_all = me.ignore_doctypes_on_cancel_all || []; + for (let doctype of doctypes) { - let docnames = links - .filter((link) => link.doctype == doctype) - .map((link) => frappe.utils.get_form_link(link.doctype, link.name, true)) - .join(", "); - links_text += `
  • ${doctype}: ${docnames}
    • `; + if (!me.ignore_doctypes_on_cancel_all.includes(doctype)) { + let docnames = links + .filter((link) => link.doctype == doctype) + .map((link) => frappe.utils.get_form_link(link.doctype, link.name, true)) + .join(", "); + links_text += `
  • ${doctype}: ${docnames}
    • `; + } } links_text = ``; @@ -728,7 +739,8 @@ frappe.ui.form.Form = class FrappeForm { frappe.call({ method: "frappe.desk.form.linked_with.cancel_all_linked_docs", args: { - docs: links + docs: links, + ignore_doctypes_on_cancel_all: me.ignore_doctypes_on_cancel_all || [] }, freeze: true, callback: (resp) => { @@ -742,7 +754,7 @@ frappe.ui.form.Form = class FrappeForm { } d.show(); - }; + } _cancel(btn, callback, on_error, skip_confirm) { const me = this; diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index f4bcecc68e..733c1bea5f 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -396,7 +396,7 @@ export default class GridRow { var field_on_change_function = field.df.onchange; field.df.onchange = function(e) { field_on_change_function && field_on_change_function(e); - me.grid.grid_rows[this.doc.idx - 1].refresh_field(this.df.fieldname); + me.grid.grid_rows[this.doc.idx - 1].refresh_field(field.df.fieldname); }; field.refresh(); if(field.$input) { diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 95a3577698..c7d001ed94 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -836,8 +836,15 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { const child_table_fields = frappe.meta.get_docfields(cdt).filter(standard_fields_filter); out[cdt] = child_table_fields; - }); + // add index column for child tables + out[cdt].push({ + label: __('Index'), + fieldname: 'idx', + fieldtype: 'Int', + parent: cdt + }); + }); return out; } @@ -857,7 +864,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { .map(df => ({ label: __(df.label), value: df.fieldname, - checked: this.fields.find(f => f[0] === df.fieldname) + checked: this.fields.find(f => f[0] === df.fieldname && f[1] === this.doctype) })) }); @@ -936,6 +943,15 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { docfield = this.group_by_control.get_group_by_docfield(); } + // child table index column + if (fieldname === 'idx' && doctype !== this.doctype) { + docfield = { + label: "Index", + fieldtype: "Int", + parent: doctype, + }; + } + if (!docfield) { docfield = frappe.model.get_std_field(fieldname, true); diff --git a/frappe/public/scss/doc.scss b/frappe/public/scss/doc.scss index 13a59ba45b..f258e2ee47 100644 --- a/frappe/public/scss/doc.scss +++ b/frappe/public/scss/doc.scss @@ -81,45 +81,10 @@ $navbar-height-lg: 4.5rem; } .doc-search { - position: relative; - width: 100%; - @include media-breakpoint-up(lg) { padding-left: 4rem; padding-right: 4rem; } - - .search-icon { - position: absolute; - left: 0; - top: 0; - width: 2.5rem; - height: 100%; - display: flex; - justify-content: center; - align-items: center; - } - - svg { - color: $gray-600; - } - - input { - padding-left: 2.5rem; - } - - .dropdown-menu { - .dropdown-item { - padding: 1rem 0.75rem; - } - - .match { - background-color: $primary-light; - color: $primary; - font-weight: 500; - padding: 0 0.125rem; - } - } } .doc-sidebar { diff --git a/frappe/public/scss/search.scss b/frappe/public/scss/search.scss new file mode 100644 index 0000000000..aee2457f31 --- /dev/null +++ b/frappe/public/scss/search.scss @@ -0,0 +1,40 @@ +.website-search { + position: relative; + width: 100%; + + .search-icon { + position: absolute; + left: 0; + top: 0; + width: 2.5rem; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + } + + svg { + color: $gray-600; + } + + input { + padding-left: 2.5rem; + } + + .dropdown-menu { + .dropdown-item { + padding: 1rem 0.75rem; + + &:focus { + background-color: $gray-100; + } + } + + .match { + background-color: $primary-light; + color: $primary; + font-weight: 500; + padding: 0 0.125rem; + } + } +} \ No newline at end of file diff --git a/frappe/public/scss/website.scss b/frappe/public/scss/website.scss index 3c11d23252..e64c090ea8 100644 --- a/frappe/public/scss/website.scss +++ b/frappe/public/scss/website.scss @@ -10,6 +10,7 @@ @import 'markdown'; @import 'sidebar'; @import 'portal'; +@import 'search'; @import 'doc'; .ql-editor.read-mode { diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py new file mode 100644 index 0000000000..0436775417 --- /dev/null +++ b/frappe/search/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe +from frappe.utils import cint +from frappe.search.website_search import WebsiteSearch +from frappe.search.full_text_search import FullTextSearch + +@frappe.whitelist(allow_guest=True) +def web_search(query, scope=None, limit=20): + limit = cint(limit) + ws = WebsiteSearch(index_name="web_routes") + return ws.search(query, scope, limit) \ No newline at end of file diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py new file mode 100644 index 0000000000..dd6e69111d --- /dev/null +++ b/frappe/search/full_text_search.py @@ -0,0 +1,136 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + +from whoosh.index import create_in, open_dir, EmptyIndexError +from whoosh.fields import TEXT, ID, Schema +from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin +from whoosh.query import Prefix + +class FullTextSearch: + """ Frappe Wrapper for Whoosh """ + + def __init__(self, index_name): + self.index_name = index_name + self.index_path = get_index_path(index_name) + self.schema = self.get_schema() + self.id = self.get_id() + + def get_schema(self): + return Schema(name=ID(stored=True), content=TEXT(stored=True)) + + def get_id(self): + return "name" + + def get_items_to_index(self): + """Get all documents to be indexed conforming to the schema""" + return [] + + def get_document_to_index(self): + return {} + + def build(self): + """ Build search index for all documents """ + self.documents = self.get_items_to_index() + self.build_index() + + def update_index_by_name(self, doc_name): + """Wraps `update_index` method, gets the document from name + and updates the index. This function changes the current user + and should only be run as administrator or in a background job. + + Args: + self (object): FullTextSearch Instance + doc_name (str): name of the document to be updated + """ + document = self.get_document_to_index(doc_name) + self.update_index(document) + + def remove_document_from_index(self, doc_name): + """Remove document from search index + + Args: + self (object): FullTextSearch Instance + doc_name (str): name of the document to be removed + """ + if not doc_name: + return + + ix = self.get_index() + with ix.searcher(): + writer = ix.writer() + writer.delete_by_term(self.id, doc_name) + writer.commit(optimize=True) + + def update_index(self, document): + """Update search index for a document + + Args: + self (object): FullTextSearch Instance + document (_dict): A dictionary with title, path and content + """ + ix = self.get_index() + + with ix.searcher(): + writer = ix.writer() + writer.delete_by_term(self.id, document[self.id]) + writer.add_document(**document) + writer.commit(optimize=True) + + def get_index(self): + try: + return open_dir(self.index_path) + except EmptyIndexError: + return self.create_index() + + def create_index(self): + frappe.create_folder(self.index_path) + return create_in(self.index_path, self.schema) + + def build_index(self): + """Build index for all parsed documents""" + ix = self.create_index() + writer = ix.writer() + + for document in self.documents: + if document: + writer.add_document(**document) + + writer.commit(optimize=True) + + def search(self, text, scope=None, limit=20): + """Search from the current index + + Args: + text (str): String to search for + scope (str, optional): Scope to limit the search. Defaults to None. + limit (int, optional): Limit number of search results. Defaults to 20. + + Returns: + [List(_dict)]: Search results + """ + ix = self.get_index() + + results = None + out = [] + + with ix.searcher() as searcher: + parser = MultifieldParser(["title", "content"], ix.schema) + parser.remove_plugin_class(FieldsPlugin) + parser.remove_plugin_class(WildcardPlugin) + query = parser.parse(text) + + filter_scoped = None + if scope: + filter_scoped = Prefix(self.id, scope) + results = searcher.search(query, limit=limit, filter=filter_scoped) + + for r in results: + out.append(self.parse_result(r)) + + return out + +def get_index_path(index_name): + return frappe.get_site_path("indexes", index_name) \ No newline at end of file diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py new file mode 100644 index 0000000000..be9669d3c5 --- /dev/null +++ b/frappe/search/test_full_text_search.py @@ -0,0 +1,128 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt +import unittest +from frappe.search.full_text_search import FullTextSearch + +class TestFullTextSearch(unittest.TestCase): + + def setUp(self): + index = get_index() + index.build() + self.index = index + + def test_search_term(self): + # Search Wikipedia + res = self.index.search("multilingual online encyclopedia") + self.assertEqual(res[0], 'site/wikipedia') + + res = self.index.search("Linux kernel") + self.assertEqual(res[0], 'os/linux') + + res = self.index.search("Enterprise Resource Planning") + self.assertEqual(res[0], 'sw/erpnext') + + def test_search_limit(self): + res = self.index.search("CommonSearchTerm") + self.assertEqual(len(res), 5) + + res = self.index.search("CommonSearchTerm", limit=3) + self.assertEqual(len(res), 3) + + res = self.index.search("CommonSearchTerm", limit=20) + self.assertEqual(len(res), 5) + + def test_search_scope(self): + # Search outside scope + res = self.index.search("multilingual online encyclopedia", scope=["os"]) + self.assertEqual(len(res), 0) + + # Search inside scope + res = self.index.search("CommonSearchTerm", scope=["os"]) + self.assertEqual(len(res), 2) + self.assertTrue('os/linux' in res) + self.assertTrue('os/gnu' in res) + + def test_remove_document_from_index(self): + self.index.remove_document_from_index("os/gnu") + res = self.index.search("GNU") + self.assertEqual(len(res), 0) + + def test_update_index(self): + # Update existing index + self.index.update_index({ + 'name': "sw/erpnext", + 'content': """AwesomeERPNext""" + }) + + res = self.index.search("CommonSearchTerm") + self.assertTrue('sw/erpnext' not in res) + + res = self.index.search("AwesomeERPNext") + self.assertEqual(res[0], "sw/erpnext") + + # Update new doc + self.index.update_index({ + 'name': "sw/frappebooks", + 'content': """DesktopAccounting""" + }) + + res = self.index.search("DesktopAccounting") + self.assertEqual(res[0], "sw/frappebooks") + + + +class TestWrapper(FullTextSearch): + def get_items_to_index(self): + return get_documents() + + def get_document_to_index(self, name): + documents = get_documents() + for doc in documents: + if doc["name"] == name: + return doc + + def parse_result(self, result): + return result["name"] + + +def get_index(): + return TestWrapper("test_frappe_index") + +def get_documents(): + docs = [] + docs.append({ + 'name': "site/wikipedia", + 'content': """Wikipedia is a multilingual online encyclopedia created and maintained + as an open collaboration project by a community of volunteer editors using a wiki-based editing system. + It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm""" + }) + + docs.append({ + 'name': "os/linux", + 'content': """Linux is a family of open source Unix-like operating systems based on the + Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds. + Linux is typically packaged in a Linux distribution. CommonSearchTerm""" + }) + + docs.append({ + 'name': "os/gnu", + 'content': """GNU is an operating system and an extensive collection of computer software. + GNU is composed wholly of free software, most of which is licensed under the GNU Project's own + General Public License. GNU is a recursive acronym for "GNU's Not Unix! ", + chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm""" + }) + + docs.append({ + 'name': "sw/erpnext", + 'content': """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by + Frappe Technologies Pvt. Ltd. and is built on MariaDB database system using a Python based server-side framework. + ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm""" + }) + + docs.append({ + 'name': "sw/frappe", + 'content': """Frappe Framework is a full-stack web framework, that includes everything you need to build and + deploy business applications with Rich Admin Interface. CommonSearchTerm""" + }) + + return docs \ No newline at end of file diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py new file mode 100644 index 0000000000..de93fea3f5 --- /dev/null +++ b/frappe/search/website_search.py @@ -0,0 +1,117 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from bs4 import BeautifulSoup +from whoosh.fields import TEXT, ID, Schema +from frappe.search.full_text_search import FullTextSearch +from frappe.website.render import render_page +from frappe.utils import set_request +import os + +INDEX_NAME = "web_routes" + +class WebsiteSearch(FullTextSearch): + """ Wrapper for WebsiteSearch """ + + def get_schema(self): + return Schema( + title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True) + ) + + def get_id(self): + return "path" + + def get_items_to_index(self): + """Get all routes to be indexed, this includes the static pages + in www/ and routes from published documents + + Returns: + self (object): FullTextSearch Instance + """ + routes = get_static_pages_from_all_apps() + routes += get_doctype_routes_with_web_view() + + documents = [self.get_document_to_index(route) for route in routes] + return documents + + def get_document_to_index(self, route): + """Render a page and parse it using BeautifulSoup + + Args: + path (str): route of the page to be parsed + + Returns: + document (_dict): A dictionary with title, path and content + """ + frappe.set_user("Guest") + frappe.local.no_cache = True + + try: + set_request(method="GET", path=route) + content = render_page(route) + soup = BeautifulSoup(content, "html.parser") + page_content = soup.find(class_="page_content") + text_content = page_content.text if page_content else "" + title = soup.title.text.strip() if soup.title else route + + return frappe._dict(title=title, content=text_content, path=route) + except Exception: + pass + finally: + frappe.set_user("Administrator") + + def parse_result(self, result): + title_highlights = result.highlights("title") + content_highlights = result.highlights("content") + + return frappe._dict( + title=result["title"], + path=result["path"], + title_highlights=title_highlights, + content_highlights=content_highlights, + ) + + +def get_doctype_routes_with_web_view(): + all_routes = [] + filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} + fields = ["name", "is_published_field"] + doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) + + for doctype in doctype_with_web_views: + if doctype.is_published_field: + routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route") + all_routes += [route.route for route in routes] + + return all_routes + +def get_static_pages_from_all_apps(): + from glob import glob + apps = frappe.get_installed_apps() + + routes_to_index = [] + for app in apps: + path_to_index = frappe.get_app_path(app, 'www') + + files_to_index = glob(path_to_index + '/**/*.html', recursive=True) + files_to_index.extend(glob(path_to_index + '/**/*.md', recursive=True)) + for file in files_to_index: + route = os.path.relpath(file, path_to_index).split('.')[0] + if route.endswith('index'): + route = route.rsplit('index', 1)[0] + routes_to_index.append(route) + return routes_to_index + +def update_index_for_path(path): + ws = WebsiteSearch(INDEX_NAME) + return ws.update_index_by_name(path) + +def remove_document_from_index(path): + ws = WebsiteSearch(INDEX_NAME) + return ws.remove_document_from_index(path) + +def build_index_for_all_routes(): + ws = WebsiteSearch(INDEX_NAME) + return ws.build() \ No newline at end of file diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html index 3e1cc5509a..3a566a1227 100644 --- a/frappe/templates/doc.html +++ b/frappe/templates/doc.html @@ -22,7 +22,7 @@
    -
    +