From 555bcc415353bfdc52ea1b783ca2e008f2e8c032 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 5 Aug 2021 11:08:01 +0530 Subject: [PATCH] feat: Improvements in Webhooks (backport #13320) (#13791) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Co-authored-by: Deepesh Garg <42651287+deepeshgarg007@users.noreply.github.com> --- .../doctype/webhook/test_webhook.py | 19 ++++- .../integrations/doctype/webhook/webhook.json | 11 ++- .../integrations/doctype/webhook/webhook.py | 24 +++--- .../doctype/webhook_request_log/__init__.py | 0 .../test_webhook_request_log.py | 8 ++ .../webhook_request_log.js | 8 ++ .../webhook_request_log.json | 81 +++++++++++++++++++ .../webhook_request_log.py | 8 ++ 8 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 frappe/integrations/doctype/webhook_request_log/__init__.py create mode 100644 frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py create mode 100644 frappe/integrations/doctype/webhook_request_log/webhook_request_log.js create mode 100644 frappe/integrations/doctype/webhook_request_log/webhook_request_log.json create mode 100644 frappe/integrations/doctype/webhook_request_log/webhook_request_log.py diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index 09ad56a190..b77e311f7e 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -4,7 +4,7 @@ import unittest import frappe -from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get_webhook_data +from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get_webhook_data, enqueue_webhook class TestWebhook(unittest.TestCase): @@ -12,6 +12,8 @@ class TestWebhook(unittest.TestCase): def setUpClass(cls): # delete any existing webhooks frappe.db.sql("DELETE FROM tabWebhook") + # Delete existing logs if any + frappe.db.sql("DELETE FROM `tabWebhook Request Log`") # create test webhooks cls.create_sample_webhooks() @@ -162,3 +164,18 @@ class TestWebhook(unittest.TestCase): data = get_webhook_data(doc=self.user, webhook=self.webhook) self.assertEqual(data, {"name": self.user.name}) + + def test_webhook_req_log_creation(self): + if not frappe.db.get_value('User', 'user2@integration.webhooks.test.com'): + user = frappe.get_doc({ + 'doctype': 'User', + 'email': 'user2@integration.webhooks.test.com', + 'first_name': 'user2' + }).insert() + else: + user = frappe.get_doc('User', 'user2@integration.webhooks.test.com') + + webhook = frappe.get_doc('Webhook', {'webhook_doctype': 'User'}) + enqueue_webhook(user, webhook) + + self.assertTrue(frappe.db.get_all('Webhook Request Log', pluck='name')) \ No newline at end of file diff --git a/frappe/integrations/doctype/webhook/webhook.json b/frappe/integrations/doctype/webhook/webhook.json index 85895c052c..880874cb25 100644 --- a/frappe/integrations/doctype/webhook/webhook.json +++ b/frappe/integrations/doctype/webhook/webhook.json @@ -18,6 +18,7 @@ "html_condition", "sb_webhook", "request_url", + "request_method", "cb_webhook", "request_structure", "sb_security", @@ -154,10 +155,18 @@ "fieldname": "enabled", "fieldtype": "Check", "label": "Enabled" + }, + { + "default": "POST", + "fieldname": "request_method", + "fieldtype": "Select", + "label": "Request Method", + "options": "POST\nPUT\nDELETE", + "reqd": 1 } ], "links": [], - "modified": "2021-04-14 05:35:28.532049", + "modified": "2021-05-25 11:11:28.555291", "modified_by": "Administrator", "module": "Integrations", "name": "Webhook", diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index 1fb2bc6743..e3a8bda2aa 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -59,7 +59,6 @@ class Webhook(Document): if self.request_structure == "Form URL-Encoded": self.webhook_json = None elif self.request_structure == "JSON": - validate_json(self.webhook_json) validate_template(self.webhook_json) self.webhook_data = [] @@ -83,18 +82,32 @@ def enqueue_webhook(doc, webhook): for i in range(3): try: - r = requests.post(webhook.request_url, data=json.dumps(data, default=str), headers=headers, timeout=5) + r = requests.request(method=webhook.request_method, url=webhook.request_url, + data=json.dumps(data, default=str), headers=headers, timeout=5) r.raise_for_status() frappe.logger().debug({"webhook_success": r.text}) + log_request(webhook.request_url, headers, data, r) break except Exception as e: frappe.logger().debug({"webhook_error": e, "try": i + 1}) + log_request(webhook.request_url, headers, data, r) sleep(3 * i + 1) if i != 2: continue else: raise e +def log_request(url, headers, data, res): + request_log = frappe.get_doc({ + "doctype": "Webhook Request Log", + "user": frappe.session.user if frappe.session.user else None, + "url": url, + "headers": json.dumps(headers, indent=4) if headers else None, + "data": json.dumps(data, indent=4) if isinstance(data, dict) else data, + "response": json.dumps(res.json(), indent=4) if res else None + }) + + request_log.save(ignore_permissions=True) def get_webhook_headers(doc, webhook): headers = {} @@ -129,10 +142,3 @@ def get_webhook_data(doc, webhook): data = json.loads(data) return data - - -def validate_json(string): - try: - json.loads(string) - except (TypeError, ValueError): - frappe.throw(_("Request Body consists of an invalid JSON structure"), title=_("Invalid JSON")) diff --git a/frappe/integrations/doctype/webhook_request_log/__init__.py b/frappe/integrations/doctype/webhook_request_log/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py new file mode 100644 index 0000000000..dd11bf8a01 --- /dev/null +++ b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + +class TestWebhookRequestLog(unittest.TestCase): + pass diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js new file mode 100644 index 0000000000..9ec4f11536 --- /dev/null +++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Webhook Request Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json new file mode 100644 index 0000000000..96690f6e8c --- /dev/null +++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json @@ -0,0 +1,81 @@ +{ + "actions": [], + "autoname": "WEBHOOK-REQ-.#####", + "creation": "2021-05-24 21:35:59.104776", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user", + "headers", + "data", + "column_break_4", + "url", + "response" + ], + "fields": [ + { + "fieldname": "url", + "fieldtype": "Data", + "label": "URL", + "read_only": 1 + }, + { + "fieldname": "headers", + "fieldtype": "Code", + "label": "Headers", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "response", + "fieldtype": "Code", + "label": "Response", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "user", + "fieldtype": "Link", + "label": "User", + "options": "User", + "read_only": 1 + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2021-05-26 23:57:58.495261", + "modified_by": "Administrator", + "module": "Integrations", + "name": "Webhook Request Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py new file mode 100644 index 0000000000..493ebfd8f7 --- /dev/null +++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class WebhookRequestLog(Document): + pass