feat: Improvements in Webhooks (backport #13320) (#13791)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Co-authored-by: Deepesh Garg <42651287+deepeshgarg007@users.noreply.github.com>

frappe/integrations/doctype/webhook/test_webhook.py

@@ -4,7 +4,7 @@
 import unittest
 
 import frappe
-from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get_webhook_data
+from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get_webhook_data, enqueue_webhook
 
 
 class TestWebhook(unittest.TestCase):
@@ -12,6 +12,8 @@ class TestWebhook(unittest.TestCase):
 	def setUpClass(cls):
 		# delete any existing webhooks
 		frappe.db.sql("DELETE FROM tabWebhook")
+		# Delete existing logs if any
+		frappe.db.sql("DELETE FROM `tabWebhook Request Log`")
 		# create test webhooks
 		cls.create_sample_webhooks()
 
@@ -162,3 +164,18 @@ class TestWebhook(unittest.TestCase):
 
 		data = get_webhook_data(doc=self.user, webhook=self.webhook)
 		self.assertEqual(data, {"name": self.user.name})
+
+	def test_webhook_req_log_creation(self):
+		if not frappe.db.get_value('User', 'user2@integration.webhooks.test.com'):
+			user = frappe.get_doc({
+				'doctype': 'User', 
+				'email': 'user2@integration.webhooks.test.com',
+				'first_name': 'user2'
+			}).insert()
+		else:
+			user = frappe.get_doc('User', 'user2@integration.webhooks.test.com')
+
+		webhook = frappe.get_doc('Webhook', {'webhook_doctype': 'User'})
+		enqueue_webhook(user, webhook)
+
+		self.assertTrue(frappe.db.get_all('Webhook Request Log', pluck='name'))

frappe/integrations/doctype/webhook/webhook.json

@@ -18,6 +18,7 @@
   "html_condition",
   "sb_webhook",
   "request_url",
+  "request_method",
   "cb_webhook",
   "request_structure",
   "sb_security",
@@ -154,10 +155,18 @@
    "fieldname": "enabled",
    "fieldtype": "Check",
    "label": "Enabled"
+  },
+  {
+   "default": "POST",
+   "fieldname": "request_method",
+   "fieldtype": "Select",
+   "label": "Request Method",
+   "options": "POST\nPUT\nDELETE",
+   "reqd": 1
   }
  ],
  "links": [],
- "modified": "2021-04-14 05:35:28.532049",
+ "modified": "2021-05-25 11:11:28.555291",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook",

frappe/integrations/doctype/webhook/webhook.py

@@ -59,7 +59,6 @@ class Webhook(Document):
 			if self.request_structure == "Form URL-Encoded":
 				self.webhook_json = None
 			elif self.request_structure == "JSON":
-				validate_json(self.webhook_json)
 				validate_template(self.webhook_json)
 				self.webhook_data = []
 
@@ -83,18 +82,32 @@ def enqueue_webhook(doc, webhook):
 
 	for i in range(3):
 		try:
-			r = requests.post(webhook.request_url, data=json.dumps(data, default=str), headers=headers, timeout=5)
+			r = requests.request(method=webhook.request_method, url=webhook.request_url,
+				data=json.dumps(data, default=str), headers=headers, timeout=5)
 			r.raise_for_status()
 			frappe.logger().debug({"webhook_success": r.text})
+			log_request(webhook.request_url, headers, data, r)
 			break
 		except Exception as e:
 			frappe.logger().debug({"webhook_error": e, "try": i + 1})
+			log_request(webhook.request_url, headers, data, r)
 			sleep(3 * i + 1)
 			if i != 2:
 				continue
 			else:
 				raise e
 
+def log_request(url, headers, data, res):
+	request_log = frappe.get_doc({
+		"doctype": "Webhook Request Log",
+		"user": frappe.session.user if frappe.session.user else None,
+		"url": url,
+		"headers": json.dumps(headers, indent=4) if headers else None,
+		"data": json.dumps(data, indent=4) if isinstance(data, dict) else data,
+		"response": json.dumps(res.json(), indent=4) if res else None
+	})
+
+	request_log.save(ignore_permissions=True)
 
 def get_webhook_headers(doc, webhook):
 	headers = {}
@@ -129,10 +142,3 @@ def get_webhook_data(doc, webhook):
 		data = json.loads(data)
 
 	return data
-
-
-def validate_json(string):
-	try:
-		json.loads(string)
-	except (TypeError, ValueError):
-		frappe.throw(_("Request Body consists of an invalid JSON structure"), title=_("Invalid JSON"))

frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+import unittest
+
+class TestWebhookRequestLog(unittest.TestCase):
+	pass

frappe/integrations/doctype/webhook_request_log/webhook_request_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Webhook Request Log', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/integrations/doctype/webhook_request_log/webhook_request_log.json

@@ -0,0 +1,81 @@
+{
+ "actions": [],
+ "autoname": "WEBHOOK-REQ-.#####",
+ "creation": "2021-05-24 21:35:59.104776",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user",
+  "headers",
+  "data",
+  "column_break_4",
+  "url",
+  "response"
+ ],
+ "fields": [
+  {
+   "fieldname": "url",
+   "fieldtype": "Data",
+   "label": "URL",
+   "read_only": 1
+  },
+  {
+   "fieldname": "headers",
+   "fieldtype": "Code",
+   "label": "Headers",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "response",
+   "fieldtype": "Code",
+   "label": "Response",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "data",
+   "fieldtype": "Code",
+   "label": "Data",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "label": "User",
+   "options": "User",
+   "read_only": 1
+  }
+ ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-05-26 23:57:58.495261",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Webhook Request Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/integrations/doctype/webhook_request_log/webhook_request_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class WebhookRequestLog(Document):
+	pass