test: smtp integration test using smtp4dev

.github/helper/db/mariadb.json

@@ -6,7 +6,8 @@
     "allow_tests": true,
     "db_type": "mariadb",
     "auto_email_id": "test@example.com",
-    "mail_server": "smtp.example.com",
+    "mail_server": "localhost",
+    "mail_port": 2525,
     "mail_login": "test@example.com",
     "mail_password": "test",
     "admin_password": "admin",

.github/helper/db/postgres.json

@@ -6,7 +6,8 @@
     "db_type": "postgres",
     "allow_tests": true,
     "auto_email_id": "test@example.com",
-    "mail_server": "smtp.example.com",
+    "mail_server": "localhost",
+    "mail_port": 2525,
     "mail_login": "test@example.com",
     "mail_password": "test",
     "admin_password": "admin",

.github/workflows/server-tests.yml

@@ -72,6 +72,12 @@ jobs:
         ports:
           - 5432:5432
 
+      smtp_server:
+        image: rnwood/smtp4dev
+        ports:
+          - 2525:25
+          - 3000:80
+
     steps:
       - name: Clone
         uses: actions/checkout@v4

frappe/email/doctype/email_queue/email_queue.py

@@ -167,14 +167,14 @@ class EmailQueue(Document):
 				if method := get_hook_method("override_email_send"):
 					method(self, self.sender, recipient.recipient, message)
 				else:
-					if not frappe.flags.in_test:
+					if not frappe.flags.in_test or frappe.flags.testing_email:
 						ctx.smtp_server.session.sendmail(
 							from_addr=self.sender, to_addrs=recipient.recipient, msg=message
 						)
 
 				ctx.update_recipient_status_to_sent(recipient)
 
-			if frappe.flags.in_test:
+			if frappe.flags.in_test and not frappe.flags.testing_email:
 				frappe.flags.sent_mail = message
 				return
 

frappe/parallel_test_runner.py

@@ -10,7 +10,7 @@ import requests
 
 import frappe
 
-from .test_runner import SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config
+from .test_runner import SLOW_TEST_THRESHOLD, make_test_records
 
 click_ctx = click.get_current_context(True)
 if click_ctx:
@@ -38,7 +38,6 @@ class ParallelTestRunner:
 		frappe.flags.in_test = True
 		frappe.clear_cache()
 		frappe.utils.scheduler.disable_scheduler()
-		set_test_email_config()
 		self.before_test_setup()
 
 	def before_test_setup(self):

frappe/test_runner.py

@@ -78,8 +78,6 @@ def main(
 		if not scheduler_disabled_by_user:
 			frappe.utils.scheduler.disable_scheduler()
 
-		set_test_email_config()
-
 		if not frappe.flags.skip_before_tests:
 			if verbose:
 				print('Running "before_tests" hooks')
@@ -126,17 +124,6 @@ def main(
 			xmloutput_fh.close()
 
 
-def set_test_email_config():
-	frappe.conf.update(
-		{
-			"auto_email_id": "test@example.com",
-			"mail_server": "smtp.example.com",
-			"mail_login": "test@example.com",
-			"mail_password": "test",
-		}
-	)
-
-
 class TimeLoggingTestResult(unittest.TextTestResult):
 	def startTest(self, test):
 		self._started_at = time.monotonic()

frappe/tests/test_email.py

@@ -5,6 +5,8 @@ import email
 import re
 from unittest.mock import patch
 
+import requests
+
 import frappe
 from frappe.email.doctype.email_account.test_email_account import TestEmailAccount
 from frappe.email.doctype.email_queue.email_queue import QueueBuilder
@@ -325,3 +327,50 @@ class TestVerifiedRequests(FrappeTestCase):
 			set_request(method="GET", query_string=signed_url)
 			self.assertTrue(verify_request())
 		frappe.local.request = None
+
+
+class TestEmailIntegrationTest(FrappeTestCase):
+	"""Sends email to local SMTP server and verifies correctness.
+
+	SMTP4Dev runs as a service in unit test CI job.
+	If you need to run this test locally, you must setup SMTP4dev locally.
+
+	WARNING: SMTP4dev doesn't have stable API, it can break anytime.
+	"""
+
+	SMTP4DEV_WEB = "http://localhost:3000"
+
+	def setUp(self) -> None:
+		# Frappe code is configured to not attempting sending emails during test.
+		frappe.flags.testing_email = True
+		requests.delete(f"{self.SMTP4DEV_WEB}/api/Messages/*")
+		return super().setUp()
+
+	def tearDown(self) -> None:
+		frappe.flags.testing_email = False
+		return super().tearDown()
+
+	def get_last_sent_emails(self):
+		return requests.get(
+			f"{self.SMTP4DEV_WEB}/api/Messages?sortColumn=receivedDate&sortIsDescending=true"
+		).json()
+
+	def test_send_email(self):
+		sender = "a@example.io"
+		recipients = "b@example.io,c@example.io"
+		subject = "checking if email works"
+		content = "is email working?"
+
+		frappe.sendmail(sender=sender, recipients=recipients, subject=subject, content=content, now=True)
+		email = frappe.get_last_doc("Email Queue")
+		self.assertEqual(email.sender, sender)
+		self.assertEqual(len(email.recipients), 2)
+		self.assertEqual(email.status, "Sent")
+
+		sent_mails = self.get_last_sent_emails()
+		self.assertEqual(len(sent_mails), 2)
+
+		for sent_mail in sent_mails:
+			self.assertEqual(sent_mail["from"], sender)
+			self.assertEqual(sent_mail["subject"], subject)
+		self.assertSetEqual(set(recipients.split(",")), {m["to"] for m in sent_mails})