From 5b170c031dd622018133b0f88a1e751c54db69d5 Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Mon, 13 Apr 2020 16:40:09 +0530 Subject: [PATCH 1/5] perf: enqueue instance of send_one for better performance Usecase: Suppose I have hundred's of emails in the Email Queue and the flush job executes. While the execution of the flush job is going another flush job starts and so on. Because of this a list of flush jobs gets lined up in the background jobs queue. To avoid this it is better to enqueue the send_one function inside the flush job this will very efficiently improve the performance of the flush job. --- frappe/email/queue.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index b35f5944b2..9b1cfdd6b4 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -348,7 +348,19 @@ def flush(from_test=False): smtpserver = SMTPServer() smtpserver_dict[email.sender] = smtpserver - send_one(email.name, smtpserver, auto_commit, from_test=from_test) + # send_one(email.name, smtpserver, auto_commit, from_test=from_test) + from frappe import enqueue + send_one_args = { + 'email': email.name, + 'smtpserver':smtpserver, + 'auto_commit': auto_commit, + 'from_test':from_test + } + enqueue( + method='frappe.email.queue.send_one', + queue='short', + **send_one_args + ) # NOTE: removing commit here because we pass auto_commit # finally: From 7b10a2d88a2f15a3487f8ceeb337f3cbf29dbc35 Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Mon, 13 Apr 2020 20:07:46 +0530 Subject: [PATCH 2/5] style: removed comment and formatted dictionary --- frappe/email/queue.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 9b1cfdd6b4..d82d93fa75 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -348,13 +348,12 @@ def flush(from_test=False): smtpserver = SMTPServer() smtpserver_dict[email.sender] = smtpserver - # send_one(email.name, smtpserver, auto_commit, from_test=from_test) from frappe import enqueue send_one_args = { 'email': email.name, - 'smtpserver':smtpserver, + 'smtpserver': smtpserver, 'auto_commit': auto_commit, - 'from_test':from_test + 'from_test': from_test } enqueue( method='frappe.email.queue.send_one', From 8e9d642f44d0f394b153f45b7d4031ea85ec92a8 Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Mon, 13 Apr 2020 20:11:54 +0530 Subject: [PATCH 3/5] style: formatted parameters of enqueue function --- frappe/email/queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index d82d93fa75..4b60ee858b 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -356,8 +356,8 @@ def flush(from_test=False): 'from_test': from_test } enqueue( - method='frappe.email.queue.send_one', - queue='short', + method = 'frappe.email.queue.send_one', + queue = 'short', **send_one_args ) From 47446469bd4f21742fb7def8a8bf2797e062f60f Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Mon, 13 Apr 2020 20:20:02 +0530 Subject: [PATCH 4/5] refactor: moved import to the top of the file --- frappe/email/queue.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 4b60ee858b..d4fbf798c5 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -6,7 +6,7 @@ import frappe import sys from six.moves import html_parser as HTMLParser import smtplib, quopri, json -from frappe import msgprint, _, safe_decode, safe_encode +from frappe import msgprint, _, safe_decode, safe_encode, enqueue from frappe.email.smtp import SMTPServer, get_outgoing_email_account from frappe.email.email_body import get_email, get_formatted_html, add_attachment from frappe.utils.verified_command import get_signed_params, verify_request @@ -347,8 +347,7 @@ def flush(from_test=False): if not smtpserver: smtpserver = SMTPServer() smtpserver_dict[email.sender] = smtpserver - - from frappe import enqueue + send_one_args = { 'email': email.name, 'smtpserver': smtpserver, From fda7a45c52da717a1f52e565b2cd913bbe725bff Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Mon, 27 Apr 2020 18:25:16 +0530 Subject: [PATCH 5/5] fix: corrected flush job for passing test_flush --- frappe/email/queue.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index d4fbf798c5..a31ebe3f96 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -348,17 +348,20 @@ def flush(from_test=False): smtpserver = SMTPServer() smtpserver_dict[email.sender] = smtpserver - send_one_args = { - 'email': email.name, - 'smtpserver': smtpserver, - 'auto_commit': auto_commit, - 'from_test': from_test - } - enqueue( - method = 'frappe.email.queue.send_one', - queue = 'short', - **send_one_args - ) + if from_test: + send_one(email.name, smtpserver, auto_commit, from_test=from_test) + else: + send_one_args = { + 'email': email.name, + 'smtpserver': smtpserver, + 'auto_commit': auto_commit, + 'from_test': from_test + } + enqueue( + method = 'frappe.email.queue.send_one', + queue = 'short', + **send_one_args + ) # NOTE: removing commit here because we pass auto_commit # finally: @@ -387,8 +390,13 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals `tabEmail Queue` where name=%s - for update''', email, as_dict=True)[0] - + for update''', email, as_dict=True) + + if len(email): + email = email[0] + else: + return + recipients_list = frappe.db.sql('''select name, recipient, status from `tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1)