[IMAP] uid based imap email sync
This commit is contained in:
mbauskar
parent
53d6aefde9
commit
d8bfa0210d
4 changed files with 143 additions and 16 deletions
|
|
@ -633,6 +633,33 @@
|
|||
"set_only_once": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "uid",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "UID",
|
||||
"length": 0,
|
||||
"no_copy": 1,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
|
|
@ -1428,5 +1455,5 @@
|
|||
"sort_order": "DESC",
|
||||
"title_field": "subject",
|
||||
"track_changes": 1,
|
||||
"track_seen": 0
|
||||
"track_seen": 1
|
||||
}
|
||||
|
|
@ -530,6 +530,65 @@
|
|||
"set_only_once": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "UNSEEN",
|
||||
"depends_on": "eval: doc.enable_incoming",
|
||||
"fieldname": "email_sync_option",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Email Sync Option",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "ALL\nUNSEEN",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"description": "Total number of emails to sync in initial sync process ",
|
||||
"fieldname": "initial_sync_count",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Initial Sync Count",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "100\n250\n500",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
|
|
@ -1138,7 +1197,7 @@
|
|||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "uid_validity",
|
||||
"fieldname": "uidvalidity",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
|
|
@ -1147,7 +1206,7 @@
|
|||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "uid validity",
|
||||
"label": "UIDVALIDITY",
|
||||
"length": 0,
|
||||
"no_copy": 1,
|
||||
"permlevel": 0,
|
||||
|
|
@ -1176,7 +1235,7 @@
|
|||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "uidnext",
|
||||
"label": "UIDNEXT",
|
||||
"length": 0,
|
||||
"no_copy": 1,
|
||||
"permlevel": 0,
|
||||
|
|
|
|||
|
|
@ -75,8 +75,6 @@ class EmailAccount(Document):
|
|||
if self.append_to not in valid_doctypes:
|
||||
frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
|
||||
|
||||
|
||||
|
||||
def on_update(self):
|
||||
"""Check there is only one default of each type."""
|
||||
self.there_must_be_only_one_default()
|
||||
|
|
@ -131,7 +129,7 @@ class EmailAccount(Document):
|
|||
server.password = self.get_password()
|
||||
server.sess
|
||||
|
||||
def get_incoming_server(self, in_receive=False):
|
||||
def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
|
||||
"""Returns logged in POP3/IMAP connection object."""
|
||||
if frappe.cache().get_value("workers:no-internet") == True:
|
||||
return None
|
||||
|
|
@ -142,6 +140,7 @@ class EmailAccount(Document):
|
|||
"use_ssl": self.use_ssl,
|
||||
"username": getattr(self, "login_id", None) or self.email_id,
|
||||
"use_imap": self.use_imap,
|
||||
"email_sync_rule": email_sync_rule
|
||||
})
|
||||
if self.password:
|
||||
args.password = self.get_password()
|
||||
|
|
@ -221,17 +220,24 @@ class EmailAccount(Document):
|
|||
def receive(self, test_mails=None):
|
||||
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
|
||||
if self.enable_incoming:
|
||||
uid_list = []
|
||||
exceptions = []
|
||||
|
||||
if frappe.local.flags.in_test:
|
||||
incoming_mails = test_mails
|
||||
else:
|
||||
email_server = self.get_incoming_server(in_receive=True)
|
||||
incoming_mails = email_server.get_messages()
|
||||
email_sync_rule = self.build_email_sync_rule()
|
||||
|
||||
for msg in incoming_mails:
|
||||
email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
|
||||
emails = email_server.get_messages()
|
||||
|
||||
incoming_mails = emails.get("latest_messages")
|
||||
uid_list = emails.get("uid_list", [])
|
||||
|
||||
for idx, msg in enumerate(incoming_mails):
|
||||
try:
|
||||
|
||||
communication = self.insert_communication(msg)
|
||||
uid = None if not uid_list else uid_list[idx]
|
||||
communication = self.insert_communication(msg, uid)
|
||||
#self.notify_update()
|
||||
|
||||
except SentEmailInInbox:
|
||||
|
|
@ -277,12 +283,15 @@ class EmailAccount(Document):
|
|||
unhandled_email.save()
|
||||
frappe.db.commit()
|
||||
|
||||
def insert_communication(self, msg):
|
||||
def insert_communication(self, msg, _uid=None):
|
||||
if isinstance(msg,list):
|
||||
raw, uid, seen = msg
|
||||
else:
|
||||
raw = msg
|
||||
seen = uid = None
|
||||
|
||||
if _uid: uid = _uid
|
||||
|
||||
email = Email(raw)
|
||||
|
||||
if email.from_email == self.email_id and not email.mail.get("Reply-To"):
|
||||
|
|
@ -511,6 +520,17 @@ class EmailAccount(Document):
|
|||
def after_rename(self, old, new, merge=False):
|
||||
frappe.db.set_value("Email Account", new, "email_account_name", new)
|
||||
|
||||
def build_email_sync_rule(self):
|
||||
if not self.use_imap:
|
||||
return "UNSEEN"
|
||||
|
||||
if self.email_sync_option == "ALL":
|
||||
max_uid = get_max_email_uid(self.name)
|
||||
last_uid = max_uid + int(self.initial_sync_count or 100) if max_uid == 1 else "*"
|
||||
return "UID {}:{}".format(max_uid, last_uid)
|
||||
else:
|
||||
return self.email_sync_option
|
||||
|
||||
@frappe.whitelist()
|
||||
def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None):
|
||||
if not txt: txt = ""
|
||||
|
|
@ -584,3 +604,19 @@ def pull_from_email_account(email_account):
|
|||
'''Runs within a worker process'''
|
||||
email_account = frappe.get_doc("Email Account", email_account)
|
||||
email_account.receive()
|
||||
|
||||
def get_max_email_uid(email_account):
|
||||
# get maximum uid of emails
|
||||
max_uid = 1
|
||||
|
||||
result = frappe.db.get_all("Communication", filters={
|
||||
"communication_medium": "Email",
|
||||
"sent_or_received": "Received",
|
||||
"email_account": email_account
|
||||
}, fields=["ifnull(max(uid), 0) as uid"])
|
||||
|
||||
if not result:
|
||||
return 1
|
||||
else:
|
||||
max_uid = int(result[0].get("uid", 0)) + 1
|
||||
return max_uid
|
||||
|
|
@ -101,12 +101,14 @@ class EmailServer:
|
|||
if not self.connect():
|
||||
return []
|
||||
|
||||
uid_list = []
|
||||
|
||||
try:
|
||||
# track if errors arised
|
||||
self.errors = False
|
||||
self.latest_messages = []
|
||||
|
||||
email_list = self.get_new_mails()
|
||||
uid_list = email_list = self.get_new_mails()
|
||||
num = num_copy = len(email_list)
|
||||
|
||||
# WARNING: Hard coded max no. of messages to be popped
|
||||
|
|
@ -149,13 +151,16 @@ class EmailServer:
|
|||
else:
|
||||
self.pop.quit()
|
||||
|
||||
return self.latest_messages
|
||||
out = { "latest_messages": self.latest_messages }
|
||||
if self.settings.use_imap: out.update({ "uid_list": uid_list })
|
||||
|
||||
return out
|
||||
|
||||
def get_new_mails(self):
|
||||
"""Return list of new mails"""
|
||||
if cint(self.settings.use_imap):
|
||||
self.imap.select("Inbox")
|
||||
response, message = self.imap.uid('search', None, "UNSEEN")
|
||||
response, message = self.imap.uid('search', None, self.settings.email_sync_rule)
|
||||
email_list = message[0].split()
|
||||
else:
|
||||
email_list = self.pop.list()[1]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue