diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 05e2b0378d..75452073bb 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -565,6 +565,7 @@ "bold": 0, "collapsible": 0, "columns": 0, + "default": "250", "description": "Total number of emails to sync in initial sync process ", "fieldname": "initial_sync_count", "fieldtype": "Select", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 14f686ec9c..a3e2cc05f7 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -140,8 +140,10 @@ class EmailAccount(Document): "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, "use_imap": self.use_imap, - "email_sync_rule": email_sync_rule + "email_sync_rule": email_sync_rule, + "uid_validity": self.uidvalidity }) + if self.password: args.password = self.get_password() @@ -219,6 +221,12 @@ class EmailAccount(Document): def receive(self, test_mails=None): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" + def get_seen(status): + if not status: + return None + seen = 0 if status == "SEEN" else 1 + return seen + if self.enable_incoming: uid_list = [] exceptions = [] @@ -233,11 +241,13 @@ class EmailAccount(Document): incoming_mails = emails.get("latest_messages") uid_list = emails.get("uid_list", []) + seen_status = email.get("seen_status", []) for idx, msg in enumerate(incoming_mails): try: uid = None if not uid_list else uid_list[idx] - communication = self.insert_communication(msg, uid) + seen = None if not seen_status else get_seen(seen_status.get(uid, None)) + communication = self.insert_communication(msg, _uid=uid, _seen=seen) #self.notify_update() except SentEmailInInbox: @@ -283,7 +293,7 @@ class EmailAccount(Document): unhandled_email.save() frappe.db.commit() - def insert_communication(self, msg, _uid=None): + def insert_communication(self, msg, _uid=None, _seen=None): if isinstance(msg,list): raw, uid, seen = msg else: @@ -291,6 +301,7 @@ class EmailAccount(Document): seen = uid = None if _uid: uid = _uid + if _seen: seen = _seen email = Email(raw) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 30d3a242f7..c1b8abd3ea 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -11,6 +11,7 @@ from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now, cint, cstr, strip, markdown) from frappe.utils.scheduler import log from frappe.utils.file_manager import get_random_filename, save_file, MaxFileSizeReachedError +import re class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass @@ -107,6 +108,7 @@ class EmailServer: # track if errors arised self.errors = False self.latest_messages = [] + self.seen_status = {} uid_list = email_list = self.get_new_mails() num = num_copy = len(email_list) @@ -128,7 +130,6 @@ class EmailServer: self.retrieve_message(message_meta, i+1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): break - # WARNING: Mark as read - message number 101 onwards from the pop list # This is to avoid having too many messages entering the system num = num_copy @@ -152,13 +153,20 @@ class EmailServer: self.pop.quit() out = { "latest_messages": self.latest_messages } - if self.settings.use_imap: out.update({ "uid_list": uid_list }) + if self.settings.use_imap: + out.update({ + "uid_list": uid_list, + "seen_status": self.seen_status + }) return out def get_new_mails(self): """Return list of new mails""" if cint(self.settings.use_imap): + if not self.check_imap_uidvalidity(): + frappe.throw(_("UIDVALIDITY is changed in imap server")) + self.imap.select("Inbox") response, message = self.imap.uid('search', None, self.settings.email_sync_rule) email_list = message[0].split() @@ -167,18 +175,46 @@ class EmailServer: return email_list + def check_imap_uidvalidity(self): + # compare the UIDVALIDITY of email account and imap server + uid_validity = self.settings.uid_validity + + responce, message = self.imap.status("Inbox", "(UIDVALIDITY)") + current_uid_validity = self.parse_imap_responce("UIDVALIDITY", message[0]) + + if not uid_validity: + # uid validity is not available for email account + frappe.db.set_value("Email Account", self.settings.email_account, "uidvalidity", current_uid_validity) + return True + elif uid_validity == current_uid_validity: + return True + else: + # UIDs are reindexed on imap server + # self.settings.email_sync_rule = "UNSEEN" + return False + + def parse_imap_responce(self, cmd, responce): + pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd) + match = re.search(pattern, responce, re.U | re.I) + if match: + return match.group(0) + else: + return None + def retrieve_message(self, message_meta, msg_num=None): incoming_mail = None try: self.validate_message_limits(message_meta) if cint(self.settings.use_imap): + status, response = self.imap.uid("fetch", message_meta, "(FLAGS)") + self.get_mail_seen_status(message_meta, response[0]) + status, message = self.imap.uid('fetch', message_meta, '(RFC822)') self.latest_messages.append(message[0][1]) else: msg = self.pop.retr(msg_num) self.latest_messages.append(b'\n'.join(msg[1])) - except (TotalSizeExceededError, EmailTimeoutError): # propagate this error to break the loop self.errors = True @@ -207,6 +243,22 @@ class EmailServer: # mark as seen self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') + def get_mail_seen_status(self, uid, flag_string): + # parse the mail flags + if not flag_string: + return None + + flags = [] + for flag in imaplib.ParseFlags(flag_string) or []: + pattern = re.compile("\w+") + match = re.search(pattern, flag) + flags.append(match.group(0)) + + if "Seen" in flags: + self.seen_status.update({ uid: "SEEN" }) + else: + self.seen_status.update({ uid: "UNSEEN" }) + def has_login_limit_exceeded(self, e): return "-ERR Exceeded the login limit" in strip(cstr(e.message))