diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 96e8ce0bc8..e019f9bdb4 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -639,7 +639,7 @@ "collapsible": 0, "columns": 0, "fieldname": "uid", - "fieldtype": "Data", + "fieldtype": "Int", "hidden": 1, "ignore_user_permissions": 0, "ignore_xss_filter": 0, @@ -660,6 +660,33 @@ "set_only_once": 0, "unique": 0 }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "unique_id", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Unique ID", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, { "allow_on_submit": 0, "bold": 0, diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index f105ff2e16..ca1d77d5b6 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -20,21 +20,6 @@ class Communication(Document): no_feed_on_delete = True """Communication represents an external communication like Email.""" - def onload(self): - # set mail as seen - if self.communication_medium == "Email" and not self.seen and \ - not frappe.db.get_value("Email Flag Queue", { "communication": self.name }): - # create email flag queue - - doc = frappe.get_doc({ - "flag": "Seen", - "uid": self.uid, - "action": "+FLAGS", - "doctype": "Email Flag Queue", - "email_account": self.email_account, - "communication": self.name, - }).insert(ignore_permissions=True) - def validate(self): if self.reference_doctype and self.reference_name: if not self.reference_owner: diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 75452073bb..1144985e94 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -1228,7 +1228,7 @@ "collapsible": 0, "columns": 0, "fieldname": "uidnext", - "fieldtype": "Data", + "fieldtype": "Int", "hidden": 1, "ignore_user_permissions": 0, "ignore_xss_filter": 0, diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index f48022f37d..87f58bd531 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import frappe import imaplib import re +import json import socket from frappe import _ from frappe.model.document import Document @@ -231,6 +232,7 @@ class EmailAccount(Document): uid_list = [] exceptions = [] seen_status = [] + unique_id_list = [] if frappe.local.flags.in_test: incoming_mails = test_mails @@ -242,13 +244,15 @@ class EmailAccount(Document): incoming_mails = emails.get("latest_messages") uid_list = emails.get("uid_list", []) - seen_status = email.get("seen_status", []) + seen_status = emails.get("seen_status", []) + unique_id_list = emails.get("unique_id_list", []) for idx, msg in enumerate(incoming_mails): try: uid = None if not uid_list else uid_list[idx] seen = None if not seen_status else get_seen(seen_status.get(uid, None)) - communication = self.insert_communication(msg, _uid=uid, _seen=seen) + unique_id = None if not unique_id_list else unique_id_list.get(uid, None) + communication = self.insert_communication(msg, _uid=uid, _seen=seen, unique_id=unique_id) #self.notify_update() except SentEmailInInbox: @@ -294,7 +298,7 @@ class EmailAccount(Document): unhandled_email.save() frappe.db.commit() - def insert_communication(self, msg, _uid=None, _seen=None): + def insert_communication(self, msg, _uid=None, _seen=None, unique_id=None): if isinstance(msg,list): raw, uid, seen = msg else: @@ -312,6 +316,16 @@ class EmailAccount(Document): # dont count emails sent by the system get those raise SentEmailInInbox + name = frappe.db.get_value("Communication", { "unique_id": unique_id }) + if name: + # email is already available update communication uid instead + communication = frappe.get_doc("Communication", name) + communication.uid = uid + communication.save(ignore_permissions=True) + communication._attachments = [] + + return communication + communication = frappe.get_doc({ "doctype": "Communication", "subject": email.subject, @@ -328,7 +342,8 @@ class EmailAccount(Document): "message_id": email.message_id, "communication_date": email.date, "has_attachment": 1 if email.attachments else 0, - "seen": seen + "seen": seen, + "unique_id": unique_id }) self.set_thread(communication, email) @@ -346,7 +361,6 @@ class EmailAccount(Document): communication._attachments = email.save_attachments_in_doc(communication) # replace inline images - dirty = False for file in communication._attachments: if file.name in email.cid_map and email.cid_map[file.name]: @@ -553,20 +567,17 @@ class EmailAccount(Document): if not self.use_imap: return - flags = frappe.get_all("Email Flag Queue", { - "email_account": self.name, - "action": "Seen" - }, ["name", "uid", "communication"]) + flags = frappe.db.sql("""select name, uid from `tabCommunication` where sent_or_received = "Received" + and seen = 0 and communication_medium = "Email" and email_account='{email_account}' and + ifnull(_seen, '') = ''""".format(email_account=self.name), as_dict=True) - uid_list = [ flag.get("uid") for flag in flags ] + uid_list = list(set([ flag.get("uid") for flag in flags ])) if flags and uid_list: email_server = self.get_incoming_server() - email_server.mark_as_seen(uid_list=uid_list) + marked_as_seen = email_server.mark_as_seen(uid_list=uid_list) - # delete Email Flag Queue - for flag in flags: - frappe.db.set_value("Communication", flag.get("communication"), "seen", 1) - frappe.delete_doc("Email Flag Queue", flag.get("name")) + docnames = ",".join([ "'%s'"%uid for uid in uid_list ]) + frappe.db.sql(""" update `tabCommunication` set seen=1 where name in ({docnames})""".format(docnames=docnames)) @frappe.whitelist() def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None): diff --git a/frappe/email/receive.py b/frappe/email/receive.py index a93efc229b..7aa3a6f3c3 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -2,7 +2,7 @@ # MIT License. See license.txt from __future__ import unicode_literals -import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re +import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re, hashlib from email_reply_parser import EmailReplyParser from email.header import decode_header import frappe @@ -109,12 +109,13 @@ class EmailServer: self.errors = False self.latest_messages = [] self.seen_status = {} + self.unique_id_list = {} uid_list = email_list = self.get_new_mails() num = num_copy = len(email_list) # WARNING: Hard coded max no. of messages to be popped - if num > 20: num = 20 + if num > 50: num = 50 # size limits self.total_size = 0 @@ -156,7 +157,8 @@ class EmailServer: if self.settings.use_imap: out.update({ "uid_list": uid_list, - "seen_status": self.seen_status + "seen_status": self.seen_status, + "unique_id_list": self.unique_id_list }) return out @@ -164,10 +166,9 @@ class EmailServer: def get_new_mails(self): """Return list of new mails""" if cint(self.settings.use_imap): - if not self.check_imap_uidvalidity(): - frappe.throw(_("UIDVALIDITY is changed in imap server")) + self.check_imap_uidvalidity() - self.imap.select("Inbox") + self.imap.select("Inbox", readonly=True) response, message = self.imap.uid('search', None, self.settings.email_sync_rule) email_list = message[0].split() else: @@ -179,19 +180,27 @@ class EmailServer: # compare the UIDVALIDITY of email account and imap server uid_validity = self.settings.uid_validity - responce, message = self.imap.status("Inbox", "(UIDVALIDITY)") + responce, message = self.imap.status("Inbox", "(UIDVALIDITY UIDNEXT)") current_uid_validity = self.parse_imap_responce("UIDVALIDITY", message[0]) + if not current_uid_validity: + frappe.throw(_("Can not find UIDVALIDITY in imap status response")) - if not uid_validity: - # uid validity is not available for email account - frappe.db.set_value("Email Account", self.settings.email_account, "uidvalidity", current_uid_validity) - return True + uidnext = int(self.parse_imap_responce("UIDNEXT", message[0]) or "1") + frappe.db.set_value("Email Account", self.settings.email_account, "uidnext", uidnext) + + if not uid_validity or uid_validity != current_uid_validity: + # uidvalidity changed & all email uids are reindexed by server + frappe.db.sql("""update `tabCommunication` set uid=-1 where communication_medium='Email' + and email_account='{email_account}'""".format(email_account=self.settings.email_account)) + frappe.db.sql(""" update `tabEmail Account` set uidvalidity='{uidvalidity}', uidnext={uidnext} where + name='{email_account}'""".format(uidvalidity=current_uid_validity, uidnext=uidnext, email_account=self.settings.email_account)) + + from_uid = 1 if uidnext < 101 or (uidnext - 100) < 1 else uidnext - 100 + # sync last 100 email + self.settings.email_sync_rule = "UID {}:{}".format(from_uid, uidnext) + elif uid_validity == current_uid_validity: - return True - else: - # UIDs are reindexed on imap server - # self.settings.email_sync_rule = "UNSEEN" - return False + return def parse_imap_responce(self, cmd, responce): pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd) @@ -207,11 +216,13 @@ class EmailServer: self.validate_message_limits(message_meta) if cint(self.settings.use_imap): - status, response = self.imap.uid("fetch", message_meta, "(FLAGS)") - self.get_mail_seen_status(message_meta, response[0]) + status, message = self.imap.uid('fetch', message_meta, '(RFC822 BODY.PEEK[HEADER] FLAGS)') + raw, header, ignore = message - status, message = self.imap.uid('fetch', message_meta, '(RFC822)') - self.latest_messages.append(message[0][1]) + self.get_email_seen_status(message_meta, header[0]) + self.get_email_headers_hash(message_meta, header[1]) + + self.latest_messages.append(raw[1]) else: msg = self.pop.retr(msg_num) self.latest_messages.append(b'\n'.join(msg[1])) @@ -221,6 +232,7 @@ class EmailServer: raise except Exception, e: + print e if self.has_login_limit_exceeded(e): self.errors = True raise LoginLimitExceeded, e @@ -243,8 +255,8 @@ class EmailServer: # mark as seen self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') - def get_mail_seen_status(self, uid, flag_string): - # parse the mail flags + def get_email_seen_status(self, uid, flag_string): + """ parse the email FLAGS response """ if not flag_string: return None @@ -259,6 +271,28 @@ class EmailServer: else: self.seen_status.update({ uid: "UNSEEN" }) + def get_email_headers_hash(self, uid, headers): + """ generate the email unique id from header hash + unique id can be used to update uid if UID is reindexed""" + + hash = hashlib.sha1() + for header in headers: + if header[0] == 'Content-Type': + # skip variable boundaries + continue + + try: + decoded_header = decode_header(header[1]) + decoded = ''.join([val[0].decode(val[1]).encode('ascii', 'ignore') \ + if val[1] is not None else val[0] for val in decoded_header]) + cleaned = re.sub(r"\s+", u"", decoded, flags=re.UNICODE) + hash.update(cleaned) + except: + pass + + self.unique_id_list.update({ uid: hash.hexdigest() }) + print self.unique_id_list + def has_login_limit_exceeded(self, e): return "-ERR Exceeded the login limit" in strip(cstr(e.message)) @@ -302,13 +336,21 @@ class EmailServer: return error_msg - def mark_as_seen(uid_list=[]): + def mark_as_seen(self, uid_list=[]): """ set all uids mails the flag as seen """ + if not uid_list: return - uid = ",".join(uid_list) - self.imap.uid('STORE', uid, '+FLAGS', '(\\SEEN)') + if not self.connect(): + return + + self.imap.select("Inbox") + for uid in uid_list: + try: + self.imap.uid('STORE', uid, '+FLAGS', '(\\SEEN)') + except Exception as e: + continue class Email: """Wrapper for an email."""