vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[imap] saved email unique id from header and email reindexing

Browse source
This commit is contained in:
mbauskar 2017-02-15 14:39:57 +05:30
parent 62fe7d19a4
commit 41c5ce8ffd
5 changed files with 122 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
frappe/core/doctype/communication/communication.json
View file

@ -639,7 +639,7 @@
"collapsible": 0,
"columns": 0,
"fieldname": "uid",
"fieldtype": "Data",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
@ -660,6 +660,33 @@
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "unique_id",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Unique ID",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,

15
frappe/core/doctype/communication/communication.py
View file

@ -20,21 +20,6 @@ class Communication(Document):
no_feed_on_delete = True
"""Communication represents an external communication like Email."""
def onload(self):
# set mail as seen
if self.communication_medium == "Email" and not self.seen and \
not frappe.db.get_value("Email Flag Queue", { "communication": self.name }):
# create email flag queue
doc = frappe.get_doc({
"flag": "Seen",
"uid": self.uid,
"action": "+FLAGS",
"doctype": "Email Flag Queue",
"email_account": self.email_account,
"communication": self.name,
}).insert(ignore_permissions=True)
def validate(self):
if self.reference_doctype and self.reference_name:
if not self.reference_owner:

2
frappe/email/doctype/email_account/email_account.json
View file

@ -1228,7 +1228,7 @@
"collapsible": 0,
"columns": 0,
"fieldname": "uidnext",
"fieldtype": "Data",
"fieldtype": "Int",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,

41
frappe/email/doctype/email_account/email_account.py
View file

@ -5,6 +5,7 @@ from __future__ import unicode_literals
import frappe
import imaplib
import re
import json
import socket
from frappe import _
from frappe.model.document import Document
@ -231,6 +232,7 @@ class EmailAccount(Document):
uid_list = []
exceptions = []
seen_status = []
unique_id_list = []
if frappe.local.flags.in_test:
incoming_mails = test_mails
@ -242,13 +244,15 @@ class EmailAccount(Document):
incoming_mails = emails.get("latest_messages")
uid_list = emails.get("uid_list", [])
seen_status = email.get("seen_status", [])
seen_status = emails.get("seen_status", [])
unique_id_list = emails.get("unique_id_list", [])
for idx, msg in enumerate(incoming_mails):
try:
uid = None if not uid_list else uid_list[idx]
seen = None if not seen_status else get_seen(seen_status.get(uid, None))
communication = self.insert_communication(msg, _uid=uid, _seen=seen)
unique_id = None if not unique_id_list else unique_id_list.get(uid, None)
communication = self.insert_communication(msg, _uid=uid, _seen=seen, unique_id=unique_id)
#self.notify_update()
except SentEmailInInbox:
@ -294,7 +298,7 @@ class EmailAccount(Document):
unhandled_email.save()
frappe.db.commit()
def insert_communication(self, msg, _uid=None, _seen=None):
def insert_communication(self, msg, _uid=None, _seen=None, unique_id=None):
if isinstance(msg,list):
raw, uid, seen = msg
else:
@ -312,6 +316,16 @@ class EmailAccount(Document):
# dont count emails sent by the system get those
raise SentEmailInInbox
name = frappe.db.get_value("Communication", { "unique_id": unique_id })
if name:
# email is already available update communication uid instead
communication = frappe.get_doc("Communication", name)
communication.uid = uid
communication.save(ignore_permissions=True)
communication._attachments = []
return communication
communication = frappe.get_doc({
"doctype": "Communication",
"subject": email.subject,
@ -328,7 +342,8 @@ class EmailAccount(Document):
"message_id": email.message_id,
"communication_date": email.date,
"has_attachment": 1 if email.attachments else 0,
"seen": seen
"seen": seen,
"unique_id": unique_id
})
self.set_thread(communication, email)
@ -346,7 +361,6 @@ class EmailAccount(Document):
communication._attachments = email.save_attachments_in_doc(communication)
# replace inline images
dirty = False
for file in communication._attachments:
if file.name in email.cid_map and email.cid_map[file.name]:
@ -553,20 +567,17 @@ class EmailAccount(Document):
if not self.use_imap:
return
flags = frappe.get_all("Email Flag Queue", {
"email_account": self.name,
"action": "Seen"
}, ["name", "uid", "communication"])
flags = frappe.db.sql("""select name, uid from `tabCommunication` where sent_or_received = "Received"
and seen = 0 and communication_medium = "Email" and email_account='{email_account}' and
ifnull(_seen, '') = ''""".format(email_account=self.name), as_dict=True)
uid_list = [ flag.get("uid") for flag in flags ]
uid_list = list(set([ flag.get("uid") for flag in flags ]))
if flags and uid_list:
email_server = self.get_incoming_server()
email_server.mark_as_seen(uid_list=uid_list)
marked_as_seen = email_server.mark_as_seen(uid_list=uid_list)
# delete Email Flag Queue
for flag in flags:
frappe.db.set_value("Communication", flag.get("communication"), "seen", 1)
frappe.delete_doc("Email Flag Queue", flag.get("name"))
docnames = ",".join([ "'%s'"%uid for uid in uid_list ])
frappe.db.sql(""" update `tabCommunication` set seen=1 where name in ({docnames})""".format(docnames=docnames))
@frappe.whitelist()
def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None):

92
frappe/email/receive.py
View file

@ -2,7 +2,7 @@
# MIT License. See license.txt
from __future__ import unicode_literals
import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re
import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re, hashlib
from email_reply_parser import EmailReplyParser
from email.header import decode_header
import frappe
@ -109,12 +109,13 @@ class EmailServer:
self.errors = False
self.latest_messages = []
self.seen_status = {}
self.unique_id_list = {}
uid_list = email_list = self.get_new_mails()
num = num_copy = len(email_list)
# WARNING: Hard coded max no. of messages to be popped
if num > 20: num = 20
if num > 50: num = 50
# size limits
self.total_size = 0
@ -156,7 +157,8 @@ class EmailServer:
if self.settings.use_imap:
out.update({
"uid_list": uid_list,
"seen_status": self.seen_status
"seen_status": self.seen_status,
"unique_id_list": self.unique_id_list
})
return out
@ -164,10 +166,9 @@ class EmailServer:
def get_new_mails(self):
"""Return list of new mails"""
if cint(self.settings.use_imap):
if not self.check_imap_uidvalidity():
frappe.throw(_("UIDVALIDITY is changed in imap server"))
self.check_imap_uidvalidity()
self.imap.select("Inbox")
self.imap.select("Inbox", readonly=True)
response, message = self.imap.uid('search', None, self.settings.email_sync_rule)
email_list = message[0].split()
else:
@ -179,19 +180,27 @@ class EmailServer:
# compare the UIDVALIDITY of email account and imap server
uid_validity = self.settings.uid_validity
responce, message = self.imap.status("Inbox", "(UIDVALIDITY)")
responce, message = self.imap.status("Inbox", "(UIDVALIDITY UIDNEXT)")
current_uid_validity = self.parse_imap_responce("UIDVALIDITY", message[0])
if not current_uid_validity:
frappe.throw(_("Can not find UIDVALIDITY in imap status response"))
if not uid_validity:
# uid validity is not available for email account
frappe.db.set_value("Email Account", self.settings.email_account, "uidvalidity", current_uid_validity)
return True
uidnext = int(self.parse_imap_responce("UIDNEXT", message[0]) or "1")
frappe.db.set_value("Email Account", self.settings.email_account, "uidnext", uidnext)
if not uid_validity or uid_validity != current_uid_validity:
# uidvalidity changed & all email uids are reindexed by server
frappe.db.sql("""update `tabCommunication` set uid=-1 where communication_medium='Email'
and email_account='{email_account}'""".format(email_account=self.settings.email_account))
frappe.db.sql(""" update `tabEmail Account` set uidvalidity='{uidvalidity}', uidnext={uidnext} where
name='{email_account}'""".format(uidvalidity=current_uid_validity, uidnext=uidnext, email_account=self.settings.email_account))
from_uid = 1 if uidnext < 101 or (uidnext - 100) < 1 else uidnext - 100
# sync last 100 email
self.settings.email_sync_rule = "UID {}:{}".format(from_uid, uidnext)
elif uid_validity == current_uid_validity:
return True
else:
# UIDs are reindexed on imap server
# self.settings.email_sync_rule = "UNSEEN"
return False
return
def parse_imap_responce(self, cmd, responce):
pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd)
@ -207,11 +216,13 @@ class EmailServer:
self.validate_message_limits(message_meta)
if cint(self.settings.use_imap):
status, response = self.imap.uid("fetch", message_meta, "(FLAGS)")
self.get_mail_seen_status(message_meta, response[0])
status, message = self.imap.uid('fetch', message_meta, '(RFC822 BODY.PEEK[HEADER] FLAGS)')
raw, header, ignore = message
status, message = self.imap.uid('fetch', message_meta, '(RFC822)')
self.latest_messages.append(message[0][1])
self.get_email_seen_status(message_meta, header[0])
self.get_email_headers_hash(message_meta, header[1])
self.latest_messages.append(raw[1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
@ -221,6 +232,7 @@ class EmailServer:
raise
except Exception, e:
print e
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
@ -243,8 +255,8 @@ class EmailServer:
# mark as seen
self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
def get_mail_seen_status(self, uid, flag_string):
# parse the mail flags
def get_email_seen_status(self, uid, flag_string):
""" parse the email FLAGS response """
if not flag_string:
return None
@ -259,6 +271,28 @@ class EmailServer:
else:
self.seen_status.update({ uid: "UNSEEN" })
def get_email_headers_hash(self, uid, headers):
""" generate the email unique id from header hash
unique id can be used to update uid if UID is reindexed"""
hash = hashlib.sha1()
for header in headers:
if header[0] == 'Content-Type':
# skip variable boundaries
continue
try:
decoded_header = decode_header(header[1])
decoded = ''.join([val[0].decode(val[1]).encode('ascii', 'ignore') \
if val[1] is not None else val[0] for val in decoded_header])
cleaned = re.sub(r"\s+", u"", decoded, flags=re.UNICODE)
hash.update(cleaned)
except:
pass
self.unique_id_list.update({ uid: hash.hexdigest() })
print self.unique_id_list
def has_login_limit_exceeded(self, e):
return "-ERR Exceeded the login limit" in strip(cstr(e.message))
@ -302,13 +336,21 @@ class EmailServer:
return error_msg
def mark_as_seen(uid_list=[]):
def mark_as_seen(self, uid_list=[]):
""" set all uids mails the flag as seen """
if not uid_list:
return
uid = ",".join(uid_list)
self.imap.uid('STORE', uid, '+FLAGS', '(\\SEEN)')
if not self.connect():
return
self.imap.select("Inbox")
for uid in uid_list:
try:
self.imap.uid('STORE', uid, '+FLAGS', '(\\SEEN)')
except Exception as e:
continue
class Email:
"""Wrapper for an email."""