[fixes] [imap] cleanup, rename fields, re-org

This commit is contained in:
Rushabh Mehta 2015-12-02 13:06:51 +05:30
commit 8950bfce82
8 changed files with 202 additions and 49 deletions

View file

@ -0,0 +1 @@
- Extract emails using IMAP. Contributed by Gangadhar Kadam ([New Indictrans](http://indictranstech.com/))

View file

@ -1,6 +1,6 @@
email_defaults = {
"GMail": {
"pop3_server": "pop.gmail.com",
"email_server": "pop.gmail.com",
"use_ssl": 1,
"enable_outgoing": 1,
"smtp_server": "smtp.gmail.com",
@ -8,7 +8,7 @@ email_defaults = {
"use_tls": 1
},
"Outlook.com": {
"pop3_server": "pop3.live.com",
"email_server": "pop3.live.com",
"use_ssl": 1,
"enable_outgoing": 1,
"smtp_server": "smtp.live.com",
@ -16,7 +16,7 @@ email_defaults = {
"use_tls": 1
},
"Yahoo Mail": {
"pop3_server": "pop.mail.yahoo.com",
"email_server": "pop.mail.yahoo.com",
"use_ssl": 1,
"enable_outgoing": 1,
"smtp_server": "smtp.mail.yahoo.com",
@ -24,7 +24,7 @@ email_defaults = {
"use_tls": 1
},
"Yandex.Mail": {
"pop3_server": "pop.yandex.com",
"email_server": "pop.yandex.com",
"use_ssl": 1,
"enable_outgoing": 1,
"smtp_server": "smtp.yandex.com",
@ -33,11 +33,44 @@ email_defaults = {
},
};
email_defaults_imap = {
"GMail": {
"email_server": "imap.gmail.com"
},
"Outlook.com": {
"email_server": "imap.live.com"
},
"Yahoo Mail": {
"email_server": "imap.mail.yahoo.com"
},
"Yandex.Mail": {
"email_server": "imap.yandex.com"
},
};
frappe.ui.form.on("Email Account", {
service: function(frm) {
$.each(email_defaults[frm.doc.service], function(key, value) {
frm.set_value(key, value);
})
if (frm.doc.use_imap) {
$.each(email_defaults_imap[frm.doc.service], function(key, value) {
frm.set_value(key, value);
});
}
},
use_imap: function(frm) {
if (frm.doc.use_imap) {
$.each(email_defaults_imap[frm.doc.service], function(key, value) {
frm.set_value(key, value);
});
}
else{
$.each(email_defaults[frm.doc.service], function(key, value) {
frm.set_value(key, value);
});
}
},
email_id: function(frm) {
if(!frm.doc.email_account_name) {

View file

@ -25,6 +25,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -49,6 +50,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -73,6 +75,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 1,
@ -96,6 +99,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -120,6 +124,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -143,6 +148,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -167,6 +173,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -190,6 +197,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -215,6 +223,32 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"unique": 0
},
{
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"depends_on": "",
"fieldname": "use_imap",
"fieldtype": "Check",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "Use IMAP",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -227,19 +261,20 @@
"bold": 0,
"collapsible": 0,
"depends_on": "enable_incoming",
"description": "e.g. pop.gmail.com",
"fieldname": "pop3_server",
"description": "e.g. pop.gmail.com / imap.gmail.com",
"fieldname": "email_server",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"in_filter": 0,
"in_list_view": 0,
"label": "POP3 Server",
"label": "Email Server",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -264,6 +299,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -290,6 +326,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -316,6 +353,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -341,6 +379,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -364,6 +403,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -387,6 +427,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -412,6 +453,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -437,6 +479,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -460,6 +503,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -485,6 +529,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -510,6 +555,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -534,6 +580,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -559,6 +606,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -584,6 +632,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -609,6 +658,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -632,6 +682,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -656,6 +707,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -680,6 +732,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -703,6 +756,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -726,6 +780,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -751,6 +806,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -774,6 +830,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -797,6 +854,7 @@
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"report_hide": 0,
"reqd": 0,
@ -808,13 +866,14 @@
"hide_heading": 0,
"hide_toolbar": 0,
"icon": "icon-inbox",
"idx": 0,
"in_create": 0,
"in_dialog": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2015-11-16 06:29:45.876335",
"modified": "2015-12-02 02:27:34.031592",
"modified_by": "Administrator",
"module": "Email",
"name": "Email Account",

View file

@ -3,20 +3,21 @@
from __future__ import unicode_literals
import frappe
import imaplib
import re
import socket
from frappe import _
from frappe.model.document import Document
from frappe.utils import validate_email_add, cint, get_datetime, DATE_FORMAT, strip, comma_or
from frappe.utils.user import is_system_user
from frappe.utils.jinja import render_template
from frappe.email.smtp import SMTPServer
from frappe.email.receive import POP3Server, Email
from frappe.email.receive import EmailServer, Email
from poplib import error_proto
import re
from dateutil.relativedelta import relativedelta
from datetime import datetime, timedelta
from frappe.desk.form import assign_to
from frappe.utils.user import get_system_managers
import socket
class SentEmailInInbox(Exception): pass
@ -33,7 +34,7 @@ class EmailAccount(Document):
self.name = self.email_account_name
def validate(self):
"""Validate email id and check POP3 and SMTP connections is enabled."""
"""Validate email id and check POP3/IMAP and SMTP connections is enabled."""
if self.email_id:
validate_email_add(self.email_id, True)
@ -51,7 +52,7 @@ class EmailAccount(Document):
if not frappe.local.flags.in_install and not frappe.local.flags.in_patch:
if self.enable_incoming:
self.get_pop3()
self.get_server()
if self.enable_outgoing:
self.check_smtp()
@ -99,23 +100,23 @@ class EmailAccount(Document):
)
server.sess
def get_pop3(self, in_receive=False):
def get_server(self, in_receive=False):
"""Returns logged in POP3 connection object."""
args = {
"host": self.pop3_server,
"host": self.email_server,
"use_ssl": self.use_ssl,
"username": getattr(self, "login_id", None) or self.email_id,
"password": self.password
"password": self.password,
"use_imap": self.use_imap
}
if not self.pop3_server:
frappe.throw(_("{0} is required").format("POP3 Server"))
if not args.get("host"):
frappe.throw(_("{0} is required").format("Email Server"))
pop3 = POP3Server(frappe._dict(args))
email_server = EmailServer(frappe._dict(args))
try:
pop3.connect()
except error_proto, e:
email_server.connect()
except (error_proto, imaplib.IMAP4.error), e:
if in_receive and e.message=="-ERR authentication failed":
# if called via self.receive and it leads to authentication error, disable incoming
# and send email to system manager
@ -139,7 +140,7 @@ class EmailAccount(Document):
else:
raise
return pop3
return email_server
def handle_incoming_connect_error(self, description):
self.db_set("enable_incoming", 0)
@ -155,16 +156,16 @@ class EmailAccount(Document):
})
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3."""
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
if self.enable_incoming:
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
pop3 = self.get_pop3(in_receive=True)
if not pop3:
email_server = self.get_server(in_receive=True)
if not email_server:
return
incoming_mails = pop3.get_messages()
incoming_mails = email_server.get_messages()
exceptions = []
for raw in incoming_mails:
@ -356,11 +357,10 @@ def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len
if not txt: txt = ""
return [[d] for d in frappe.get_hooks("email_append_to") if txt in d]
def pull(now=False):
"""Will be called via scheduler, pull emails from all enabled POP3 email accounts."""
def pull(now=True):
"""Will be called via scheduler, pull emails from all enabled Email accounts."""
import frappe.tasks
for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1}):
#frappe.tasks.pull_from_email_account(frappe.local.site, email_account.name)
if now:
frappe.tasks.pull_from_email_account(frappe.local.site, email_account.name)
else:

View file

@ -3,7 +3,7 @@
from __future__ import unicode_literals
import time
import socket, poplib
import _socket, poplib, imaplib
import frappe
from frappe import _
from frappe.utils import extract_email_id, convert_utc_to_user_timezone, now, cint, cstr, strip
@ -17,7 +17,7 @@ class EmailTimeoutError(frappe.ValidationError): pass
class TotalSizeExceededError(frappe.ValidationError): pass
class LoginLimitExceeded(frappe.ValidationError): pass
class POP3Server:
class EmailServer:
"""Wrapper for POP server to pull emails."""
def __init__(self, args=None):
self.setup(args)
@ -36,6 +36,33 @@ class POP3Server:
def connect(self):
"""Connect to **Email Account**."""
if cint(self.settings.use_imap):
return self.connect_imap()
else:
return self.connect_pop()
def connect_imap(self):
"""Connect to IMAP"""
try:
if cint(self.settings.use_ssl):
self.imap = Timed_IMAP4_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
else:
self.imap = Timed_IMAP4(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
self.imap.login(self.settings.username, self.settings.password)
# connection established!
return True
except _socket.error:
# Invalid mail server -- due to refusing connection
frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
raise
except Exception, e:
frappe.msgprint(_('Cannot connect: {0}').format(str(e)))
raise
def connect_pop(self):
#this method return pop connection
try:
if cint(self.settings.use_ssl):
self.pop = Timed_POP3_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
@ -48,7 +75,7 @@ class POP3Server:
# connection established!
return True
except socket.error:
except _socket.error:
# Invalid mail server -- due to refusing connection
frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
raise
@ -75,8 +102,9 @@ class POP3Server:
# track if errors arised
self.errors = False
self.latest_messages = []
pop_list = self.pop.list()[1]
num = num_copy = len(pop_list)
email_list = self.get_new_mails()
num = num_copy = len(email_list)
# WARNING: Hard coded max no. of messages to be popped
if num > 20: num = 20
@ -86,22 +114,23 @@ class POP3Server:
self.max_email_size = cint(frappe.local.conf.get("max_email_size"))
self.max_total_size = 5 * self.max_email_size
for i, pop_meta in enumerate(pop_list):
for i, message_meta in enumerate(email_list):
# do not pull more than NUM emails
if (i+1) > num:
break
try:
self.retrieve_message(pop_meta, i+1)
self.retrieve_message(message_meta, i+1)
except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded):
break
# WARNING: Mark as read - message number 101 onwards from the pop list
# This is to avoid having too many messages entering the system
num = num_copy
if num > 100 and not self.errors:
for m in xrange(101, num+1):
self.pop.dele(m)
if not cint(self.settings.use_imap):
if num > 100 and not self.errors:
for m in xrange(101, num+1):
self.pop.dele(m)
except Exception, e:
if self.has_login_limit_exceeded(e):
@ -112,17 +141,35 @@ class POP3Server:
finally:
# no matter the exception, pop should quit if connected
self.pop.quit()
if cint(self.settings.use_imap):
self.imap.logout()
else:
self.pop.quit()
return self.latest_messages
def retrieve_message(self, pop_meta, msg_num):
def get_new_mails(self):
"""Return list of new mails"""
if cint(self.settings.use_imap):
self.imap.select("Inbox")
response, message = self.imap.uid('search', None, "UNSEEN")
email_list = message[0].split()
else:
email_list = self.pop.list()[1]
return email_list
def retrieve_message(self, message_meta, msg_num=None):
incoming_mail = None
try:
self.validate_pop(pop_meta)
msg = self.pop.retr(msg_num)
self.validate_message_limits(message_meta)
self.latest_messages.append(b'\n'.join(msg[1]))
if cint(self.settings.use_imap):
status, message = self.imap.uid('fetch', message_meta, '(RFC822)')
self.latest_messages.append(message[0][1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
@ -140,9 +187,11 @@ class POP3Server:
self.errors = True
frappe.db.rollback()
self.pop.dele(msg_num)
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
else:
self.pop.dele(msg_num)
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
def has_login_limit_exceeded(self, e):
return "-ERR Exceeded the login limit" in strip(cstr(e.message))
@ -157,12 +206,12 @@ class POP3Server:
return True
return False
def validate_pop(self, pop_meta):
def validate_message_limits(self, message_meta):
# throttle based on email size
if not self.max_email_size:
return
m, size = pop_meta.split()
m, size = message_meta.split()
size = cint(size)
if size < self.max_email_size:
@ -355,3 +404,8 @@ class Timed_POP3(TimerMixin, poplib.POP3):
class Timed_POP3_SSL(TimerMixin, poplib.POP3_SSL):
_super = poplib.POP3_SSL
class Timed_IMAP4(TimerMixin, imaplib.IMAP4):
_super = imaplib.IMAP4
class Timed_IMAP4_SSL(TimerMixin, imaplib.IMAP4_SSL):
_super = imaplib.IMAP4_SSL

View file

@ -106,3 +106,4 @@ frappe.patches.v6_6.user_last_active
frappe.patches.v6_6.rename_slovak_language
frappe.patches.v6_6.fix_file_url
frappe.patches.v6_9.rename_burmese_language
frappe.patches.v6_11.rename_field_in_email_account

View file

View file

@ -0,0 +1,5 @@
import frappe
def execute():
frappe.reload_doctype("Email Account")
frappe.db.sql("update `tabEmail Account` set email_server = pop3_server")