diff --git a/frappe/change_log/v6/v6_12_0.md b/frappe/change_log/v6/v6_12_0.md new file mode 100644 index 0000000000..abcdd16275 --- /dev/null +++ b/frappe/change_log/v6/v6_12_0.md @@ -0,0 +1 @@ +- Extract emails using IMAP. Contributed by Gangadhar Kadam ([New Indictrans](http://indictranstech.com/)) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 8781ce6a0c..b039ba6fea 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -1,6 +1,6 @@ email_defaults = { "GMail": { - "pop3_server": "pop.gmail.com", + "email_server": "pop.gmail.com", "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.gmail.com", @@ -8,7 +8,7 @@ email_defaults = { "use_tls": 1 }, "Outlook.com": { - "pop3_server": "pop3.live.com", + "email_server": "pop3.live.com", "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.live.com", @@ -16,7 +16,7 @@ email_defaults = { "use_tls": 1 }, "Yahoo Mail": { - "pop3_server": "pop.mail.yahoo.com", + "email_server": "pop.mail.yahoo.com", "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.mail.yahoo.com", @@ -24,7 +24,7 @@ email_defaults = { "use_tls": 1 }, "Yandex.Mail": { - "pop3_server": "pop.yandex.com", + "email_server": "pop.yandex.com", "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.yandex.com", @@ -33,11 +33,44 @@ email_defaults = { }, }; +email_defaults_imap = { + "GMail": { + "email_server": "imap.gmail.com" + }, + "Outlook.com": { + "email_server": "imap.live.com" + }, + "Yahoo Mail": { + "email_server": "imap.mail.yahoo.com" + }, + "Yandex.Mail": { + "email_server": "imap.yandex.com" + }, + +}; + frappe.ui.form.on("Email Account", { service: function(frm) { $.each(email_defaults[frm.doc.service], function(key, value) { frm.set_value(key, value); }) + if (frm.doc.use_imap) { + $.each(email_defaults_imap[frm.doc.service], function(key, value) { + frm.set_value(key, value); + }); + } + }, + use_imap: function(frm) { + if (frm.doc.use_imap) { + $.each(email_defaults_imap[frm.doc.service], function(key, value) { + frm.set_value(key, value); + }); + } + else{ + $.each(email_defaults[frm.doc.service], function(key, value) { + frm.set_value(key, value); + }); + } }, email_id: function(frm) { if(!frm.doc.email_account_name) { diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 1239a257d6..e2a0ef04e3 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -25,6 +25,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -49,6 +50,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -73,6 +75,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 1, @@ -96,6 +99,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -120,6 +124,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -143,6 +148,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -167,6 +173,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -190,6 +197,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -215,6 +223,32 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "depends_on": "", + "fieldname": "use_imap", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "in_filter": 0, + "in_list_view": 0, + "label": "Use IMAP", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -227,19 +261,20 @@ "bold": 0, "collapsible": 0, "depends_on": "enable_incoming", - "description": "e.g. pop.gmail.com", - "fieldname": "pop3_server", + "description": "e.g. pop.gmail.com / imap.gmail.com", + "fieldname": "email_server", "fieldtype": "Data", "hidden": 0, "ignore_user_permissions": 0, "in_filter": 0, "in_list_view": 0, - "label": "POP3 Server", + "label": "Email Server", "length": 0, "no_copy": 0, "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -264,6 +299,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -290,6 +326,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -316,6 +353,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -341,6 +379,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -364,6 +403,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -387,6 +427,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -412,6 +453,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -437,6 +479,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -460,6 +503,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -485,6 +529,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -510,6 +555,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -534,6 +580,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -559,6 +606,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -584,6 +632,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -609,6 +658,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -632,6 +682,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -656,6 +707,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -680,6 +732,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -703,6 +756,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -726,6 +780,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -751,6 +806,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -774,6 +830,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -797,6 +854,7 @@ "permlevel": 0, "precision": "", "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "report_hide": 0, "reqd": 0, @@ -808,13 +866,14 @@ "hide_heading": 0, "hide_toolbar": 0, "icon": "icon-inbox", + "idx": 0, "in_create": 0, "in_dialog": 0, "is_submittable": 0, "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2015-11-16 06:29:45.876335", + "modified": "2015-12-02 02:27:34.031592", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 0c7583874b..7ca1448dde 100644 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -3,20 +3,21 @@ from __future__ import unicode_literals import frappe +import imaplib +import re +import socket from frappe import _ from frappe.model.document import Document from frappe.utils import validate_email_add, cint, get_datetime, DATE_FORMAT, strip, comma_or from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import POP3Server, Email +from frappe.email.receive import EmailServer, Email from poplib import error_proto -import re from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta from frappe.desk.form import assign_to from frappe.utils.user import get_system_managers -import socket class SentEmailInInbox(Exception): pass @@ -33,7 +34,7 @@ class EmailAccount(Document): self.name = self.email_account_name def validate(self): - """Validate email id and check POP3 and SMTP connections is enabled.""" + """Validate email id and check POP3/IMAP and SMTP connections is enabled.""" if self.email_id: validate_email_add(self.email_id, True) @@ -51,7 +52,7 @@ class EmailAccount(Document): if not frappe.local.flags.in_install and not frappe.local.flags.in_patch: if self.enable_incoming: - self.get_pop3() + self.get_server() if self.enable_outgoing: self.check_smtp() @@ -99,23 +100,23 @@ class EmailAccount(Document): ) server.sess - def get_pop3(self, in_receive=False): + def get_server(self, in_receive=False): """Returns logged in POP3 connection object.""" args = { - "host": self.pop3_server, + "host": self.email_server, "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, - "password": self.password + "password": self.password, + "use_imap": self.use_imap } - if not self.pop3_server: - frappe.throw(_("{0} is required").format("POP3 Server")) + if not args.get("host"): + frappe.throw(_("{0} is required").format("Email Server")) - pop3 = POP3Server(frappe._dict(args)) + email_server = EmailServer(frappe._dict(args)) try: - pop3.connect() - - except error_proto, e: + email_server.connect() + except (error_proto, imaplib.IMAP4.error), e: if in_receive and e.message=="-ERR authentication failed": # if called via self.receive and it leads to authentication error, disable incoming # and send email to system manager @@ -139,7 +140,7 @@ class EmailAccount(Document): else: raise - return pop3 + return email_server def handle_incoming_connect_error(self, description): self.db_set("enable_incoming", 0) @@ -155,16 +156,16 @@ class EmailAccount(Document): }) def receive(self, test_mails=None): - """Called by scheduler to receive emails from this EMail account using POP3.""" + """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" if self.enable_incoming: if frappe.local.flags.in_test: incoming_mails = test_mails else: - pop3 = self.get_pop3(in_receive=True) - if not pop3: + email_server = self.get_server(in_receive=True) + if not email_server: return - incoming_mails = pop3.get_messages() + incoming_mails = email_server.get_messages() exceptions = [] for raw in incoming_mails: @@ -356,11 +357,10 @@ def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len if not txt: txt = "" return [[d] for d in frappe.get_hooks("email_append_to") if txt in d] -def pull(now=False): - """Will be called via scheduler, pull emails from all enabled POP3 email accounts.""" +def pull(now=True): + """Will be called via scheduler, pull emails from all enabled Email accounts.""" import frappe.tasks for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1}): - #frappe.tasks.pull_from_email_account(frappe.local.site, email_account.name) if now: frappe.tasks.pull_from_email_account(frappe.local.site, email_account.name) else: diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 7b5711d17a..4eb2de27f6 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import time -import socket, poplib +import _socket, poplib, imaplib import frappe from frappe import _ from frappe.utils import extract_email_id, convert_utc_to_user_timezone, now, cint, cstr, strip @@ -17,7 +17,7 @@ class EmailTimeoutError(frappe.ValidationError): pass class TotalSizeExceededError(frappe.ValidationError): pass class LoginLimitExceeded(frappe.ValidationError): pass -class POP3Server: +class EmailServer: """Wrapper for POP server to pull emails.""" def __init__(self, args=None): self.setup(args) @@ -36,6 +36,33 @@ class POP3Server: def connect(self): """Connect to **Email Account**.""" + if cint(self.settings.use_imap): + return self.connect_imap() + else: + return self.connect_pop() + + def connect_imap(self): + """Connect to IMAP""" + try: + if cint(self.settings.use_ssl): + self.imap = Timed_IMAP4_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout")) + else: + self.imap = Timed_IMAP4(self.settings.host, timeout=frappe.conf.get("pop_timeout")) + self.imap.login(self.settings.username, self.settings.password) + # connection established! + return True + + except _socket.error: + # Invalid mail server -- due to refusing connection + frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.')) + raise + + except Exception, e: + frappe.msgprint(_('Cannot connect: {0}').format(str(e))) + raise + + def connect_pop(self): + #this method return pop connection try: if cint(self.settings.use_ssl): self.pop = Timed_POP3_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout")) @@ -48,7 +75,7 @@ class POP3Server: # connection established! return True - except socket.error: + except _socket.error: # Invalid mail server -- due to refusing connection frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.')) raise @@ -75,8 +102,9 @@ class POP3Server: # track if errors arised self.errors = False self.latest_messages = [] - pop_list = self.pop.list()[1] - num = num_copy = len(pop_list) + + email_list = self.get_new_mails() + num = num_copy = len(email_list) # WARNING: Hard coded max no. of messages to be popped if num > 20: num = 20 @@ -86,22 +114,23 @@ class POP3Server: self.max_email_size = cint(frappe.local.conf.get("max_email_size")) self.max_total_size = 5 * self.max_email_size - for i, pop_meta in enumerate(pop_list): + for i, message_meta in enumerate(email_list): # do not pull more than NUM emails if (i+1) > num: break try: - self.retrieve_message(pop_meta, i+1) + self.retrieve_message(message_meta, i+1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): break # WARNING: Mark as read - message number 101 onwards from the pop list # This is to avoid having too many messages entering the system num = num_copy - if num > 100 and not self.errors: - for m in xrange(101, num+1): - self.pop.dele(m) + if not cint(self.settings.use_imap): + if num > 100 and not self.errors: + for m in xrange(101, num+1): + self.pop.dele(m) except Exception, e: if self.has_login_limit_exceeded(e): @@ -112,17 +141,35 @@ class POP3Server: finally: # no matter the exception, pop should quit if connected - self.pop.quit() + if cint(self.settings.use_imap): + self.imap.logout() + else: + self.pop.quit() return self.latest_messages - def retrieve_message(self, pop_meta, msg_num): + def get_new_mails(self): + """Return list of new mails""" + if cint(self.settings.use_imap): + self.imap.select("Inbox") + response, message = self.imap.uid('search', None, "UNSEEN") + email_list = message[0].split() + else: + email_list = self.pop.list()[1] + + return email_list + + def retrieve_message(self, message_meta, msg_num=None): incoming_mail = None try: - self.validate_pop(pop_meta) - msg = self.pop.retr(msg_num) + self.validate_message_limits(message_meta) - self.latest_messages.append(b'\n'.join(msg[1])) + if cint(self.settings.use_imap): + status, message = self.imap.uid('fetch', message_meta, '(RFC822)') + self.latest_messages.append(message[0][1]) + else: + msg = self.pop.retr(msg_num) + self.latest_messages.append(b'\n'.join(msg[1])) except (TotalSizeExceededError, EmailTimeoutError): # propagate this error to break the loop @@ -140,9 +187,11 @@ class POP3Server: self.errors = True frappe.db.rollback() - self.pop.dele(msg_num) + if not cint(self.settings.use_imap): + self.pop.dele(msg_num) else: - self.pop.dele(msg_num) + if not cint(self.settings.use_imap): + self.pop.dele(msg_num) def has_login_limit_exceeded(self, e): return "-ERR Exceeded the login limit" in strip(cstr(e.message)) @@ -157,12 +206,12 @@ class POP3Server: return True return False - def validate_pop(self, pop_meta): + def validate_message_limits(self, message_meta): # throttle based on email size if not self.max_email_size: return - m, size = pop_meta.split() + m, size = message_meta.split() size = cint(size) if size < self.max_email_size: @@ -355,3 +404,8 @@ class Timed_POP3(TimerMixin, poplib.POP3): class Timed_POP3_SSL(TimerMixin, poplib.POP3_SSL): _super = poplib.POP3_SSL +class Timed_IMAP4(TimerMixin, imaplib.IMAP4): + _super = imaplib.IMAP4 + +class Timed_IMAP4_SSL(TimerMixin, imaplib.IMAP4_SSL): + _super = imaplib.IMAP4_SSL diff --git a/frappe/patches.txt b/frappe/patches.txt index 377b46889b..601c6587ba 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -106,3 +106,4 @@ frappe.patches.v6_6.user_last_active frappe.patches.v6_6.rename_slovak_language frappe.patches.v6_6.fix_file_url frappe.patches.v6_9.rename_burmese_language +frappe.patches.v6_11.rename_field_in_email_account diff --git a/frappe/patches/v6_11/__init__.py b/frappe/patches/v6_11/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/patches/v6_11/rename_field_in_email_account.py b/frappe/patches/v6_11/rename_field_in_email_account.py new file mode 100644 index 0000000000..77549027aa --- /dev/null +++ b/frappe/patches/v6_11/rename_field_in_email_account.py @@ -0,0 +1,5 @@ +import frappe + +def execute(): + frappe.reload_doctype("Email Account") + frappe.db.sql("update `tabEmail Account` set email_server = pop3_server")