diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 374e398b9e..a610127a6a 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -44,7 +44,7 @@ "allow_on_submit": 0, "bold": 0, "collapsible": 1, - "collapsible_depends_on": "eval:doc.communication_type==='Communication'", + "collapsible_depends_on": "", "columns": 0, "fieldname": "section_break_10", "fieldtype": "Section Break", @@ -1186,7 +1186,7 @@ "bold": 0, "collapsible": 0, "columns": 0, - "fieldname": "uid", + "fieldname": "signature", "fieldtype": "Data", "hidden": 0, "ignore_user_permissions": 0, @@ -1194,7 +1194,7 @@ "in_filter": 0, "in_list_view": 0, "in_standard_filter": 0, - "label": "UID", + "label": "Signature", "length": 0, "no_copy": 0, "permlevel": 0, @@ -1208,146 +1208,6 @@ "search_index": 0, "set_only_once": 0, "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "unique_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Unique id", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Deleted", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "nomatch", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "No Match", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "has_attachment", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Has Attachment", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "timeline_hide", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Hidden in Timeline", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 } ], "hide_heading": 0, @@ -1361,7 +1221,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2017-01-20 04:19:00.510797", + "modified": "2017-01-20 05:20:58.187840", "modified_by": "Administrator", "module": "Core", "name": "Communication", diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 34f3c6af83..e5f7e83dc3 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -48,25 +48,24 @@ class Communication(Document): def after_insert(self): if not (self.reference_doctype and self.reference_name): return - if not self.timeline_hide: - if self.communication_type in ("Communication", "Comment"): - # send new comment to listening clients - frappe.publish_realtime('new_communication', self.as_dict(), - doctype=self.reference_doctype, docname=self.reference_name, - after_commit=True) + if self.communication_type in ("Communication", "Comment"): + # send new comment to listening clients + frappe.publish_realtime('new_communication', self.as_dict(), + doctype=self.reference_doctype, docname=self.reference_name, + after_commit=True) - if self.communication_type == "Comment": - notify_mentions(self) + if self.communication_type == "Comment": + notify_mentions(self) - elif self.communication_type in ("Chat", "Notification", "Bot"): - if self.reference_name == frappe.session.user: - message = self.as_dict() - message['broadcast'] = True - frappe.publish_realtime('new_message', message, after_commit=True) - else: - # reference_name contains the user who is addressed in the messages' page comment - frappe.publish_realtime('new_message', self.as_dict(), - user=self.reference_name, after_commit=True) + elif self.communication_type in ("Chat", "Notification", "Bot"): + if self.reference_name == frappe.session.user: + message = self.as_dict() + message['broadcast'] = True + frappe.publish_realtime('new_message', message, after_commit=True) + else: + # reference_name contains the user who is addressed in the messages' page comment + frappe.publish_realtime('new_message', self.as_dict(), + user=self.reference_name, after_commit=True) def on_update(self): """Update parent status as `Open` or `Replied`.""" diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index ed8fe1ff5d..eb4867f876 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -148,14 +148,13 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= if not fields: fields = '''name, communication_type, communication_medium, comment_type, - content, sender, sender_full_name, communication_date, subject, delivery_status, _liked_by, + content, sender, sender_full_name, creation, subject, delivery_status, _liked_by, timeline_doctype, timeline_name, reference_doctype, reference_name, link_doctype, link_name, "Communication" as doctype''' conditions = '''communication_type in ("Communication", "Comment") - and timeline_hide is null and ( (reference_doctype=%(doctype)s and reference_name=%(name)s) or ( @@ -171,7 +170,7 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields= if after: # find after a particular date - conditions+= ' and communication_date > {0}'.format(after) + conditions+= ' and creation > {0}'.format(after) communications = frappe.db.sql("""select {fields} from tabCommunication diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 90f632c8a3..a99e1e184d 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -12,7 +12,7 @@ from frappe.utils import validate_email_add, cint, get_datetime, DATE_FORMAT, st from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, Email, get_unique_id +from frappe.email.receive import EmailServer, Email from poplib import error_proto from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta @@ -50,13 +50,14 @@ class EmailAccount(Document): #if self.enable_incoming and not self.append_to: # frappe.throw(_("Append To is mandatory for incoming mails")) - if not self.awaiting_password and not frappe.local.flags.in_install and not frappe.local.flags.in_patch: + if (not self.awaiting_password and not frappe.local.flags.in_install + and not frappe.local.flags.in_patch): if self.password: if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 - - + + if self.enable_outgoing: self.check_smtp() else: @@ -74,7 +75,7 @@ class EmailAccount(Document): if self.append_to not in valid_doctypes: frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) - + def on_update(self): """Check there is only one default of each type.""" @@ -141,9 +142,6 @@ class EmailAccount(Document): "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, "use_imap": self.use_imap, - "uid_validity":self.uid_validity, - "uidnext":self.uidnext, - "no_remaining":self.no_remaining }) if self.password: args.password = self.get_password() @@ -193,7 +191,7 @@ class EmailAccount(Document): if test_internet(): if self.get_failed_attempts_count() > 2: self.db_set("enable_incoming", 0) - + for user in get_system_managers(only_name=True): try: assign_to.add({ @@ -226,13 +224,7 @@ class EmailAccount(Document): incoming_mails = test_mails else: email_server = self.get_incoming_server(in_receive=True) - if not email_server: - return - try: - incoming_mails = email_server.get_messages() - except Exception as e: - frappe.db.sql("update `tabEmail Account` set no_remaining = NULL where name = %s",(email_server.settings.email_account), auto_commit=1) - incoming_mails = [] + incoming_mails = email_server.get_messages() for msg in incoming_mails: try: @@ -240,13 +232,10 @@ class EmailAccount(Document): communication = self.insert_communication(msg) #self.notify_update() - except SentEmailInInbox as e: + except SentEmailInInbox: frappe.db.rollback() - if self.use_imap: - self.handle_bad_emails(email_server, msg[1], msg[0], "sent email in inbox") - - except Exception as e: + except Exception: frappe.db.rollback() log('email_account.receive') if self.use_imap: @@ -257,16 +246,7 @@ class EmailAccount(Document): frappe.db.commit() attachments = [d.file_name for d in communication._attachments] - if communication.message_id and not communication.timeline_hide: - first = frappe.db.get_value("Communication", {"message_id": communication.message_id},["name"],order_by="creation",as_dict=1) - if first: - if first.name != communication.name: - frappe.db.sql("""update tabCommunication set timeline_hide =%s where name = %s""",(first.name,communication.name),auto_commit=1) - - if self.no_remaining == '0' and not frappe.local.flags.in_test: - if communication.reference_doctype : - if not communication.timeline_hide and not communication.unread_notification_sent: - communication.notify(attachments=attachments, fetched_from_email_account=True) + communication.notify(attachments=attachments, fetched_from_email_account=True) #notify if user is linked to account if len(incoming_mails)>0 and not frappe.local.flags.in_test: @@ -275,13 +255,12 @@ class EmailAccount(Document): if exceptions: raise Exception, frappe.as_json(exceptions) - def handle_bad_emails(self,email_server,uid,raw,reason): + def handle_bad_emails(self, email_server, uid, raw, reason): if cint(email_server.settings.use_imap): import email try: mail = email.message_from_string(raw) - unique_id = get_unique_id(mail) message_id = mail.get('Message-ID') except Exception: message_id = "can't be parsed" @@ -291,7 +270,6 @@ class EmailAccount(Document): "email_account": email_server.settings.email_account, "uid": uid, "message_id": message_id, - "unique_id":unique_id, "reason":reason }) unhandled_email.save() @@ -310,7 +288,7 @@ class EmailAccount(Document): # and we don't want emails sent by us to be pulled back into the system again # dont count emails sent by the system get those raise SentEmailInInbox - + communication = frappe.get_doc({ "doctype": "Communication", "subject": email.subject, @@ -319,23 +297,19 @@ class EmailAccount(Document): "sent_or_received": "Received", "sender_full_name": email.from_real_name, "sender": email.from_email, - "recipients": email.To, - "cc": email.CC, + "recipients": email.mail.get("To"), + "cc": email.mail.get("CC"), "email_account": self.name, "communication_medium": "Email", - "uid":uid, - "message_id":email.message_id, - "communication_date":email.date, + "uid": uid, + "message_id": email.message_id, + "communication_date": email.date, "has_attachment": 1 if email.attachments else 0, - "seen":seen, - "unique_id":email.unique_id + "seen": seen }) self.set_thread(communication, email) - if not self.no_remaining == '0': - communication.unread_notification_sent = 1 - communication.flags.in_receive = True communication.insert(ignore_permissions = 1) @@ -497,14 +471,6 @@ class EmailAccount(Document): # the true parent is the communication parent parent = frappe.get_doc(parent.reference_doctype, parent.reference_name) - if email.message_id: - first = frappe.db.get_value("Communication", {"message_id": email.message_id},["name", "reference_doctype", "reference_name"], order_by="creation", as_dict=1) - - if first: - # set timeline hide to parent doc so are linked - communication.timeline_hide = first.name - if frappe.db.exists(first.reference_doctype, first.reference_name): - parent = frappe._dict(doctype=first.reference_doctype, name=first.reference_name) return parent @@ -548,11 +514,12 @@ def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len return [[d] for d in frappe.get_hooks("email_append_to") if txt in d] def test_internet(host="8.8.8.8", port=53, timeout=3): - """ + """Returns True if internet is connected + Host: 8.8.8.8 (google-public-dns-a.google.com) - OpenPort: 53/tcp - Service: domain (DNS/TCP) - """ + OpenPort: 53/tcp + Service: domain (DNS/TCP) + """ try: socket.setdefaulttimeout(timeout) socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) @@ -597,7 +564,8 @@ def pull(now=False): else: return queued_jobs = get_jobs(site=frappe.local.site, key='job_name')[frappe.local.site] - for email_account in frappe.get_list("Email Account",["name", "no_remaining"], filters={"enable_incoming": 1, "awaiting_password": 0}): + for email_account in frappe.get_list("Email Account", + filters={"enable_incoming": 1, "awaiting_password": 0}): if now: pull_from_email_account(email_account.name) @@ -606,12 +574,8 @@ def pull(now=False): job_name = 'pull_from_email_account|{0}'.format(email_account.name) if job_name not in queued_jobs: - if email_account.no_remaining == '0': - enqueue(pull_from_email_account, 'short', event='all', job_name=job_name, - email_account=email_account.name) - else: - enqueue(pull_from_email_account, 'long', event='all', job_name=job_name, - email_account=email_account.name) + enqueue(pull_from_email_account, 'short', event='all', job_name=job_name, + email_account=email_account.name) def pull_from_email_account(email_account): '''Runs within a worker process''' diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.json b/frappe/email/doctype/unhandled_email/unhandled_email.json index ab1972bd71..07a86bccfb 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.json +++ b/frappe/email/doctype/unhandled_email/unhandled_email.json @@ -2,23 +2,27 @@ "allow_copy": 0, "allow_import": 0, "allow_rename": 0, + "beta": 0, "creation": "2016-04-14 09:41:45.892975", "custom": 0, "docstatus": 0, "doctype": "DocType", "document_type": "Setup", + "editable_grid": 0, "fields": [ { "allow_on_submit": 0, "bold": 0, "collapsible": 0, + "columns": 0, "fieldname": "email_account", "fieldtype": "Link", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, - "in_filter": 0, + "in_filter": 0, "in_list_view": 1, + "in_standard_filter": 0, "label": "Email Account", "length": 0, "no_copy": 0, @@ -28,6 +32,7 @@ "print_hide": 0, "print_hide_if_no_value": 0, "read_only": 0, + "remember_last_selected_value": 0, "report_hide": 0, "reqd": 0, "search_index": 0, @@ -38,13 +43,15 @@ "allow_on_submit": 0, "bold": 0, "collapsible": 0, + "columns": 0, "fieldname": "uid", "fieldtype": "Data", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, - "in_filter": 0, + "in_filter": 0, "in_list_view": 0, + "in_standard_filter": 0, "label": "uid", "length": 0, "no_copy": 0, @@ -53,6 +60,7 @@ "print_hide": 0, "print_hide_if_no_value": 0, "read_only": 0, + "remember_last_selected_value": 0, "report_hide": 0, "reqd": 0, "search_index": 0, @@ -63,13 +71,15 @@ "allow_on_submit": 0, "bold": 0, "collapsible": 0, + "columns": 0, "fieldname": "reason", "fieldtype": "Long Text", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, - "in_filter": 0, + "in_filter": 0, "in_list_view": 1, + "in_standard_filter": 0, "label": "Reason", "length": 0, "no_copy": 0, @@ -78,6 +88,7 @@ "print_hide": 0, "print_hide_if_no_value": 0, "read_only": 0, + "remember_last_selected_value": 0, "report_hide": 0, "reqd": 0, "search_index": 0, @@ -88,13 +99,15 @@ "allow_on_submit": 0, "bold": 0, "collapsible": 0, + "columns": 0, "fieldname": "message_id", "fieldtype": "Code", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, - "in_filter": 0, + "in_filter": 0, "in_list_view": 0, + "in_standard_filter": 0, "label": "Message-id", "length": 0, "no_copy": 0, @@ -103,6 +116,7 @@ "print_hide": 0, "print_hide_if_no_value": 0, "read_only": 0, + "remember_last_selected_value": 0, "report_hide": 0, "reqd": 0, "search_index": 0, @@ -113,38 +127,15 @@ "allow_on_submit": 0, "bold": 0, "collapsible": 0, - "fieldname": "unique_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "label": "Unique id", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, + "columns": 0, "fieldname": "raw", "fieldtype": "Code", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, - "in_filter": 0, + "in_filter": 0, "in_list_view": 0, + "in_standard_filter": 0, "label": "Raw Email", "length": 0, "no_copy": 0, @@ -153,6 +144,7 @@ "print_hide": 0, "print_hide_if_no_value": 0, "read_only": 0, + "remember_last_selected_value": 0, "report_hide": 0, "reqd": 0, "search_index": 0, @@ -163,13 +155,14 @@ "hide_heading": 0, "hide_toolbar": 0, "idx": 0, + "image_view": 0, "in_create": 0, "in_dialog": 0, "is_submittable": 0, "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2016-07-21 00:35:02.752145", + "modified": "2017-01-20 05:15:57.216825", "modified_by": "Administrator", "module": "Email", "name": "Unhandled Email", @@ -197,8 +190,11 @@ "write": 1 } ], + "quick_entry": 0, "read_only": 0, "read_only_onload": 0, "sort_field": "modified", - "sort_order": "DESC" + "sort_order": "DESC", + "track_changes": 0, + "track_seen": 0 } \ No newline at end of file diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index edbb410106..77ba8aea13 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -7,5 +7,4 @@ import frappe from frappe.model.document import Document class UnhandledEmail(Document): - def on_trash(self): - frappe.db.set_value("Email Account",self.email_account,"no_remaining",None) + pass diff --git a/frappe/email/page/email_inbox/email_inbox.js b/frappe/email/page/email_inbox/email_inbox.js index 6169455b79..e794951b67 100644 --- a/frappe/email/page/email_inbox/email_inbox.js +++ b/frappe/email/page/email_inbox/email_inbox.js @@ -153,7 +153,7 @@ frappe.Inbox = frappe.ui.Listing.extend({ doctype: this.doctype, fields:["name", "sender", "sender_full_name", "communication_date", "recipients", "cc","communication_medium", "subject", "status" ,"reference_doctype", "reference_name", "timeline_doctype", "timeline_name", - "timeline_label", "sent_or_received", "uid", "message_id", "seen", "nomatch", "has_attachment", "timeline_hide"], + "timeline_label", "sent_or_received", "uid", "message_id", "seen", "nomatch", "has_attachment"], filters: this.filter_list.get_filters(), order_by: 'communication_date desc', save_list_settings: false @@ -451,7 +451,7 @@ frappe.Inbox = frappe.ui.Listing.extend({ row.reference_doctype = values["reference_doctype"]; row.reference_name = values["reference_name"]; } - frappe.timeline.relink_dialog(row.name, row.reference_doctype, row.reference_name, callback, row.timeline_hide); + frappe.timeline.relink_dialog(row.name, row.reference_doctype, row.reference_name, callback); }, run:function(more,footer) { var me = this; diff --git a/frappe/email/receive.py b/frappe/email/receive.py index f6898de432..7bceaab1be 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -2,7 +2,7 @@ # MIT License. See license.txt from __future__ import unicode_literals -import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re, hashlib +import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re from email_reply_parser import EmailReplyParser from email.header import decode_header import frappe @@ -11,9 +11,6 @@ from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now, cint, cstr, strip, markdown) from frappe.utils.scheduler import log from frappe.utils.file_manager import get_random_filename, save_file, MaxFileSizeReachedError -from email_reply_parser import EmailReplyParser -from email.header import decode_header -from frappe.utils.file_manager import get_random_filename class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass @@ -49,7 +46,6 @@ class EmailServer: try: if cint(self.settings.use_ssl): self.imap = Timed_IMAP4_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout")) - #self.imap = imaplib.IMAP4_SSL(self.settings.host) else: self.imap = Timed_IMAP4(self.settings.host, timeout=frappe.conf.get("pop_timeout")) self.imap.login(self.settings.username, self.settings.password) @@ -102,56 +98,42 @@ class EmailServer: frappe.db.commit() + if not self.connect(): + return [] + try: # track if errors arised self.errors = False self.latest_messages = [] - if cint(self.settings.use_imap): - uid_validity = self.get_status() - else: - email_list = self.get_new_mails() + email_list = self.get_new_mails() + num = num_copy = len(email_list) + + # WARNING: Hard coded max no. of messages to be popped + if num > 20: num = 20 # size limits self.total_size = 0 self.max_email_size = cint(frappe.local.conf.get("max_email_size")) self.max_total_size = 5 * self.max_email_size - if cint(self.settings.use_imap): - #try: - if self.check_uid_validity(uid_validity): - email_list = self.get_new_mails() - if email_list: - self.get_imap_messages(email_list) - self.sync_flags() - self.get_seen() - self.push_deleted() - else: - pass + for i, message_meta in enumerate(email_list): + # do not pull more than NUM emails + if (i+1) > num: + break - else: - num = num_copy = len(email_list) + try: + self.retrieve_message(message_meta, i+1) + except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): + break - # WARNING: Hard coded max no. of messages to be popped - if num > 20: num = 20 #20 - - for i, message_meta in enumerate(email_list): - # do not pull more than NUM emails - if (i+1) > num: - break - - try: - self.retrieve_message(message_meta, i+1) - except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): - break - - # WARNING: Mark as read - message number 101 onwards from the pop list - # This is to avoid having too many messages entering the system - num = num_copy - if not cint(self.settings.use_imap): - if num > 100 and not self.errors: - for m in xrange(101, num+1): - self.pop.dele(m) + # WARNING: Mark as read - message number 101 onwards from the pop list + # This is to avoid having too many messages entering the system + num = num_copy + if not cint(self.settings.use_imap): + if num > 100 and not self.errors: + for m in xrange(101, num+1): + self.pop.dele(m) except Exception, e: if self.has_login_limit_exceeded(e): @@ -169,316 +151,17 @@ class EmailServer: return self.latest_messages - def get_status(self): - passed, status = self.imap.status("Inbox", "(UIDNEXT UIDVALIDITY)") - match = re.search(r"(?<=UIDVALIDITY )[0-9]*", status[0], re.U | re.I) - if match: - uid_validity = match.group(0) - match = re.search(r"(?<=UIDNEXT )[0-9]*", status[0], re.U | re.I) - if match: - uidnext = match.group(0) - frappe.db.sql("update `tabEmail Account` set uidnext = %s where name = %s",(uidnext, self.settings.email_account), auto_commit=1) - self.settings.newuidnext = uidnext - return uid_validity - def get_new_mails(self): """Return list of new mails""" if cint(self.settings.use_imap): self.imap.select("Inbox") - if self.settings.no_remaining == '0' and self.settings.uidnext: - if self.settings.uidnext == self.settings.newuidnext: - return False - else: - #request all messages between last uidnext and new - return True - else: - response, message = self.imap.uid('search', None, "ALL") + response, message = self.imap.uid('search', None, "UNSEEN") email_list = message[0].split() else: email_list = self.pop.list()[1] return email_list - def check_uid_validity(self, uid_validity): - if self.settings.uid_validity: - if self.settings.uid_validity == uid_validity: - return True - else: - #validity changed - self.settings.no_remaining = None - self.rebuild_uid(uid_validity) - return True - - else:#if email account settings is blank - uid_list = frappe.db.sql("""select uid - from tabCommunication - where email_account = %(email_account)s and uid is not Null - order by uid - """,{"email_account":self.settings.email_account}, as_list=1) - new_uid_list = [] - for i in uid_list: - new_uid_list.append(i[0]) - - if new_uid_list:#if email account - self.rebuild_uid(uid_validity) - return True - else:# if no uid and no emails with uid - frappe.db.set_value("Email Account", self.settings.email_account, "uid_validity", uid_validity) - frappe.db.commit() - return True - - def rebuild_uid(self,uid_validity): - uid_list = frappe.db.sql("""select name,uid ,unique_id - from `tabCommunication` - where email_account = %(email_account)s and unique_id is not Null and sent_or_received = 'Received' - #order by uid - """, {"email_account": self.settings.email_account}, as_dict=1) - - unhandled_uid_list = frappe.db.sql("""select name,uid ,unique_id - from `tabUnhandled Email` - where email_account = %(email_account)s and unique_id is not Null - #order by uid - """, {"email_account": self.settings.email_account}, as_dict=1) - - - message_list = [] - #get message-id's to link new uid's to - import email - self.imap.select("Inbox") - #messages = self.imap.uid('fetch', "1:*", '(BODY.PEEK[HEADER.FIELDS (FROM TO ENVELOPE-TO DATE RECEIVED)])') - messages = self.imap.uid('fetch', "1:*", '(BODY.PEEK[HEADER])') - for i, item in enumerate(messages[1]): - if isinstance(item, tuple): - # check for uid appended to the end - uid = re.search(r'UID [0-9]*', messages[1][i + 1], re.U | re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - - # check for uid at start - if not uid: - # for m in item: - uid = re.search(r'UID [0-9]*', messages[1][i][0], re.U | re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - continue - mail = email.message_from_string(item[1]) - unique_id = get_unique_id(mail) - message_list.append([uid, unique_id]) - # clear out - frappe.db.sql("""update `tabCommunication` - set uid = NULL - where email_account = %(email_account)s - """, {"email_account": self.settings.email_account}) - frappe.db.sql("""update `tabUnhandled Email` - set uid = NULL - where email_account = %(email_account)s - """, {"email_account": self.settings.email_account}) - - # write new uid - new_uid = [] - for old in uid_list: - for new in message_list: - if old["unique_id"] == new[1]: - frappe.db.sql("""update `tabCommunication` - set uid = %(uid)s - where name = %(name)s - """, {"name": old["name"], - "uid": new[0]}) - break - for old in unhandled_uid_list: - for new in message_list: - if old["unique_id"] == new[1]: - frappe.db.sql("""update `tabUnhandled Email` - set uid = %(uid)s - where name = %(name)s - """, {"name": old["name"], - "uid": new[0]}) - break - - frappe.db.set_value("Email Account", self.settings.email_account, "uid_validity", uid_validity) - frappe.db.set_value("Email Account", self.settings.email_account, "no_remaining", None) - frappe.db.commit() - - - - def get_imap_messages(self,email_list): - if self.settings.no_remaining == '0' and self.settings.uidnext: - download_list = range(int(self.settings.uidnext), int(self.settings.newuidnext)) - else: - #compare stored uid to new uid list to dl any missing messages - uid_list = frappe.db.sql("""select uid - from tabCommunication - where email_account = %(email_account)s and uid is not Null - order by uid - """,{"email_account":self.settings.email_account},as_list=1) - uid_list = uid_list + (frappe.db.sql("""select uid - from `tabUnhandled Email` - where email_account = %(email_account)s and uid is not Null - order by uid - """,{"email_account":self.settings.email_account},as_list=1)) - new_uid_list = [] - for i in uid_list: - new_uid_list.append(i[0]) - - download_list = [] - for new in email_list: - if new not in new_uid_list: - download_list.append(cint(new)) - - from itertools import count, groupby - num = 50 - - # set number of email remaining to be synced - dl_length = len(download_list) - - lcount =1 - while len(download_list)>0: - # trim list to specified num emails to dl at a time - - dlength = len(download_list) - cur_download_list = download_list[dlength - num:dlength] - if cur_download_list: - download_list = download_list[:dlength - num] - - if lcount>=4: - download_list = [] - - # compress download list into ranges - G=(list(x) for _,x in groupby(cur_download_list, lambda x,c=count(): next(c)-x)) - message_meta = ",".join(":".join(map(str,(g[0],g[-1])[:len(g)])) for g in G) - - messages =[] - - try: - messages = self.imap.uid('fetch', message_meta, '(BODY.PEEK[])') - - except (TotalSizeExceededError, EmailTimeoutError), e: - print("timeout or size exceed") - pass - except (imaplib.IMAP4.error),e: - - print (e) - pass - - if messages and messages[0]=='OK': - for i, item in enumerate(messages[1]): - if isinstance(item, tuple): - #check for uid appended to the end - uid = re.search(r'UID [0-9]*', messages[1][i + 1], re.U|re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - - - #check for uid at start - if not uid: - #for m in item: - uid = re.search(r'UID [0-9]*', messages[1][i][0], re.U|re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - continue - - - if uid: - self.latest_messages.append([item[1],uid,1])#message,uid,seen - - frappe.db.sql("update `tabEmail Account` set no_remaining = %s where name = %s", - (dl_length-len(self.latest_messages), self.settings.email_account), auto_commit=1) - lcount = lcount +1 - - def sync_flags(self): - #get flags from email flag queue + join them to the matching email account and uid - queue = frappe.db.sql("""select que.name,comm.uid,que.action,que.flag - from tabCommunication as comm,`tabEmail Flag Queue` as que - where comm.name = que.comm_name and comm.uid is not null and comm.email_account=%(email_account)s""", - {"email_account":self.settings.email_account}, as_dict=1) - #loop though flags - - for item in queue: - try: - self.imap.uid('STORE', item.uid, item.action, item.flag) - #delete flag matching email account - frappe.delete_doc("Email Flag Queue", item["name"]) - except Exception,e: - #need to do - pass - - def get_seen(self): - comm_list = frappe.db.sql("""select name,uid,seen from `tabCommunication` - where email_account = %(email_account)s and uid is not null""", - {"email_account":self.settings.email_account}, as_dict=1) - - try: - #response, messages = self.imap.uid('fetch', '1:*', '(FLAGS)') - response, seen_list = self.imap.uid('search', None, "SEEN") - response, unseen_list = self.imap.uid('search', None, "UNSEEN") - except Exception,e: - print("failed get seen sync download") - return - unseen_list = unseen_list[0].split() - for unseen in unseen_list: - for msg in self.latest_messages: - if unseen == msg[1]: - msg[2] = 0 - - for comm in comm_list: - if comm.uid == unseen: - if comm.seen: - frappe.db.sql("update `tabCommunication` set seen=%s where name = %s",(0, comm.name)) - comm_list.remove(comm) - break - seen_list = seen_list[0].split() - for seen in seen_list: - for msg in self.latest_messages: - if seen == msg[1]: - msg[2] = 1 - - for comm in comm_list: - if comm.uid == seen: - if not comm.seen: - frappe.db.sql("update `tabCommunication` set seen=%s where name = %s", (1, comm.name)) - comm_list.remove(comm) - break - ''' - for item in messages: - uid = re.search(r'UID [0-9]*', item, re.U | re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - - # flag = re.search(r"(?<=FLAGS \()(.*?)(?=\))", item, re.U | re.I) - flag = re.search(r"\\Seen", item, re.U | re.I) - - for msg in self.latest_messages: - if uid == msg[1]: - if flag: - msg[2]=0 - - for comm in comm_list: - if comm.uid==uid: - if flag: - if not comm.email_seen: - frappe.db.set_value('Communication',comm.name,'email_seen','1',update_modified=False) - else: - if comm.email_seen: - frappe.db.set_value('Communication', comm.name, 'email_seen', '0', update_modified=False) - comm_list.remove(comm) - break - ''' - frappe.db.commit() - - - def push_deleted(self): - pass - def retrieve_message(self, message_meta, msg_num=None): incoming_mail = None try: @@ -511,15 +194,13 @@ class EmailServer: self.pop.dele(msg_num) else: # mark as seen - #self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') - pass + self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') else: if not cint(self.settings.use_imap): self.pop.dele(msg_num) else: # mark as seen - #self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') - pass + self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') def has_login_limit_exceeded(self, e): return "-ERR Exceeded the login limit" in strip(cstr(e.message)) @@ -564,23 +245,6 @@ class EmailServer: return error_msg - -def get_unique_id(mail): - hash = hashlib.sha1() - # loop though headers to make unique id looping used to resolve encoding issue of adding together - for h in mail._headers: - if h[0] != 'Content-Type': # skip variable boundaries - try: - temp = decode_header(h[1]) - decoded = ''.join( - [d[0].decode(d[1]).encode('ascii', 'ignore') if d[1] is not None else d[0] for d in temp]) - cleaned = re.sub(r"\s+", u"", decoded, - flags=re.UNICODE) # gmail fix as returns different whitespace if download only headers - hash.update(cleaned) - except: - pass - return hash.hexdigest() - class Email: """Wrapper for an email.""" def __init__(self, content): @@ -600,52 +264,10 @@ class Email: self.set_from() self.message_id = (self.mail.get('Message-ID') or "").strip(" <>") - - self.unique_id = get_unique_id(self.mail) - - # gmail mailing-list compatibility - # use X-Original-Sender if available, as gmail sometimes modifies the 'From' - # _from_email = self.mail.get("X-Original-From") or self.mail["From"] - # - # self.from_email = extract_email_id(_from_email) - # if self.from_email: - # self.from_email = self.from_email.lower() - # - # #self.from_real_name = email.utils.parseaddr(_from_email)[0] - # - # _from_real_name = decode_header(email.utils.parseaddr(_from_email)[0]) - # self.from_real_name = decode_header(email.utils.parseaddr(_from_email)[0])[0][0] or "" - # - # try: - # if _from_real_name[0][1]: - # self.from_real_name = self.from_real_name.decode(_from_real_name[0][1]) - # else: - # # assume that the encoding is utf-8 - # self.from_real_name = self.from_real_name.decode("utf-8") - # except UnicodeDecodeError,e: - # print e - # pass - - #self.from_real_name = email.Header.decode_header(email.utils.parseaddr(_from_email)[0])[0][0] - self.To = self.mail.get("To") - if self.To: - to = u"" - for name, encoding in decode_header(self.To): - if encoding: - to += name.decode(encoding) - else: - to += name - self.To = to.lower() - self.CC = self.mail.get("CC") - if self.CC: - self.CC = self.CC.lower() if self.mail["Date"]: - try: - utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) - utc_dt = datetime.datetime.utcfromtimestamp(utc) - self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S') - except: - self.date = now() + utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) + utc_dt = datetime.datetime.utcfromtimestamp(utc) + self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S') else: self.date = now() @@ -656,32 +278,13 @@ class Email: def set_subject(self): """Parse and decode `Subject` header.""" - from email.errors import HeaderParseError - try: - _subject = decode_header(self.mail.get("Subject", "No Subject")) - self.subject = _subject[0][0] or "" - - if _subject[0][1]: - self.subject = self.subject.decode(_subject[0][1]) - else: - # assume that the encoding is utf-8 - self.subject = self.subject.decode("utf-8")[:140] - except (UnicodeDecodeError, HeaderParseError): - #try: - # self.subject = self.subject.decode("gb18030") - #except UnicodeDecodeError: - self.subject = u'Error Decoding Subject' - #if self.subject and len(self.subject)>140: - # self.subject = self.subject[:135] - import re - - emoji_pattern = re.compile("[" - u"\U0001F600-\U0001F64F" # emoticons - u"\U0001F300-\U0001F5FF" # symbols & pictographs - u"\U0001F680-\U0001F6FF" # transport & map symbols - u"\U0001F1E0-\U0001F1FF" # flags (iOS) - "]+", flags=re.UNICODE) - self.subject = emoji_pattern.sub(r'', self.subject) + _subject = decode_header(self.mail.get("Subject", "No Subject")) + self.subject = _subject[0][0] or "" + if _subject[0][1]: + self.subject = self.subject.decode(_subject[0][1]) + else: + # assume that the encoding is utf-8 + self.subject = self.subject.decode("utf-8")[:140] if not self.subject: self.subject = "No Subject" @@ -691,27 +294,14 @@ class Email: # use X-Original-Sender if available, as gmail sometimes modifies the 'From' _from_email = self.mail.get("X-Original-From") or self.mail["From"] _from_email, encoding = decode_header(_from_email)[0] - _reply_to, _reply_to_encoding = decode_header(self.mail.get("Reply-To"))[0] if encoding: _from_email = _from_email.decode(encoding) else: _from_email = _from_email.decode('utf-8') - - if _reply_to_encoding: - _reply_to = _from_email.decode(encoding) - else: - _reply_to = _from_email.decode('utf-8') - if _reply_to and not frappe.db.get_value('Email Account', {"email_id":_reply_to}, 'email_id'): - self.from_email = extract_email_id(_reply_to) - else: - self.from_email = extract_email_id(_from_email) - - if self.from_email: - self.from_email = self.from_email.lower() - - self.from_real_name = email.utils.parseaddr(_from_email)[0] if "@" in _from_email else _from_email + self.from_email = extract_email_id(_from_email) + self.from_real_name = email.utils.parseaddr(_from_email)[0] def set_content_and_type(self): self.content, self.content_type = '[Blank Email]', 'text/plain' diff --git a/frappe/patches/v6_24/imap_get_unique.py b/frappe/patches/v6_24/imap_get_unique.py deleted file mode 100644 index 4e07947fd4..0000000000 --- a/frappe/patches/v6_24/imap_get_unique.py +++ /dev/null @@ -1,82 +0,0 @@ -import frappe -import imaplib -import hashlib -import re -import email, email.utils,email.header -from email.header import decode_header -from frappe.email.receive import get_unique_id - -def execute(): - frappe.reload_doctype("Communication") - frappe.reload_doctype("Unhandled Email") - for email_account in frappe.get_list("Email Account", filters={"awaiting_password": 0}): - email_acc = frappe.get_doc("Email Account", email_account) - try: - email_server = email_acc.get_server(in_receive=True) - email_server.imap.select("Inbox") - #messages =email_server.imap.uid('fetch', "1:*", '(BODY.PEEK[HEADER.FIELDS (FROM TO ENVELOPE-TO DATE RECEIVED)])') - messages = email_server.imap.uid('fetch', "1:*",'(BODY.PEEK[HEADER])') - comms = frappe.db.sql("""select uid,name from `tabCommunication` - where email_account=%(email_account)s""",{"email_account":email_account.name},as_dict=1 ) - unhandled = frappe.db.sql("""select uid,name from `tabUnhandled Email` - where email_account=%(email_account)s""",{"email_account":email_account.name},as_dict=1 ) - count =0; - for i, item in enumerate(messages[1]): - if isinstance(item, tuple): - - # check for uid appended to the end - uid = re.search(r'UID [0-9]*', messages[1][i + 1], re.U | re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - - # check for uid at start - if not uid: - # for m in item: - uid = re.search(r'UID [0-9]*', messages[1][i][0], re.U | re.I) - if uid: - uid = uid.group()[4:] - else: - uid = "" - continue - mail = email.message_from_string(item[1]) - unique_id = get_unique_id(mail) - - #unique_id = hashlib.md5((mail.get("X-Original-From") or mail["From"])+(mail.get("To") or mail.get("Envelope-to"))+(mail.get("Received") or mail["Date"])).hexdigest() - found =False - for comm in comms: - if comm.uid == uid: - found =True - frappe.db.sql("""update `tabCommunication` - set unique_id = %(unique_id)s - where name = %(name)s - """, {"unique_id": unique_id, - "name":comm.name}) - - if not found: - for comm in unhandled: - if comm.uid == uid: - found = True - frappe.db.sql("""update `tabUnhandled Email` - set unique_id = %(unique_id)s - where name = %(name)s - """, {"unique_id": unique_id, - "name": comm.name}) - if found: - count += 1 - - #frappe.db.sql("""update `tabCommunication` - # set unique_id = %(unique_id)s - # where email_account= %(email_account)s and uid = %(uid)s - # """, {"unique_id": h, - # "email_account":email_account.name, - # "uid": uid}) - print email_account.name,count - except Exception, e: - print e - finally: - try: - email_server.imap.logout() - except: - pass diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js index 84af60d8b4..f74b96f988 100644 --- a/frappe/public/js/frappe/form/footer/timeline.js +++ b/frappe/public/js/frappe/form/footer/timeline.js @@ -96,7 +96,7 @@ frappe.ui.form.Timeline = Class.extend({ var communications = this.get_communications(true); - $.each(communications.sort(function(a, b) { return a.communication_date > b.communication_date ? -1 : 1 }), + $.each(communications.sort(function(a, b) { return a.creation > b.communication_date ? -1 : 1 }), function(i, c) { if(c.content) { c.frm = me.frm; @@ -240,7 +240,7 @@ frappe.ui.form.Timeline = Class.extend({ } } - c.comment_on = comment_when(c.communication_date || c.creation); + c.comment_on = comment_when(c.creation); c.fullname = c.sender_full_name || frappe.user.full_name(c.sender); if(c.attachments && typeof c.attachments==="string") @@ -560,7 +560,7 @@ frappe.ui.form.Timeline = Class.extend({ communications = this.frm.get_docinfo().communications, email = this.get_recipient(); - $.each(communications.sort(function(a, b) { return a.communication_date > b.communication_date ? -1 : 1 }), function(i, c) { + $.each(communications.sort(function(a, b) { return a.creation > b.creation ? -1 : 1 }), function(i, c) { if(c.communication_type=='Communication' && c.communication_medium=="Email") { if(from_recipient) { if(c.sender.indexOf(email)!==-1) {