move message_id threading to communication to work with inbox (#2458)

* move message_id threading to communication to work with inbox

* move message_id threading fix

* rebase fix

* reset threading to email queue but keep in communications

frappe/__init__.py

@@ -359,7 +359,7 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 		as_markdown=False, delayed=True, reference_doctype=None, reference_name=None,
 		unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
 		attachments=None, content=None, doctype=None, name=None, reply_to=None,
-		cc=[], in_reply_to=None, send_after=None, expose_recipients=None,
+		cc=[], message_id=None, in_reply_to=None, send_after=None, expose_recipients=None,
 		send_priority=1, communication=None, retry=1, now=None):
 	"""Send email using user's default **Email Account** or global default **Email Account**.
 
@@ -377,6 +377,7 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 	:param unsubscribe_params: Unsubscribe paramaters to be loaded on the unsubscribe_method [optional] (dict).
 	:param attachments: List of attachments.
 	:param reply_to: Reply-To email id.
+	:param message_id: Used for threading. If a reply is received to this email, Message-Id is sent back as In-Reply-To in received email.
 	:param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To.
 	:param send_after: Send after the given datetime.
 	:param expose_recipients: Display all recipients in the footer message - "This email was sent to"
@@ -396,7 +397,7 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 		subject=subject, message=message,
 		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
-		attachments=attachments, reply_to=reply_to, cc=cc, in_reply_to=in_reply_to,
+		attachments=attachments, reply_to=reply_to, cc=cc, message_id=message_id, in_reply_to=in_reply_to,
 		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority,
 		communication=communication, now=now)
 

frappe/core/doctype/communication/communication.json

@@ -1040,6 +1040,34 @@
    "set_only_once": 0, 
    "unique": 0
   }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "message_id", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 1, 
+   "in_filter": 0, 
+   "in_standard_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Message ID", 
+   "length": 995, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
   {
    "allow_on_submit": 0, 
    "bold": 0, 
@@ -1079,7 +1107,7 @@
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2016-11-10 16:02:10.145975", 
+ "modified": "2016-12-14 16:02:10.145975", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "Communication", 

frappe/core/doctype/communication/communication.py

@@ -229,6 +229,7 @@ def on_doctype_update():
 	frappe.db.add_index("Communication", ["status", "communication_type"])
 	frappe.db.add_index("Communication", ["creation"])
 	frappe.db.add_index("Communication", ["modified"])
+	frappe.db.add_index("Communication", ["message_id(200)"])
 
 def has_permission(doc, ptype, user):
 	if ptype=="read":

frappe/core/doctype/communication/email.py

@@ -10,6 +10,7 @@ from frappe.utils import (get_url, get_formatted_email, cint,
 from frappe.utils.file_manager import get_file
 from frappe.email.queue import check_email_limit
 from frappe.utils.scheduler import log
+from frappe.email.email_body import get_message_id
 import frappe.email.smtp
 import MySQLdb
 import time
@@ -57,7 +58,8 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 		"communication_medium": communication_medium,
 		"sent_or_received": sent_or_received,
 		"reference_doctype": doctype,
-		"reference_name": name
+		"reference_name": name,
+		"message_id":get_message_id().strip(" <>")
 	})
 	comm.insert(ignore_permissions=True)
 
@@ -146,6 +148,7 @@ def _notify(doc, print_html=None, print_format=None, attachments=None,
 		reference_doctype=doc.reference_doctype,
 		reference_name=doc.reference_name,
 		attachments=doc.attachments,
+		message_id=doc.message_id,
 		unsubscribe_message=unsubscribe_message,
 		delayed=True,
 		communication=doc.name
@@ -411,6 +414,8 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments
 		for i in xrange(3):
 			try:
 				communication = frappe.get_doc("Communication", communication_name)
+				if communication.sent_or_received == "Received":
+					communication.message_id = None
 				communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
 					recipients=recipients, cc=cc)
 

frappe/email/doctype/email_account/email_account.py

@@ -288,6 +288,12 @@ class EmailAccount(Document):
 			communication.reference_doctype = parent.doctype
 			communication.reference_name = parent.name
 
+		# check if message is notification and disable notifications for this message
+		references = email.mail.get("References")
+		if references:
+			if "notification" in references:
+				communication.unread_notification_sent = 1
+
 	def set_sender_field_and_subject_field(self):
 		'''Identify the sender and subject fields from the `append_to` DocType'''
 		# set subject_field and sender_field
@@ -311,7 +317,7 @@ class EmailAccount(Document):
 				# try and match by subject and sender
 				# if sent by same sender with same subject,
 				# append it to old coversation
-				subject = strip(re.sub("^\s*(Re|RE)[^:]*:\s*", "", email.subject))
+				subject = strip(re.sub("(^\s*(Fw|FW|fwd)[^:]*:|\s*(Re|RE)[^:]*:\s*)*", "", email.subject))
 
 				parent = frappe.db.get_all(self.append_to, filters={
 					self.sender_field: email.from_email,

frappe/email/email_body.py

@@ -60,7 +60,6 @@ class EMail:
 		self.html_set = False
 
 		self.email_account = email_account or get_outgoing_email_account()
-		self.set_message_id()
 
 	def set_html(self, message, text_content = None, footer=None, print_html=None, formatted=None):
 		"""Attach message in the html portion of multipart/alternative"""
@@ -184,8 +183,12 @@ class EMail:
 			sender_name, sender_email = email.utils.parseaddr(self.sender)
 			self.sender = email.utils.formataddr((sender_name or self.email_account.name, self.email_account.email_id))
 
-	def set_message_id(self):
-		self.msg_root["Message-Id"] = get_message_id()
+	def set_message_id(self, message_id):
+		if message_id:
+			self.msg_root["Message-Id"] = message_id
+		else:
+			self.msg_root["Message-Id"] = get_message_id()
+			self.msg_root["References"] = '<notification>'
 
 	def set_in_reply_to(self, in_reply_to):
 		"""Used to send the Message-Id of a received email back as In-Reply-To"""

frappe/email/queue.py

@@ -18,7 +18,7 @@ class EmailLimitCrossedError(frappe.ValidationError): pass
 
 def send(recipients=None, sender=None, subject=None, message=None, reference_doctype=None,
 		reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
-		attachments=None, reply_to=None, cc=[], in_reply_to=None, send_after=None,
+		attachments=None, reply_to=None, cc=[], message_id=None, in_reply_to=None, send_after=None,
 		expose_recipients=None, send_priority=1, communication=None, now=False):
 	"""Add email to sending queue (Email Queue)
 
@@ -84,7 +84,7 @@ def send(recipients=None, sender=None, subject=None, message=None, reference_doc
 
 	# add to queue
 	email_queue = add(recipients, sender, subject, email_content, email_text_context, reference_doctype,
-		reference_name, attachments, reply_to, cc, in_reply_to, send_after, send_priority, email_account=email_account, communication=communication,
+		reference_name, attachments, reply_to, cc, message_id, in_reply_to, send_after, send_priority, email_account=email_account, communication=communication,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, expose_recipients=expose_recipients)
 	if now:
 		send_one(email_queue.name, now=True)
@@ -92,7 +92,7 @@ def send(recipients=None, sender=None, subject=None, message=None, reference_doc
 
 def add(recipients, sender, subject, formatted, text_content=None,
 	reference_doctype=None, reference_name=None, attachments=None, reply_to=None,
-	cc=[], in_reply_to=None, send_after=None, send_priority=1, email_account=None,
+	cc=[], message_id=None, in_reply_to=None, send_after=None, send_priority=1, email_account=None,
 	communication=None, unsubscribe_method=None, unsubscribe_params=None, expose_recipients=None):
 	"""Add to Email Queue"""
 	e = frappe.new_doc('Email Queue')
@@ -103,6 +103,7 @@ def add(recipients, sender, subject, formatted, text_content=None,
 			text_content=text_content, attachments=attachments, reply_to=reply_to,
 			cc=cc, email_account=email_account, expose_recipients=expose_recipients)
 
+		mail.set_message_id(message_id)
 		if in_reply_to:
 			mail.set_in_reply_to(in_reply_to)
 

frappe/email/receive.py

@@ -262,6 +262,7 @@ class Email:
 		self.set_content_and_type()
 		self.set_subject()
 		self.set_from()
+		self.message_id = (self.mail.get('Message-ID') or "").strip(" <>")
 
 		if self.mail["Date"]:
 			utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))