From b018236d183dfb524afc6d6849562ef906210d4b Mon Sep 17 00:00:00 2001 From: shariquerik Date: Mon, 10 May 2021 11:16:47 +0530 Subject: [PATCH 01/29] fix: Grid row deletion fix --- frappe/public/js/frappe/form/grid.js | 7 ++++++- frappe/public/js/frappe/form/grid_row.js | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 4d381c9be7..9b05809aba 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -196,7 +196,9 @@ export default class Grid { tasks.push(() => { if (dirty) { this.refresh(); - this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); + if (this.frm) { + this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); + } } }); @@ -345,6 +347,9 @@ export default class Grid { if (d.idx === undefined) { d.idx = ri + 1; } + if (d.name === undefined) { + d.name = "row" + d.idx + } if (this.grid_rows[ri] && !append_row) { var grid_row = this.grid_rows[ri]; grid_row.doc = d; diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index f6da88df57..bbcf0f707b 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -93,8 +93,11 @@ export default class GridRow { } else { data = this.grid.df.data; } - - const index = data.findIndex(d => d.name === me.doc.name); + let index = -1; + + if (me.doc.name) { + index = data.findIndex(d => d.name === me.doc.name); + } if (index > -1) { // mutate array directly, From 6a9b93bfa5d6b5569681c358cc0a2c66351603fa Mon Sep 17 00:00:00 2001 From: shariquerik Date: Mon, 10 May 2021 11:37:48 +0530 Subject: [PATCH 02/29] fix: sider fix --- frappe/public/js/frappe/form/grid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 9b05809aba..00f9d6abca 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -348,7 +348,7 @@ export default class Grid { d.idx = ri + 1; } if (d.name === undefined) { - d.name = "row" + d.idx + d.name = "row" + d.idx; } if (this.grid_rows[ri] && !append_row) { var grid_row = this.grid_rows[ri]; From c551338c0d7a665730ba845ccda5b4f09107301e Mon Sep 17 00:00:00 2001 From: prssanna Date: Tue, 18 May 2021 15:41:31 +0530 Subject: [PATCH 03/29] fix: ensure website theme is applied correctly --- frappe/public/scss/website/css_variables.scss | 2 ++ frappe/public/scss/website/sidebar.scss | 6 +++--- .../doctype/website_theme/website_theme_template.scss | 11 +++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/frappe/public/scss/website/css_variables.scss b/frappe/public/scss/website/css_variables.scss index 2aeb842802..463a0db49e 100644 --- a/frappe/public/scss/website/css_variables.scss +++ b/frappe/public/scss/website/css_variables.scss @@ -28,4 +28,6 @@ --font-size-4xl: #{$font-size-4xl}; --font-size-5xl: #{$font-size-5xl}; --font-size-6xl: #{$font-size-6xl}; + + --card-border-radius: #{$card-border-radius}; } diff --git a/frappe/public/scss/website/sidebar.scss b/frappe/public/scss/website/sidebar.scss index b13eaf2a74..76956c9136 100644 --- a/frappe/public/scss/website/sidebar.scss +++ b/frappe/public/scss/website/sidebar.scss @@ -10,7 +10,7 @@ margin-top: 0.25rem; border-radius: 0.375rem; font-size: $font-size-sm; - color: $gray-600; + color: var(--text-color); text-decoration: none; font-weight: 500; @include transition(); @@ -26,8 +26,8 @@ } .sidebar-item a.active { - color: $primary; - background-color: $primary-light; + color: var(--primary); + background-color: var(--primary-light); } .sidebar-item-icon { diff --git a/frappe/website/doctype/website_theme/website_theme_template.scss b/frappe/website/doctype/website_theme/website_theme_template.scss index fbd640690b..2a67baf541 100644 --- a/frappe/website/doctype/website_theme/website_theme_template.scss +++ b/frappe/website/doctype/website_theme/website_theme_template.scss @@ -30,3 +30,14 @@ body { // Custom Theme {{ custom_scss or '' }} + +:root { + --primary: #{$primary}; + --primary-color: #{$primary}; + --primary-light: #{$primary-light}; + --light: #{$light}; + --bg-color: #{$body-bg}; + --text-color: #{$body-color}; + --text-light: #{$body-color}; +} + From a74352cfad16c3ba3650087c7403aaee7431bae1 Mon Sep 17 00:00:00 2001 From: prssanna Date: Tue, 18 May 2021 16:19:25 +0530 Subject: [PATCH 04/29] fix: button colors --- frappe/public/scss/common/buttons.scss | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/public/scss/common/buttons.scss b/frappe/public/scss/common/buttons.scss index 591dc5bba6..de3a4cfc20 100644 --- a/frappe/public/scss/common/buttons.scss +++ b/frappe/public/scss/common/buttons.scss @@ -48,13 +48,13 @@ $active-border: darken($primary-light, 12.5%) ); - color: var(--blue-500); + color: var(--primary); &:hover, &:active { - color: var(--blue-500); + color: var(--primary); } &:focus { - box-shadow: 0 0 0 0.2rem var(--blue-50) + box-shadow: 0 0 0 0.2rem var(--primary-light); } } @@ -77,11 +77,11 @@ } .btn.btn-primary { - background-color: var(--primary-color); + background-color: var(--primary); color: var(--white); white-space: nowrap; --icon-stroke: currentColor; - --icon-fill-bg: var(--primary-color); + --icon-fill-bg: var(--primary); } .btn.btn-danger { From 2b8aaa53759f4f6168f6572dbda79092b2517437 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 18 May 2021 21:12:45 +0530 Subject: [PATCH 05/29] fix(minor): expose limited methods of json module --- frappe/utils/safe_exec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 6c1fa21685..2472de70f0 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -61,7 +61,9 @@ def get_safe_globals(): out = NamespaceDict( # make available limited methods of frappe - json=json, + json=NamespaceDict( + loads = json.loads, + dumps = json.dumps), dict=dict, log=frappe.log, _dict=frappe._dict, From 5b96b79ed43a391ad82f43bda408aeb99596a8ae Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 12 May 2021 13:49:43 +0530 Subject: [PATCH 06/29] refactor: incoming mails --- .../doctype/communication/communication.py | 40 +- .../doctype/email_account/email_account.py | 395 +++--------------- .../email/doctype/email_queue/email_queue.py | 5 + frappe/email/email_body.py | 4 +- frappe/email/receive.py | 368 +++++++++++++++- frappe/hooks.py | 6 +- 6 files changed, 461 insertions(+), 357 deletions(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 5ebf714645..a9c1c1f5a3 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -21,9 +21,11 @@ from frappe.automation.doctype.assignment_rule.assignment_rule import apply as a exclude_from_linked_with = True class Communication(Document): + """Communication represents an external communication like Email. + """ no_feed_on_delete = True + DOCTYPE = 'Communication' - """Communication represents an external communication like Email.""" def onload(self): """create email flag queue""" if self.communication_type == "Communication" and self.communication_medium == "Email" \ @@ -149,6 +151,23 @@ class Communication(Document): self.email_status = "Spam" + @classmethod + def find(cls, name, ignore_error=False): + try: + return frappe.get_doc(cls.DOCTYPE, name) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @classmethod + def find_one_by_filters(cls, *, order_by=None, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs, order_by=order_by) + return cls.find(name) if name else None + + def update_db(self, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + def set_sender_full_name(self): if not self.sender_full_name and self.sender: if self.sender == "Administrator": @@ -180,6 +199,8 @@ class Communication(Document): if not self.sender_full_name: self.sender_full_name = sender_email + + def send(self, print_html=None, print_format=None, attachments=None, send_me_a_copy=False, recipients=None): """Send communication via Email. @@ -306,6 +327,21 @@ class Communication(Document): if autosave: self.save(ignore_permissions=ignore_permissions) + # @classmethod + # def add_incoming_mail(cls, email, uid, seen_status): + # if exists_already(): + # pass # Modify UID + # else: + # pass # Call new() to create object +class InMailMixin: + @classmethod + def add_incoming_mail(cls, email, uid, seen_status): + if exists_already(): + pass # Modify UID + else: + pass # Call new() to create object + + def on_doctype_update(): """Add indexes in `tabCommunication`""" frappe.db.add_index("Communication", ["reference_doctype", "reference_name"]) @@ -485,4 +521,4 @@ def set_avg_response_time(parent, communication): response_times.append(response_time) if response_times: avg_response_time = sum(response_times) / len(response_times) - parent.db_set("avg_response_time", avg_response_time) \ No newline at end of file + parent.db_set("avg_response_time", avg_response_time) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 36b662bb39..0a66efb480 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -19,7 +19,7 @@ from frappe.utils import (validate_email_address, cint, cstr, get_datetime, from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, Email +from frappe.email.receive import EmailServer, Email, InboundMail, SentEmailInInboxError from poplib import error_proto from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta @@ -430,89 +430,79 @@ class EmailAccount(Document): def receive(self, test_mails=None): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" - def get_seen(status): - if not status: - return None - seen = 1 if status == "SEEN" else 0 - return seen + exceptions = [] + inbound_mails = self.get_inbound_mails(test_mails=test_mails) + for mail in inbound_mails: + try: + communication = mail.process() + frappe.db.commit() + # If email already exists in the system + # then do not send notifications for the same email. + if communication and mail.flags.is_new_communication: + # notify all participants of this thread + if self.enable_auto_reply: + self.send_auto_reply(communication, mail) - if self.enable_incoming: - uid_list = [] - exceptions = [] - seen_status = [] - uid_reindexed = False - email_server = None + attachments = [] + if hasattr(communication, '_attachments'): + attachments = [d.file_name for d in communication._attachments] + communication.notify(attachments=attachments, fetched_from_email_account=True) + except SentEmailInInboxError: + frappe.db.rollback() + except Exception: + frappe.db.rollback() + frappe.log_error('email_account.receive') + if self.use_imap: + self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback()) + exceptions.append(frappe.get_traceback()) - if frappe.local.flags.in_test: - incoming_mails = test_mails or [] - else: - email_sync_rule = self.build_email_sync_rule() + #notify if user is linked to account + if len(inbound_mails)>0 and not frappe.local.flags.in_test: + frappe.publish_realtime('new_email', + {"account":self.email_account_name, "number":len(inbound_mails)} + ) - try: - email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) - except Exception: - frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + if exceptions: + raise Exception(frappe.as_json(exceptions)) - if not email_server: - return + def get_inbound_mails(self, test_mails=None): + """retrive and return inbound mails. - emails = email_server.get_messages() - if not emails: - return + TODO: Handle for tests + """ + if not self.enable_incoming: + return [] - incoming_mails = emails.get("latest_messages", []) - uid_list = emails.get("uid_list", []) - seen_status = emails.get("seen_status", []) - uid_reindexed = emails.get("uid_reindexed", False) + if frappe.local.flags.in_test: + return [InboundMail(msg, self) for msg in test_mails or []] - for idx, msg in enumerate(incoming_mails): - uid = None if not uid_list else uid_list[idx] - self.flags.notify = True + email_sync_rule = self.build_email_sync_rule() + try: + email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) + messages = email_server.get_messages() or {} + except Exception: + raise + print("In exception...") + frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + return [] - try: - args = { - "uid": uid, - "seen": None if not seen_status else get_seen(seen_status.get(uid, None)), - "uid_reindexed": uid_reindexed - } - communication = self.insert_communication(msg, args=args) + mails = [] + for index, message in enumerate(messages.get("latest_messages", [])): + print("RAW MESSAGE TYPE: ", type(message)) + uid = messages['uid_list'][index] + seen_status = 1 if messages['seen_status'][uid]=='SEEN' else 0 + mails.append(InboundMail(message, self, uid, seen_status)) - except SentEmailInInbox: - frappe.db.rollback() + return mails - except Exception: - frappe.db.rollback() - frappe.log_error('email_account.receive') - if self.use_imap: - self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback()) - exceptions.append(frappe.get_traceback()) - - else: - frappe.db.commit() - if communication and self.flags.notify: - - # If email already exists in the system - # then do not send notifications for the same email. - - attachments = [] - - if hasattr(communication, '_attachments'): - attachments = [d.file_name for d in communication._attachments] - - communication.notify(attachments=attachments, fetched_from_email_account=True) - - #notify if user is linked to account - if len(incoming_mails)>0 and not frappe.local.flags.in_test: - frappe.publish_realtime('new_email', {"account":self.email_account_name, "number":len(incoming_mails)}) - - if exceptions: - raise Exception(frappe.as_json(exceptions)) - - def handle_bad_emails(self, email_server, uid, raw, reason): - if email_server and cint(email_server.settings.use_imap): + def handle_bad_emails(self, uid, raw, reason): + if cint(self.use_imap): import email try: - mail = email.message_from_string(raw) + if isinstance(raw, bytes): + mail = email.message_from_bytes(raw) + else: + mail = email.message_from_string(raw) message_id = mail.get('Message-ID') except Exception: @@ -524,275 +514,18 @@ class EmailAccount(Document): "reason":reason, "message_id": message_id, "doctype": "Unhandled Email", - "email_account": email_server.settings.email_account + "email_account": self.name }) unhandled_email.insert(ignore_permissions=True) frappe.db.commit() - def insert_communication(self, msg, args=None): - if isinstance(msg, list): - raw, uid, seen = msg - else: - raw = msg - uid = -1 - seen = 0 - if isinstance(args, dict): - if args.get("uid", -1): uid = args.get("uid", -1) - if args.get("seen", 0): seen = args.get("seen", 0) - - email = Email(raw) - - if email.from_email == self.email_id and not email.mail.get("Reply-To"): - # gmail shows sent emails in inbox - # and we don't want emails sent by us to be pulled back into the system again - # dont count emails sent by the system get those - if frappe.flags.in_test: - print('WARN: Cannot pull email. Sender sames as recipient inbox') - raise SentEmailInInbox - - if email.message_id: - # https://stackoverflow.com/a/18367248 - names = frappe.db.sql("""SELECT DISTINCT `name`, `creation` FROM `tabCommunication` - WHERE `message_id`='{message_id}' - ORDER BY `creation` DESC LIMIT 1""".format( - message_id=email.message_id - ), as_dict=True) - - if names: - name = names[0].get("name") - # email is already available update communication uid instead - frappe.db.set_value("Communication", name, "uid", uid, update_modified=False) - - self.flags.notify = False - - return frappe.get_doc("Communication", name) - - if email.content_type == 'text/html': - email.content = clean_email_html(email.content) - - communication = frappe.get_doc({ - "doctype": "Communication", - "subject": email.subject, - "content": email.content, - 'text_content': email.text_content, - "sent_or_received": "Received", - "sender_full_name": email.from_real_name, - "sender": email.from_email, - "recipients": email.mail.get("To"), - "cc": email.mail.get("CC"), - "email_account": self.name, - "communication_medium": "Email", - "uid": int(uid or -1), - "message_id": email.message_id, - "communication_date": email.date, - "has_attachment": 1 if email.attachments else 0, - "seen": seen or 0 - }) - - self.set_thread(communication, email) - if communication.seen: - # get email account user and set communication as seen - users = frappe.get_all("User Email", filters={ "email_account": self.name }, - fields=["parent"]) - users = list(set([ user.get("parent") for user in users ])) - communication._seen = json.dumps(users) - - communication.flags.in_receive = True - communication.insert(ignore_permissions=True) - - # save attachments - communication._attachments = email.save_attachments_in_doc(communication) - - # replace inline images - dirty = False - for file in communication._attachments: - if file.name in email.cid_map and email.cid_map[file.name]: - dirty = True - - email.content = email.content.replace("cid:{0}".format(email.cid_map[file.name]), - file.file_url) - - if dirty: - # not sure if using save() will trigger anything - communication.db_set("content", sanitize_html(email.content)) - - # notify all participants of this thread - if self.enable_auto_reply and getattr(communication, "is_first", False): - self.send_auto_reply(communication, email) - - return communication - - def set_thread(self, communication, email): - """Appends communication to parent based on thread ID. Will extract - parent communication and will link the communication to the reference of that - communication. Also set the status of parent transaction to Open or Replied. - - If no thread id is found and `append_to` is set for the email account, - it will create a new parent transaction (e.g. Issue)""" - parent = None - - parent = self.find_parent_from_in_reply_to(communication, email) - - if not parent and self.append_to: - self.set_sender_field_and_subject_field() - - if not parent and self.append_to: - parent = self.find_parent_based_on_subject_and_sender(communication, email) - - if not parent and self.append_to and self.append_to!="Communication": - parent = self.create_new_parent(communication, email) - - if parent: - communication.reference_doctype = parent.doctype - communication.reference_name = parent.name - - # check if message is notification and disable notifications for this message - isnotification = email.mail.get("isnotification") - if isnotification: - if "notification" in isnotification: - communication.unread_notification_sent = 1 - - def set_sender_field_and_subject_field(self): - '''Identify the sender and subject fields from the `append_to` DocType''' - # set subject_field and sender_field - meta = frappe.get_meta(self.append_to) - self.subject_field = None - self.sender_field = None - - if hasattr(meta, "subject_field"): - self.subject_field = meta.subject_field - - if hasattr(meta, "sender_field"): - self.sender_field = meta.sender_field - - def find_parent_based_on_subject_and_sender(self, communication, email): - '''Find parent document based on subject and sender match''' - parent = None - - if self.append_to and self.sender_field: - if self.subject_field: - if '#' in email.subject: - # try and match if ID is found - # document ID is appended to subject - # example "Re: Your email (#OPP-2020-2334343)" - parent_id = email.subject.rsplit('#', 1)[-1].strip(' ()') - if parent_id: - parent = frappe.db.get_all(self.append_to, filters = dict(name = parent_id), - fields = 'name') - - if not parent: - # try and match by subject and sender - # if sent by same sender with same subject, - # append it to old coversation - subject = frappe.as_unicode(strip(re.sub(r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", - "", email.subject, 0, flags=re.IGNORECASE))) - - parent = frappe.db.get_all(self.append_to, filters={ - self.sender_field: email.from_email, - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - if not parent and len(subject) > 10 and is_system_user(email.from_email): - # match only subject field - # when the from_email is of a user in the system - # and subject is atleast 10 chars long - parent = frappe.db.get_all(self.append_to, filters={ - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - - - if parent: - parent = frappe._dict(doctype=self.append_to, name=parent[0].name) - return parent - - def create_new_parent(self, communication, email): - '''If no parent found, create a new reference document''' - - # no parent found, but must be tagged - # insert parent type doc - parent = frappe.new_doc(self.append_to) - - if self.subject_field: - parent.set(self.subject_field, frappe.as_unicode(email.subject)[:140]) - - if self.sender_field: - parent.set(self.sender_field, frappe.as_unicode(email.from_email)) - - if parent.meta.has_field("email_account"): - parent.email_account = self.name - - parent.flags.ignore_mandatory = True - - try: - parent.insert(ignore_permissions=True) - except frappe.DuplicateEntryError: - # try and find matching parent - parent_name = frappe.db.get_value(self.append_to, {self.sender_field: email.from_email}) - if parent_name: - parent.name = parent_name - else: - parent = None - - # NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True - communication.is_first = True - - return parent - - def find_parent_from_in_reply_to(self, communication, email): - '''Returns parent reference if embedded in In-Reply-To header - - Message-ID is formatted as `{message_id}@{site}`''' - parent = None - in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>") - - if in_reply_to: - if "@{0}".format(frappe.local.site) in in_reply_to: - # reply to a communication sent from the system - email_queue = frappe.db.get_value('Email Queue', dict(message_id=in_reply_to), ['communication','reference_doctype', 'reference_name']) - if email_queue: - parent_communication, parent_doctype, parent_name = email_queue - if parent_communication: - communication.in_reply_to = parent_communication - else: - reference, domain = in_reply_to.split("@", 1) - parent_doctype, parent_name = 'Communication', reference - - if frappe.db.exists(parent_doctype, parent_name): - parent = frappe._dict(doctype=parent_doctype, name=parent_name) - - # set in_reply_to of current communication - if parent_doctype=='Communication': - # communication.in_reply_to = email_queue.communication - - if parent.reference_name: - # the true parent is the communication parent - parent = frappe.get_doc(parent.reference_doctype, - parent.reference_name) - else: - comm = frappe.db.get_value('Communication', - dict( - message_id=in_reply_to, - creation=['>=', add_days(get_datetime(), -30)]), - ['reference_doctype', 'reference_name'], as_dict=1) - if comm: - parent = frappe._dict(doctype=comm.reference_doctype, name=comm.reference_name) - - return parent - def send_auto_reply(self, communication, email): """Send auto reply if set.""" from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts - if self.enable_auto_reply: set_incoming_outgoing_accounts(communication) - if self.send_unsubscribe_message: - unsubscribe_message = _("Leave this conversation") - else: - unsubscribe_message = "" + unsubscribe_message = (self.send_unsubscribe_message and _("Leave this conversation")) or "" frappe.sendmail(recipients = [email.from_email], sender = self.email_id, diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 076dfc5417..3c34182fe6 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -45,6 +45,11 @@ class EmailQueue(Document): def find(cls, name): return frappe.get_doc(cls.DOCTYPE, name) + @classmethod + def find_one_by_filters(cls, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs) + return cls.find(name) if name else None + def update_db(self, commit=False, **kwargs): frappe.db.set_value(self.DOCTYPE, self.name, kwargs) if commit: diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 45888119ea..7a24583c83 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -359,9 +359,7 @@ def add_attachment(fname, fcontent, content_type=None, def get_message_id(): '''Returns Message ID created from doctype and name''' - return "<{unique}@{site}>".format( - site=frappe.local.site, - unique=email.utils.make_msgid(random_string(10)).split('@')[0].split('<')[1]) + return email.utils.make_msgid(domain=frappe.local.site) def get_signature(email_account): if email_account and email_account.add_signature and email_account.signature: diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 949da4a343..52b9110ddb 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -8,6 +8,7 @@ import imaplib import poplib import re import time +import json from email.header import decode_header import _socket @@ -20,13 +21,26 @@ from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file.file import (MaxFileSizeReachedError, get_random_filename) from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, - extract_email_id, markdown, now, parse_addr, strip) + extract_email_id, markdown, now, parse_addr, strip, get_datetime, + add_days, sanitize_html) +from frappe.utils.user import is_system_user +from frappe.utils.html_utils import clean_email_html + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass class TotalSizeExceededError(frappe.ValidationError): pass class LoginLimitExceeded(frappe.ValidationError): pass +class SentEmailInInboxError(Exception): + pass class EmailServer: """Wrapper for POP server to pull emails.""" @@ -100,14 +114,11 @@ class EmailServer: def get_messages(self): """Returns new email messages in a list.""" - if not self.check_mails(): - return # nothing to do + if not (self.check_mails() or self.connect()): + return [] frappe.db.commit() - if not self.connect(): - return - uid_list = [] try: @@ -116,7 +127,6 @@ class EmailServer: self.latest_messages = [] self.seen_status = {} self.uid_reindexed = False - uid_list = email_list = self.get_new_mails() if not email_list: @@ -132,11 +142,7 @@ class EmailServer: self.max_email_size = cint(frappe.local.conf.get("max_email_size")) self.max_total_size = 5 * self.max_email_size - for i, message_meta in enumerate(email_list): - # do not pull more than NUM emails - if (i+1) > num: - break - + for i, message_meta in enumerate(email_list[:num]): try: self.retrieve_message(message_meta, i+1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): @@ -152,7 +158,6 @@ class EmailServer: except Exception as e: if self.has_login_limit_exceeded(e): pass - else: raise @@ -284,7 +289,7 @@ class EmailServer: flags = [] for flag in imaplib.ParseFlags(flag_string) or []: - pattern = re.compile("\w+") + pattern = re.compile(r"\w+") match = re.search(pattern, frappe.as_unicode(flag)) flags.append(match.group(0)) @@ -369,6 +374,7 @@ class Email: else: self.mail = email.message_from_string(content) + self.raw_message = content self.text_content = '' self.html_content = '' self.attachments = [] @@ -391,6 +397,10 @@ class Email: if self.date > now(): self.date = now() + @property + def in_reply_to(self): + return (self.mail.get("In-Reply-To") or "").strip(" <>") + def parse(self): """Walk and process multi-part email.""" for part in self.mail.walk(): @@ -555,13 +565,335 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall('(?<=\[)[\w/-]+', self.subject) + l = re.findall(r'(?<=\[)[\w/-]+', self.subject) return l and l[0] or None + def is_reply(self): + return bool(self.in_reply_to) -# fix due to a python bug in poplib that limits it to 2048 -poplib._MAXLINE = 20480 -imaplib._MAXLINE = 20480 +class InboundMail(Email): + """Class representation of incoming mail along with mail handlers. + """ + def __init__(self, content, email_account, uid=None, seen_status=None): + super().__init__(content) + self.email_account = email_account + self.uid = uid or -1 + self.seen_status = seen_status or 0 + + # System documents related to this mail + self._parent_email_queue = None + self._parent_communication = None + self._reference_document = None + + self.flags = frappe._dict() + + def get_content(self): + if self.content_type == 'text/html': + return clean_email_html(self.content) + + def process(self): + """Create communication record from email. + """ + if self.is_sender_same_as_receiver() and not self.is_reply(): + if frappe.flags.in_test: + print('WARN: Cannot pull email. Sender same as recipient inbox') + raise SentEmailInInboxError + + communication = self.is_exist_in_system() + if communication: + communication.update_db(uid=self.uid) + communication.reload() + return communication + + self.flags.is_new_communication = True + return self._build_communication_doc() + + def _build_communication_doc(self): + data = self.as_dict() + data['doctype'] = "Communication" + + if self.parent_communication(): + data['in_reply_to'] = self.parent_communication().name + + if self.reference_document(): + data['reference_doctype'] = self.reference_document().doctype + data['reference_name'] = self.reference_document().name + elif self.email_account.append_to and self.email_account.append_to != 'Communication': + reference_doc = self._create_reference_document(self.email_account.append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True + + if self.is_notification(): + # Disable notifications for notification. + data['unread_notification_sent'] = 1 + + if self.seen_status: + data['_seen'] = json.dumps(self.get_users_linked_to_account(self.email_account)) + + communication = frappe.get_doc(data) + communication.flags.in_receive = True + communication.insert(ignore_permissions=True) + + # save attachments + communication._attachments = self.save_attachments_in_doc(communication) + communication.content = sanitize_html(self.replace_inline_images(communication._attachments)) + communication.save() + return communication + + def replace_inline_images(self, attachments): + # replace inline images + content = self.content + for file in attachments: + if file.name in self.cid_map and self.cid_map[file.name]: + content = content.replace("cid:{0}".format(self.cid_map[file.name]), + file.file_url) + return content + + def is_notification(self): + isnotification = self.mail.get("isnotification") + return isnotification and ("notification" in isnotification) + + def is_exist_in_system(self): + """Check if this email already exists in the system(as communication document). + """ + from frappe.core.doctype.communication.communication import Communication + if not self.message_id: + return + + return Communication.find_one_by_filters(message_id = self.message_id, + order_by = 'creation DESC') + + def is_sender_same_as_receiver(self): + return self.from_email == self.email_account.email_id + + def is_reply_to_system_sent_mail(self): + """Is it a reply to already sent mail. + """ + return self.is_reply() and frappe.local.site in self.in_reply_to + + def parent_email_queue(self): + """Get parent record from `Email Queue`. + + If it is a reply to already sent mail, then there will be a parent record in EMail Queue. + """ + from frappe.email.doctype.email_queue.email_queue import EmailQueue + + if self._parent_email_queue is not None: + return self._parent_email_queue + + parent_email_queue = '' + if self.is_reply_to_system_sent_mail(): + parent_email_queue = EmailQueue.find_one_by_filters(message_id=self.in_reply_to) + + self._parent_email_queue = parent_email_queue or '' + return self._parent_email_queue + + def parent_communication(self): + """Find a related communication so that we can prepare a mail thread. + + The way it happens is by using in-reply-to header, and we can't make thread if it does not exist. + + Here are the cases to handle: + 1. If mail is a reply to already sent mail, then we can get parent communicaion from + Email Queue record. + 2. Sometimes we send communication name in message-ID directly, use that to get parent communication. + 3. Sender sent a reply but reply is on top of what (s)he sent before, + then parent record exists directly in communication. + """ + from frappe.core.doctype.communication.communication import Communication + if self._parent_communication is not None: + return self._parent_communication + + if not self.is_reply(): + return '' + + if not self.is_reply_to_system_sent_mail(): + communication = Communication.find_one_by_filters(message_id=self.in_reply_to, + creation = ['>=', self.get_relative_dt(-30)]) + elif self.parent_email_queue() and self.parent_email_queue().communication: + communication = Communication.find(self.parent_email_queue().communication, ignore_error=True) + else: + reference = self.in_reply_to + if '@' in self.in_reply_to: + reference, _ = self.in_reply_to.split("@", 1) + communication = Communication.find(reference, ignore_error=True) + + self._parent_communication = communication or '' + return self._parent_communication + + def reference_document(self): + """Reference document is a document to which mail relate to. + + We can get reference document from Parent record(EmailQueue | Communication) if exists. + Otherwise we do subject match to find reference document if we know the reference(append_to) doctype. + """ + if self._reference_document is not None: + return self._reference_document + + reference_document = "" + parent = self.parent_email_queue() or self.parent_communication() + + if parent and parent.reference_doctype: + reference_doctype, reference_name = parent.reference_doctype, parent.reference_name + reference_document = self.get_doc(reference_doctype, reference_name, ignore_error=True) + + if not reference_document and self.email_account.append_to: + reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to) + + # if not reference_document: + # reference_document = Create_reference_document(self.email_account.append_to) + + self._reference_document = reference_document or '' + return self._reference_document + + def get_reference_name_from_subject(self): + """ + Ex: "Re: Your email (#OPP-2020-2334343)" + """ + return self.subject.rsplit('#', 1)[-1].strip(' ()') + + def match_record_by_subject_and_sender(self, doctype): + """Find a record in the given doctype that matches with email subject and sender. + + Cases: + 1. Sometimes record name is part of subject. We can get document by parsing name from subject + 2. Find by matching sender and subject + 3. Find by matching subject alone (Special case) + Ex: when a System User is using Outlook and replies to an email from their own client, + it reaches the Email Account with the threading info lost and the (sender + subject match) + doesn't work because the sender in the first communication was someone different to whom + the system user is replying to via the common email account in Frappe. This fix bypasses + the sender match when the sender is a system user and subject is atleast 10 chars long + (for additional safety) + + NOTE: We consider not to match by subject if match record is very old. + """ + name = self.get_reference_name_from_subject() + email_fields = self.get_email_fields(doctype) + + record = self.get_doc(doctype, name, ignore_error=True) if name else None + + if not record: + subject = self.clean_subject(self.subject) + filters = { + email_fields.subject_field: ("like", f"%{subject}%"), + "creation": (">", self.get_relative_dt(days=-60)) + } + + # Sender check is not needed incase mail is from system user. + if not (len(subject) > 10 and is_system_user(self.from_email)): + filters[email_fields.sender_field] = self.from_email + + name = frappe.db.get_value(self.email_account.append_to, filters = filters) + record = self.get_doc(doctype, name, ignore_error=True) if name else None + return record + + def _create_reference_document(self, doctype): + """ Create reference document if it does not exist in the system. + """ + parent = frappe.new_doc(doctype) + email_fileds = self.get_email_fields(doctype) + + if email_fileds.subject_field: + parent.set(email_fileds.subject_field, frappe.as_unicode(self.subject)[:140]) + + if email_fileds.sender_field: + parent.set(email_fileds.sender_field, frappe.as_unicode(self.from_email)) + + parent.flags.ignore_mandatory = True + + try: + parent.insert(ignore_permissions=True) + except frappe.DuplicateEntryError: + # try and find matching parent + parent_name = frappe.db.get_value(self.email_account.append_to, + {email_fileds.sender_field: email.from_email} + ) + if parent_name: + parent.name = parent_name + else: + parent = None + return parent + + + @staticmethod + def get_doc(doctype, docname, ignore_error=False): + try: + print(doctype, docname) + return frappe.get_doc(doctype, docname) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @staticmethod + def get_relative_dt(days): + """Get relative to current datetime. Only relative days are supported. + """ + return add_days(get_datetime(), days) + + @staticmethod + def get_users_linked_to_account(email_account): + """Get list of users who linked to Email account. + """ + users = frappe.get_all("User Email", filters={"email_account": email_account.name}, + fields=["parent"]) + return list(set([user.get("parent") for user in users])) + + @staticmethod + def clean_subject(subject): + """Remove Prefixes like 'fw', FWD', 're' etc from subject. + """ + # Match strings like "fw:", "re :" etc. + regex = r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*" + return frappe.as_unicode(strip(re.sub(regex, "", subject, 0, flags=re.IGNORECASE))) + + @staticmethod + def get_email_fields(doctype): + """Returns Email related fields of a doctype. + """ + fields = frappe._dict() + + email_fields = ['subject_field', 'sender_field'] + meta = frappe.get_meta(doctype) + + for field in email_fields: + if hasattr(meta, field): + fields[field] = getattr(meta, field) + print("FIELDS::::", fields) + return fields + + @staticmethod + def get_document(self, doctype, name): + """Is same as frappe.get_doc but suppresses the DoesNotExist error. + """ + try: + return frappe.get_doc(doctype, name) + except frappe.DoesNotExistError: + return None + + def as_dict(self): + """ + """ + return { + "subject": self.subject, + "content": self.get_content(), + 'text_content': self.text_content, + "sent_or_received": "Received", + "sender_full_name": self.from_real_name, + "sender": self.from_email, + "recipients": self.mail.get("To"), + "cc": self.mail.get("CC"), + "email_account": self.email_account.name, + "communication_medium": "Email", + "uid": self.uid, + "message_id": self.message_id, + "communication_date": self.date, + "has_attachment": 1 if self.attachments else 0, + "seen": self.seen_status or 0 + } class TimerMixin(object): def __init__(self, *args, **kwargs): diff --git a/frappe/hooks.py b/frappe/hooks.py index 1c78d47755..740aa94d9f 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -202,9 +202,9 @@ scheduler_events = { ] }, "all": [ - "frappe.email.queue.flush", - "frappe.email.doctype.email_account.email_account.pull", - "frappe.email.doctype.email_account.email_account.notify_unreplied", + # "frappe.email.queue.flush", + # "frappe.email.doctype.email_account.email_account.pull", + # "frappe.email.doctype.email_account.email_account.notify_unreplied", "frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment", 'frappe.utils.global_search.sync_global_search', "frappe.monitor.flush", From 78468508d4258ec8f810fac9bc41551d1a498c56 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 23 May 2021 13:11:02 +0530 Subject: [PATCH 07/29] fix: grid navigation with keyboard on multiple pages --- frappe/public/js/frappe/form/grid_row.js | 41 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 453b8b5f24..c25d074416 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -523,7 +523,7 @@ export default class GridRow { // hide other var open_row = this.get_open_form(); - if (show===undefined) show = !!!open_row; + if (show === undefined) show = !open_row; // call blur document.activeElement && document.activeElement.blur(); @@ -588,19 +588,42 @@ export default class GridRow { this.wrapper.removeClass("grid-row-open"); } open_prev() { - const row_index = this.wrapper.index(); - if (this.grid.grid_rows[row_index - 1]) { - this.grid.grid_rows[row_index - 1].toggle_view(true); - } + if (!this.doc) return; + this.open_row_at_index(this.doc.idx - 2); } open_next() { - const row_index = this.wrapper.index(); - if (this.grid.grid_rows[row_index + 1]) { - this.grid.grid_rows[row_index + 1].toggle_view(true); - } else { + if (!this.doc) return; + + if (!this.open_row_at_index(this.doc.idx)) { this.grid.add_new_row(null, null, true); } } + open_row_at_index(row_index) { + if (!this.grid.data[row_index]) return; + + this.change_page_if_reqd(row_index); + this.grid.grid_rows[row_index].toggle_view(true); + return true; + } + change_page_if_reqd(row_index) { + const { + page_index, + page_length + } = this.grid.grid_pagination; + + row_index++; + let new_page; + + if (row_index <= (page_index - 1) * page_length) { + new_page = page_index - 1; + } else if (row_index > page_index * page_length) { + new_page = page_index + 1; + } + + if (new_page) { + this.grid.grid_pagination.go_to_page(new_page); + } + } refresh_field(fieldname, txt) { let df = this.docfields.find(col => { return col.fieldname === fieldname; From 39e6c040de40564b8186827a437b37f3ace54c00 Mon Sep 17 00:00:00 2001 From: leela Date: Fri, 21 May 2021 18:57:40 +0530 Subject: [PATCH 08/29] test: Incoming mail tests are added --- .../doctype/communication/communication.py | 17 -- .../doctype/email_account/email_account.py | 3 - .../email_account/test_email_account.py | 245 +++++++++++++++++- .../test_mails/incoming-self-sent.raw | 91 +++++++ .../incoming-subject-placeholder.raw | 183 +++++++++++++ frappe/hooks.py | 6 +- 6 files changed, 509 insertions(+), 36 deletions(-) create mode 100644 frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw create mode 100644 frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index a9c1c1f5a3..9879807033 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -199,8 +199,6 @@ class Communication(Document): if not self.sender_full_name: self.sender_full_name = sender_email - - def send(self, print_html=None, print_format=None, attachments=None, send_me_a_copy=False, recipients=None): """Send communication via Email. @@ -327,21 +325,6 @@ class Communication(Document): if autosave: self.save(ignore_permissions=ignore_permissions) - # @classmethod - # def add_incoming_mail(cls, email, uid, seen_status): - # if exists_already(): - # pass # Modify UID - # else: - # pass # Call new() to create object -class InMailMixin: - @classmethod - def add_incoming_mail(cls, email, uid, seen_status): - if exists_already(): - pass # Modify UID - else: - pass # Call new() to create object - - def on_doctype_update(): """Add indexes in `tabCommunication`""" frappe.db.add_index("Communication", ["reference_doctype", "reference_name"]) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 0a66efb480..eea2dfc73e 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -468,7 +468,6 @@ class EmailAccount(Document): def get_inbound_mails(self, test_mails=None): """retrive and return inbound mails. - TODO: Handle for tests """ if not self.enable_incoming: return [] @@ -482,13 +481,11 @@ class EmailAccount(Document): messages = email_server.get_messages() or {} except Exception: raise - print("In exception...") frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) return [] mails = [] for index, message in enumerate(messages.get("latest_messages", [])): - print("RAW MESSAGE TYPE: ", type(message)) uid = messages['uid_list'][index] seen_status = 1 if messages['seen_status'][uid]=='SEEN' else 0 mails.append(InboundMail(message, self, uid, seen_status)) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index f87ee32bb1..7bf49593f4 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -2,6 +2,10 @@ # See license.txt from __future__ import unicode_literals +from frappe.core.doctype import communication +from frappe.core.doctype.communication.communication import Communication +from frappe.email.receive import InboundMail, SentEmailInInboxError, Email +from frappe.email.email_body import get_message_id import frappe, os import unittest, email @@ -16,30 +20,37 @@ from frappe.email.doctype.email_account.email_account import notify_unreplied from datetime import datetime, timedelta class TestEmailAccount(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + email_account.db_set("enable_auto_reply", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + def setUp(self): frappe.flags.mute_emails = False frappe.flags.sent_mail = None - - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 1) frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabUnhandled Email`') - def tearDown(self): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 0) + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() def test_incoming(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-1.raw"), "r") as f: - test_mails = [f.read()] + test_mails = [self.get_test_mail('incoming-1.raw')] email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) - # check if todo is created self.assertTrue(frappe.db.get_value(comm.reference_doctype, comm.reference_name, "name")) @@ -88,7 +99,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_incoming_attached_email_from_outlook_layers(self): @@ -101,7 +112,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_outgoing(self): @@ -166,7 +177,6 @@ class TestEmailAccount(unittest.TestCase): comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) - # both communications attached to the same reference self.assertEqual(comm_list[0].reference_doctype, comm_list[1].reference_doctype) self.assertEqual(comm_list[0].reference_name, comm_list[1].reference_name) @@ -199,6 +209,215 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(comm_list[0].reference_doctype, event.doctype) self.assertEqual(comm_list[0].reference_name, event.name) + def test_auto_reply(self): + cleanup("test_sender@example.com") + + test_mails = [self.get_test_mail('incoming-1.raw')] + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.receive(test_mails=test_mails) + + comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) + self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, + "reference_name": comm.reference_name})) + + def test_handle_bad_emails(self): + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).mail.get('Message-ID') + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing") + self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id})) + +class TestInboundMail(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + + def tearDown(self): + cleanup() + frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabToDo`') + + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() + + def new_doc(self, doctype, **data): + doc = frappe.new_doc(doctype) + for field, value in data.items(): + setattr(doc, field, value) + doc.insert() + return doc + + def new_communication(self, **kwargs): + defaults = { + 'subject': "Test Subject" + } + d = {**defaults, **kwargs} + return self.new_doc('Communication', **d) + + def new_email_queue(self, **kwargs): + defaults = { + 'message_id': get_message_id().strip(" <>") + } + d = {**defaults, **kwargs} + return self.new_doc('Email Queue', **d) + + def new_todo(self, **kwargs): + defaults = { + 'description': "Description" + } + d = {**defaults, **kwargs} + return self.new_doc('ToDo', **d) + + def test_self_sent_mail(self): + """Check that we raise SentEmailInInboxError if the inbound mail is self sent mail. + """ + mail_content = self.get_test_mail(fname="incoming-self-sent.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 1, 1) + with self.assertRaises(SentEmailInInboxError): + inbound_mail.process() + + def test_mail_exist_validation(self): + """Do not create communication record if the mail is already downloaded into the system. + """ + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).message_id + # Create new communication record in DB + communication = self.new_communication(message_id=message_id) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + new_communiction = inbound_mail.process() + + # Make sure that uid is changed to new uid + self.assertEqual(new_communiction.uid, 12345) + self.assertEqual(communication.name, new_communiction.name) + + def test_find_parent_email_queue(self): + """If the mail is reply to the already sent mail, there will be a email queue record. + """ + # Create email queue record + queue_record = self.new_email_queue() + + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_queue = inbound_mail.parent_email_queue() + self.assertEqual(queue_record.name, parent_queue.name) + + def test_find_parent_communication_through_queue(self): + """Find parent communication of an inbound mail. + Cases where parent communication does exist: + 1. No parent communication is the mail is not a reply. + + Cases where parent communication does not exist: + 2. If mail is not a reply to system sent mail, then there can exist co + """ + # Create email queue record + communication = self.new_communication() + queue_record = self.new_email_queue(communication=communication.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_for_self_reply(self): + """If the inbound email is a reply but not reply to system sent mail. + + Ex: User replied to his/her mail. + """ + message_id = "new-message-id" + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertFalse(parent_communication) + + communication = self.new_communication(message_id=message_id) + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_from_header(self): + """Incase of header contains parent communication name + """ + communication = self.new_communication() + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", f"<{communication.name}@{frappe.local.site}>" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_reference_document(self): + # Create email queue record + todo = self.new_todo() + # communication = self.new_communication(reference_doctype='ToDo', reference_name=todo.name) + queue_record = self.new_email_queue(reference_doctype='ToDo', reference_name=todo.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_record_name_in_subject(self): + # Create email queue record + todo = self.new_todo() + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: (#{todo.name})" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_subject_match(self): + subject = "New todo" + todo = self.new_todo(sender='test_sender@example.com', description=subject) + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: {subject}" + ) + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_create_communication_from_mail(self): + # Create email queue record + mail_content = self.get_test_mail(fname="incoming-2.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + communication = inbound_mail.process() + self.assertTrue(communication.is_first) + self.assertTrue(communication._attachments) + def cleanup(sender=None): filters = {} if sender: @@ -207,4 +426,4 @@ def cleanup(sender=None): names = frappe.get_list("Communication", filters=filters, fields=["name"]) for name in names: frappe.delete_doc_if_exists("Communication", name.name) - frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) \ No newline at end of file + frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) diff --git a/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw new file mode 100644 index 0000000000..a16eecccd5 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw @@ -0,0 +1,91 @@ +Delivered-To: test_receiver@example.com +Received: by 10.96.153.227 with SMTP id vj3csp416144qdb; + Mon, 15 Sep 2014 03:35:07 -0700 (PDT) +X-Received: by 10.66.119.103 with SMTP id kt7mr36981968pab.95.1410777306321; + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Return-Path: +Received: from mail-pa0-x230.google.com (mail-pa0-x230.google.com [2607:f8b0:400e:c03::230]) + by mx.google.com with ESMTPS id dg10si22178346pdb.115.2014.09.15.03.35.06 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Received-SPF: pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) client-ip=2607:f8b0:400e:c03::230; +Authentication-Results: mx.google.com; + spf=pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) smtp.mail=test@example.com; + dkim=pass header.i=@gmail.com; + dmarc=pass (p=NONE dis=NONE) header.from=gmail.com +Received: by mail-pa0-f48.google.com with SMTP id hz1so6118714pad.21 + for ; Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=gmail.com; s=20120113; + h=from:content-type:subject:message-id:date:to:mime-version; + bh=rwiLijtF3lfy9M6cP/7dv2Hm7NJuBwFZn1OFsN8Tlvs=; + b=x7U4Ny3Kz2ULRJ7a04NDBrBTVhP2ImIB9n3LVNGQDnDonPUM5Ro/wZcxPTVnBWZ2L1 + o1bGfP+lhBrvYUlHsd5r4FYC0Uvpad6hbzLr0DGUQgPTxW4cGKbtDEAq+BR2JWd9f803 + vdjSWdGk8w2dt2qbngTqIZkm5U2XWjICDOAYuPIseLUgCFwi9lLyOSARFB7mjAa2YL7Q + Nswk7mbWU1hbnHP6jaBb0m8QanTc7Up944HpNDRxIrB1ZHgKzYhXtx8nhnOx588ZGIAe + E6tyG8IwogR11vLkkrBhtMaOme9PohYx4F1CSTiwspmDCadEzJFGRe//lEXKmZHAYH6g + 90Zg== +X-Received: by 10.70.38.135 with SMTP id g7mr22078275pdk.100.1410777305744; + Mon, 15 Sep 2014 03:35:05 -0700 (PDT) +Return-Path: +Received: from [192.168.0.100] ([27.106.4.70]) + by mx.google.com with ESMTPSA id zr6sm11025126pbc.50.2014.09.15.03.35.02 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:04 -0700 (PDT) +From: Rushabh Mehta +Content-Type: multipart/alternative; boundary="Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA" +Subject: test mail 🦄🌈😎 +Message-Id: <9143999C-8456-4399-9CF1-4A2DA9DD7711@gmail.com> +Date: Mon, 15 Sep 2014 16:04:57 +0530 +To: Rushabh Mehta +Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\)) +X-Mailer: Apple Mail (2.1878.6) + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +test mail + + + +@rushabh_mehta +https://erpnext.org + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; + charset=us-ascii + +test = +mail
+



@rushabh_mehta
+
+
= + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA-- diff --git a/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw new file mode 100644 index 0000000000..35ddf06b01 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw @@ -0,0 +1,183 @@ +Return-path: +Envelope-to: test_receiver@example.com +Delivery-date: Wed, 27 Jan 2016 16:24:20 +0800 +Received: from 23-59-23-10.perm.iinet.net.au ([23.59.23.10]:62191 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtp (Exim 4.86) + (envelope-from ) + id 1aOLOj-002xFL-CP + for test_receiver@example.com; Wed, 27 Jan 2016 16:24:20 +0800 +From: +To: +References: +In-Reply-To: +Subject: RE: {{ subject }} +Date: Wed, 27 Jan 2016 16:24:09 +0800 +Message-ID: <000001d158dc$1b8363a0$528a2ae0$@example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_NextPart_000_0001_01D1591F.29A7DC20" +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AQJZfZxrgcB9KnMqoZ+S4Qq9hcoSeZ3+vGiQ +Content-Language: en-au + +This is a multipart message in MIME format. + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: multipart/alternative; + boundary="----=_NextPart_001_0002_01D1591F.29A7DC20" + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Test purely for testing with the debugger has email attached + +=20 + +From: Notification [mailto:test_receiver@example.com]=20 +Sent: Wednesday, 27 January 2016 9:30 AM +To: test_receiver@example.com +Subject: Sales Invoice: SINV-12276 + +=20 + +test no 6 sent from bench to outlook to be replied to with messaging + + + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +hi there

Test purely for testing with the debugger has email = +attached

 

From:= + = +Notification [mailto:test_receiver@example.com]
Sent: Wednesday, 27 = +January 2016 9:30 AM
To: = +test_receiver@example.com
Subject: Sales Invoice: = +SINV-12276

 

test no 3 sent from bench to outlook to be replied to with = +messaging

fizz buzz

This email was sent to test_receiver@example.= +com and copied to SuperUser

Leave this conversation = +

hi

+------=_NextPart_001_0002_01D1591F.29A7DC20-- + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment + +Received: from 203-59-223-10.perm.iinet.net.au ([23.59.23.10]:49772 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtpsa (TLSv1.2:DHE-RSA-AES256-GCM-SHA384:256) + (Exim 4.86) + (envelope-from ) + id 1aOEtO-003tI4-Kv + for test_receiver@example.com; Wed, 27 Jan 2016 09:27:30 +0800 +Return-Path: +From: "Microsoft Outlook" +To: +Subject: Microsoft Outlook Test Message +MIME-Version: 1.0 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AdFYoeN8x8wUI/+QSoCJkp33NKPVmw== + +This is an e-mail message sent automatically by Microsoft Outlook while = +testing the settings for your account. diff --git a/frappe/hooks.py b/frappe/hooks.py index 740aa94d9f..1c78d47755 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -202,9 +202,9 @@ scheduler_events = { ] }, "all": [ - # "frappe.email.queue.flush", - # "frappe.email.doctype.email_account.email_account.pull", - # "frappe.email.doctype.email_account.email_account.notify_unreplied", + "frappe.email.queue.flush", + "frappe.email.doctype.email_account.email_account.pull", + "frappe.email.doctype.email_account.email_account.notify_unreplied", "frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment", 'frappe.utils.global_search.sync_global_search', "frappe.monitor.flush", From 2bbf71ade0c12ac931a898783a6e8fc330738083 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Tue, 25 May 2021 15:30:12 +0530 Subject: [PATCH 09/29] fix: refactor code --- frappe/public/js/frappe/form/grid.js | 2 +- frappe/public/js/frappe/form/grid_row.js | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 00f9d6abca..aa3cf51314 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -348,7 +348,7 @@ export default class Grid { d.idx = ri + 1; } if (d.name === undefined) { - d.name = "row" + d.idx; + d.name = "row " + d.idx; } if (this.grid_rows[ri] && !append_row) { var grid_row = this.grid_rows[ri]; diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index bbcf0f707b..f6da88df57 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -93,11 +93,8 @@ export default class GridRow { } else { data = this.grid.df.data; } - let index = -1; - - if (me.doc.name) { - index = data.findIndex(d => d.name === me.doc.name); - } + + const index = data.findIndex(d => d.name === me.doc.name); if (index > -1) { // mutate array directly, From d30e6f6a43b371aadb9adddf019ba08549c1ec8a Mon Sep 17 00:00:00 2001 From: shariquerik Date: Tue, 25 May 2021 15:34:22 +0530 Subject: [PATCH 10/29] fix: refactor code --- frappe/public/js/frappe/form/grid.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index aa3cf51314..13b5ebe842 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -196,9 +196,7 @@ export default class Grid { tasks.push(() => { if (dirty) { this.refresh(); - if (this.frm) { - this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); - } + this.frm && this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); } }); From 45b709906fc78a50d670d3b5a7901e7b72d6d22a Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 25 May 2021 19:03:30 +0530 Subject: [PATCH 11/29] fix: display currency in auto email report Currently currency is always showed as INR(default) irrespective of what currency field option is. To get the field currency, the whole document(record) is needed. Fixed it by passing the document record. (cherry picked from commit 2e58d669e22e7fc8804ddfd759cdfe24f03d2916) --- .../email/doctype/auto_email_report/auto_email_report.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 6f1cd8eebd..91ca518e67 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -245,6 +245,7 @@ def send_monthly(): def make_links(columns, data): for row in data: + doc_name = row.get('name') for col in columns: if col.fieldtype == "Link" and col.options != "Currency": if col.options and row.get(col.fieldname): @@ -253,8 +254,9 @@ def make_links(columns, data): if col.options and row.get(col.fieldname) and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency" and row.get(col.fieldname): - row[col.fieldname] = frappe.format_value(row[col.fieldname], col) - + doc = frappe.get_doc(col.parent, doc_name) if doc_name else None + # Pass the Document to get the currency based on docfield option + row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data def update_field_types(columns): @@ -262,4 +264,4 @@ def update_field_types(columns): if col.fieldtype in ("Link", "Dynamic Link", "Currency") and col.options != "Currency": col.fieldtype = "Data" col.options = "" - return columns \ No newline at end of file + return columns From 2ccea73aa28af560ebc7eae31c9f49e0a891937f Mon Sep 17 00:00:00 2001 From: Krishna Kant Hati Date: Thu, 27 May 2021 01:52:28 +0530 Subject: [PATCH 12/29] fix: tooltip displays correct title --- frappe/public/js/frappe/form/controls/link.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index 43bd7443ab..af92f3b7a9 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -200,10 +200,11 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat if(frappe.model.can_create(doctype)) { // new item r.results.push({ - label: "" + html: "" + " " + __("Create a new {0}", [__(me.get_options())]) + "", + label: __("Create a new {0}", [__(me.get_options())]), value: "create_new__link_option", action: me.new_doc }); @@ -213,10 +214,11 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat if (locals && locals['DocType']) { // not applicable in web forms r.results.push({ - label: "" + html: "" + " " + __("Advanced Search") + "", + label: __("Advanced Search"), value: "advanced_search__link_option", action: me.open_advanced_search }); From 38efba692db8461df920bff5102b5ed9ab0b5b31 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 28 May 2021 08:44:41 +0530 Subject: [PATCH 13/29] fix: Conditionally set css variables --- .../website_theme/website_theme_template.scss | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/frappe/website/doctype/website_theme/website_theme_template.scss b/frappe/website/doctype/website_theme/website_theme_template.scss index 2a67baf541..34cd66a4fe 100644 --- a/frappe/website/doctype/website_theme/website_theme_template.scss +++ b/frappe/website/doctype/website_theme/website_theme_template.scss @@ -1,13 +1,13 @@ {% if google_font %} @import url("https://fonts.googleapis.com/css2?family={{ google_font.replace(' ', '+') }}:{{ font_properties }}&display=swap"); $font-family-sans-serif: "{{ google_font }}", -apple-system, BlinkMacSystemFont, - "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", - "Droid Sans", "Helvetica Neue", sans-serif; + "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", + "Droid Sans", "Helvetica Neue", sans-serif; {% endif -%} {% if primary_color %}$primary: {{ frappe.db.get_value('Color', primary_color, 'color') }};{% endif -%} {% if dark_color %}$dark: {{ frappe.db.get_value('Color', dark_color, 'color') }};{% endif -%} -{% if text_color %}$body-color: {{ frappe.db.get_value('Color', text_color, 'color') }};{% endif -%} +{% if text_color %}$body-text-color: {{ frappe.db.get_value('Color', text_color, 'color') }};{% endif -%} {% if background_color %}$body-bg: {{ frappe.db.get_value('Color', background_color, 'color') }};{% endif -%} $enable-shadows: {{ button_shadows and "true" or "false" }}; @@ -24,7 +24,7 @@ $enable-rounded: {{ button_rounded_corners and "true" or "false" }}; {% if font_size -%} body { - font-size: {{ font_size }}; + font-size: {{ font_size }}; } {%- endif %} @@ -32,12 +32,16 @@ body { {{ custom_scss or '' }} :root { + {% if primary_color %} --primary: #{$primary}; --primary-color: #{$primary}; - --primary-light: #{$primary-light}; - --light: #{$light}; + {% endif -%} + {% if background_color %} --bg-color: #{$body-bg}; - --text-color: #{$body-color}; - --text-light: #{$body-color}; + {% endif -%} + {% if text_color %} + --text-color: #{$body-text-color}; + --text-light: #{$body-text-color}; + {% endif -%} } From a4402868774af0ac1721fd48520cd12c158e0eb7 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 28 May 2021 09:14:27 +0530 Subject: [PATCH 14/29] fix: check if salutation already exists in email body (backport #13196) (#13358) --- frappe/public/js/frappe/views/communication.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index f65f1a83c2..e2aaec553d 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -724,9 +724,14 @@ frappe.views.CommunicationComposer = class { } message += await this.get_signature(); - if (this.real_name && !message.includes("")) { - message = `

${__('Dear')} ${this.real_name},

-
${message}`; + + const SALUTATION_END_COMMENT = ""; + if (this.real_name && !message.includes(SALUTATION_END_COMMENT)) { + this.message = ` +

${__('Dear')} ${this.real_name},

+ ${SALUTATION_END_COMMENT}
+ ${message} + `; } if (this.is_a_reply) { From f14d9bd3dade7c09577b65c6b8a75495b53285c8 Mon Sep 17 00:00:00 2001 From: leela Date: Fri, 28 May 2021 07:50:31 +0530 Subject: [PATCH 15/29] fix: refresh form section while refreshing the field If all the fields with in a section has depends_on property and in such case section itself stays hidden as all fields with in it are hidden. Currently when any form field is updated, we are refreshing only the fields but not the sections. So, field refresh is not affecting the form because of section being hidden. Fix is to refresh the sections when ever form fields are refreshed. (cherry picked from commit 20c6e24a9f58a303b36da8cce247d273e6e3582f) --- frappe/public/js/frappe/form/form.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 88d7ceaa94..b7d375be8e 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -175,6 +175,7 @@ frappe.ui.form.Form = class FrappeForm { field && ["Link", "Dynamic Link"].includes(field.df.fieldtype) && field.validate && field.validate(value); me.layout.refresh_dependency(); + me.layout.refresh_sections(); let object = me.script_manager.trigger(fieldname, doc.doctype, doc.name); return object; } @@ -1078,6 +1079,7 @@ frappe.ui.form.Form = class FrappeForm { if (this.fields_dict[fname] && this.fields_dict[fname].refresh) { this.fields_dict[fname].refresh(); this.layout.refresh_dependency(); + this.layout.refresh_sections(); } } From 4ce8a600e8c607ba418b0dd79265f917518d57a6 Mon Sep 17 00:00:00 2001 From: leela Date: Mon, 31 May 2021 09:02:29 +0530 Subject: [PATCH 16/29] fix: Tests and sider issues --- .../doctype/email_account/email_account.py | 8 +++---- .../email_account/test_email_account.py | 22 +++++++++---------- frappe/email/receive.py | 2 -- frappe/tests/test_email.py | 5 ++--- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index eea2dfc73e..6ee106bb12 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -19,7 +19,7 @@ from frappe.utils import (validate_email_address, cint, cstr, get_datetime, from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, Email, InboundMail, SentEmailInInboxError +from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from poplib import error_proto from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta @@ -469,12 +469,12 @@ class EmailAccount(Document): """retrive and return inbound mails. """ - if not self.enable_incoming: - return [] - if frappe.local.flags.in_test: return [InboundMail(msg, self) for msg in test_mails or []] + if not self.enable_incoming: + return [] + email_sync_rule = self.build_email_sync_rule() try: email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 7bf49593f4..35cacac45a 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -1,23 +1,23 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals -from frappe.core.doctype import communication -from frappe.core.doctype.communication.communication import Communication +import os +import email +import unittest +from datetime import datetime, timedelta + from frappe.email.receive import InboundMail, SentEmailInInboxError, Email from frappe.email.email_body import get_message_id -import frappe, os -import unittest, email - +import frappe from frappe.test_runner import make_test_records +from frappe.core.doctype.communication.email import make +from frappe.desk.form.load import get_attachments +from frappe.email.doctype.email_account.email_account import notify_unreplied make_test_records("User") make_test_records("Email Account") -from frappe.core.doctype.communication.email import make -from frappe.desk.form.load import get_attachments -from frappe.email.doctype.email_account.email_account import notify_unreplied -from datetime import datetime, timedelta + class TestEmailAccount(unittest.TestCase): @classmethod @@ -240,7 +240,7 @@ class TestInboundMail(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.db_set("enable_incoming", 0) - def tearDown(self): + def setUp(self): cleanup() frappe.db.sql('delete from `tabEmail Queue`') frappe.db.sql('delete from `tabToDo`') diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 52b9110ddb..7b3623df21 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -821,7 +821,6 @@ class InboundMail(Email): @staticmethod def get_doc(doctype, docname, ignore_error=False): try: - print(doctype, docname) return frappe.get_doc(doctype, docname) except frappe.DoesNotExistError: if ignore_error: @@ -862,7 +861,6 @@ class InboundMail(Email): for field in email_fields: if hasattr(meta, field): fields[field] = getattr(meta, field) - print("FIELDS::::", fields) return fields @staticmethod diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index ff7e6d534c..a37d1a519c 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import unittest, frappe, re, email from six import PY3 @@ -176,7 +174,8 @@ class TestEmail(unittest.TestCase): frappe.db.sql('''delete from `tabCommunication` where sender = 'sukh@yyy.com' ''') with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: - communication = email_account.insert_communication(raw.read()) + mails = email_account.get_inbound_mails(test_mails=[raw.read()]) + communication = mails[0].process() self.assertTrue(re.search(''']*src=["']/private/files/rtco1.png[^>]*>''', communication.content)) self.assertTrue(re.search(''']*src=["']/private/files/rtco2.png[^>]*>''', communication.content)) From 8aff94d514fe7cc898f407620764d07642d8a30e Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 31 May 2021 10:30:26 +0530 Subject: [PATCH 17/29] fix(jinja): Remove frappe.utils.jinja.get_jenv() from jinja globals --- frappe/utils/jinja_globals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 347d52dc57..2881a21aff 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -2,7 +2,6 @@ # MIT License. See license.txt from __future__ import unicode_literals -from frappe.utils.jinja import get_jenv def resolve_class(classes): @@ -22,6 +21,8 @@ def resolve_class(classes): def inspect(var, render=True): + from frappe.utils.jinja import get_jenv + context = {"var": var} if render: html = "
{{ var | pprint | e }}
" From e73efaec04b86d41eb4a773288fd2ab73aa327a4 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 31 May 2021 10:42:50 +0530 Subject: [PATCH 18/29] fix: Return promise while reloading doc (#13219) --- frappe/public/js/frappe/form/form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index b7d375be8e..35ebf9274d 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1069,7 +1069,7 @@ frappe.ui.form.Form = class FrappeForm { if(!this.doc.__islocal) { frappe.model.remove_from_locals(this.doctype, this.docname); - frappe.model.with_doc(this.doctype, this.docname, () => { + return frappe.model.with_doc(this.doctype, this.docname, () => { this.refresh(); }); } From 384d875b6235573cef9ead8e9f0b2185167f78ff Mon Sep 17 00:00:00 2001 From: leela Date: Mon, 31 May 2021 15:42:45 +0530 Subject: [PATCH 19/29] refactor: smtp session creation cleanup --- frappe/email/smtp.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 3acb76af23..2a1d6d297f 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -85,18 +85,19 @@ class SMTPServer: SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP try: - self._session = SMTP(self.server, self.port) - if not self._session: + _session = SMTP(self.server, self.port) + if not _session: frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError) - self.secure_session(self._session) + self.secure_session(_session) if self.login and self.password: - res = self._session.login(str(self.login or ""), str(self.password or "")) + res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly if res[0]!=235: frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) + self._session = _session return self._session except smtplib.SMTPAuthenticationError as e: From 77590184929bba29d08faf9823feaa0e32f55b46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 16:26:38 +0530 Subject: [PATCH 20/29] chore(deps): bump ws from 7.4.2 to 7.4.6 (#13369) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 86719d81f4..298c424b72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7608,9 +7608,9 @@ write-file-atomic@^3.0.0: typedarray-to-buffer "^3.1.5" ws@~7.4.2: - version "7.4.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.2.tgz#782100048e54eb36fe9843363ab1c68672b261dd" - integrity sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA== + version "7.4.6" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" + integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== xdg-basedir@^4.0.0: version "4.0.0" From 872299b3f3846be0c115c12bf74789b556cb2942 Mon Sep 17 00:00:00 2001 From: Fisher Yu <12823863+szufisher@users.noreply.github.com> Date: Mon, 31 May 2021 19:01:42 +0800 Subject: [PATCH 21/29] fix: Translate Strings (#13277) --- frappe/core/doctype/data_import/data_import.js | 2 +- frappe/core/doctype/doctype/doctype.js | 4 ++-- frappe/custom/doctype/customize_form/customize_form.js | 2 +- frappe/public/js/frappe/data_import/data_exporter.js | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 079bdaa09c..216db53c72 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -91,7 +91,7 @@ frappe.ui.form.on('Data Import', { if (frm.doc.status.includes('Success')) { frm.add_custom_button( - __('Go to {0} List', [frm.doc.reference_doctype]), + __('Go to {0} List', [__(frm.doc.reference_doctype)]), () => frappe.set_route('List', frm.doc.reference_doctype) ); } diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 1a173f7252..b4d3fb9a89 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -33,11 +33,11 @@ frappe.ui.form.on('DocType', { if (!frm.is_new() && !frm.doc.istable) { if (frm.doc.issingle) { - frm.add_custom_button(__('Go to {0}', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } else { - frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index d9d8ae196e..4e00456f0d 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -117,7 +117,7 @@ frappe.ui.form.on("Customize Form", { frappe.customize_form.set_primary_action(frm); frm.add_custom_button( - __("Go to {0} List", [frm.doc.doc_type]), + __("Go to {0} List", [__(frm.doc.doc_type)]), function() { frappe.set_route("List", frm.doc.doc_type); }, diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index dee4839b34..03e6288856 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -72,8 +72,8 @@ frappe.data_import.DataExporter = class DataExporter { let child_fieldname = df.fieldname; let label = df.reqd ? // prettier-ignore - __('{0} ({1}) (1 row mandatory)', [df.label || df.fieldname, doctype]) - : __('{0} ({1})', [df.label || df.fieldname, doctype]); + __('{0} ({1}) (1 row mandatory)', [__(df.label || df.fieldname), __(doctype)]) + : __('{0} ({1})', [__(df.label || df.fieldname), __(doctype)]); return { label, fieldname: child_fieldname, From 0afe774ddb1e8129a031ca136a3fff9d71dee5be Mon Sep 17 00:00:00 2001 From: getsali <30821137+getsali@users.noreply.github.com> Date: Mon, 31 May 2021 16:32:58 +0530 Subject: [PATCH 22/29] fix: Select appropriate email template response (#13051) * Update email_group.py Fix: Select appropriate email template response, Welcome email template can be either html or rich text * refactor: Simpify code Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/email/doctype/email_group/email_group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py index b19a134713..b1c1295f75 100755 --- a/frappe/email/doctype/email_group/email_group.py +++ b/frappe/email/doctype/email_group/email_group.py @@ -105,6 +105,6 @@ def send_welcome_email(welcome_email, email, email_group): email=email, email_group=email_group ) - - message = frappe.render_template(welcome_email.response, args) + email_message = welcome_email.response or welcome_email.response_html + message = frappe.render_template(email_message, args) frappe.sendmail(email, subject=welcome_email.subject, message=message) From c033e0d34d9f257e8e8242e17bfbba43e24f1256 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 31 May 2021 19:24:36 +0530 Subject: [PATCH 23/29] fix(server scripts): Restrict access to python's internal attributes --- frappe/__init__.py | 17 +++++++++++++++++ frappe/utils/safe_exec.py | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/frappe/__init__.py b/frappe/__init__.py index 9b208f7c2d..5793b224a3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1693,6 +1693,23 @@ def safe_eval(code, eval_globals=None, eval_locals=None): "round": round } + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + # Format Attributes + "format", "format_map", + } + + for attribute in UNSAFE_ATTRIBUTES: + if attribute in code: + throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) + if '__' in code: throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index d1307a3a90..632fc5df23 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -150,6 +150,7 @@ def get_safe_globals(): # default writer allows write access out._write_ = _write out._getitem_ = _getitem + out._getattr_ = _getattr # allow iterators and list comprehension out._getiter_ = iter @@ -176,6 +177,27 @@ def _getitem(obj, key): raise SyntaxError('Key starts with _') return obj[key] +def _getattr(object, name, default=None): + # guard function for RestrictedPython + # allow any key to be accessed as long as + # 1. it does not start with an underscore (safer_getattr) + # 2. it is not an UNSAFE_ATTRIBUTES + + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + } + + if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES): + raise SyntaxError("{name} is an unsafe attribute".format(name=name)) + return RestrictedPython.Guards.safer_getattr(object, name, default=default) + def _write(obj): # guard function for RestrictedPython # allow writing to any object From 39eee3e5c8fbd514ebedbd0526eb13508250336d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 1 Jun 2021 19:02:01 +0530 Subject: [PATCH 24/29] fix: Add min-width for awesomplete class --- frappe/public/scss/common/awesomeplete.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/scss/common/awesomeplete.scss b/frappe/public/scss/common/awesomeplete.scss index 096da1e2fd..619d6a9e3e 100644 --- a/frappe/public/scss/common/awesomeplete.scss +++ b/frappe/public/scss/common/awesomeplete.scss @@ -31,6 +31,7 @@ margin: 0; padding: var(--padding-xs); z-index: 1; + min-width: 250px; &> li { cursor: pointer; From f15edd21d1ca19d0353ca138a63d3d80d0848bd2 Mon Sep 17 00:00:00 2001 From: Abhishek Balam Date: Tue, 1 Jun 2021 19:41:18 +0530 Subject: [PATCH 25/29] feat: enable use of custom encryption_key in encrypt()/decrypt() --- frappe/tests/test_password.py | 12 +++++++++++- frappe/utils/password.py | 14 +++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_password.py b/frappe/tests/test_password.py index 98141072e2..9b68f3aace 100644 --- a/frappe/tests/test_password.py +++ b/frappe/tests/test_password.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import frappe import unittest -from frappe.utils.password import update_password, check_password, passlibctx +from frappe.utils.password import update_password, check_password, passlibctx, encrypt, decrypt class TestPassword(unittest.TestCase): def setUp(self): @@ -105,6 +105,16 @@ class TestPassword(unittest.TestCase): doc.save() self.assertEqual(doc.get_password(raise_exception=False), None) + def test_custom_encryption_key(self): + text = 'Frappe Framework' + custom_encryption_key = 'DFTBA' + + encrypted_text = encrypt(text, encryption_key=custom_encryption_key) + decrypted_text = decrypt(encrypted_text, encryption_key=custom_encryption_key) + + self.assertEqual(text, decrypted_text) + + pass def get_password_list(doc): return frappe.db.sql("""SELECT `password` diff --git a/frappe/utils/password.py b/frappe/utils/password.py index fbed3cd8e7..d97c788568 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -157,20 +157,20 @@ def create_auth_table(): frappe.db.create_auth_table() -def encrypt(pwd): - cipher_suite = Fernet(encode(get_encryption_key())) - cipher_text = cstr(cipher_suite.encrypt(encode(pwd))) +def encrypt(txt, encryption_key=None): + cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + cipher_text = cstr(cipher_suite.encrypt(encode(txt))) return cipher_text -def decrypt(pwd): +def decrypt(txt, encryption_key=None): try: - cipher_suite = Fernet(encode(get_encryption_key())) - plain_text = cstr(cipher_suite.decrypt(encode(pwd))) + cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + plain_text = cstr(cipher_suite.decrypt(encode(txt))) return plain_text except InvalidToken: # encryption_key in site_config is changed and not valid - frappe.throw(_('Encryption key is invalid, Please check site_config.json')) + frappe.throw(_('Encryption key is invalid' + '!' if encryption_key else ', please check site_config.json.')) def get_encryption_key(): From 464e93a405d831d6496d56956673068f3282436e Mon Sep 17 00:00:00 2001 From: Abhishek Balam Date: Tue, 1 Jun 2021 21:32:32 +0530 Subject: [PATCH 26/29] fix: Allow only use of Fernet generated key for using custom encryption_key (#13399) * fix: only allow keys generated by fernet in encrypt()/decrypt() * fix: sider and semgrep fixes --- frappe/tests/test_password.py | 4 ++-- frappe/utils/password.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/frappe/tests/test_password.py b/frappe/tests/test_password.py index 9b68f3aace..6bc4bc7729 100644 --- a/frappe/tests/test_password.py +++ b/frappe/tests/test_password.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import frappe import unittest from frappe.utils.password import update_password, check_password, passlibctx, encrypt, decrypt - +from cryptography.fernet import Fernet class TestPassword(unittest.TestCase): def setUp(self): frappe.delete_doc('Email Account', 'Test Email Account Password') @@ -107,7 +107,7 @@ class TestPassword(unittest.TestCase): def test_custom_encryption_key(self): text = 'Frappe Framework' - custom_encryption_key = 'DFTBA' + custom_encryption_key = Fernet.generate_key().decode() encrypted_text = encrypt(text, encryption_key=custom_encryption_key) decrypted_text = decrypt(encrypted_text, encryption_key=custom_encryption_key) diff --git a/frappe/utils/password.py b/frappe/utils/password.py index d97c788568..34719eb384 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -61,7 +61,7 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): except frappe.db.DataError as e: if ((frappe.db.db_type == 'mariadb' and e.args[0] == DATA_TOO_LONG) or (frappe.db.db_type == 'postgres' and e.pgcode == STRING_DATA_RIGHT_TRUNCATION)): - frappe.throw("Most probably your password is too long.", exc=e) + frappe.throw(_("Most probably your password is too long.", exc=e)) raise e @@ -158,12 +158,21 @@ def create_auth_table(): def encrypt(txt, encryption_key=None): - cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + # Only use Fernet.generate_key().decode() to enter encyption_key value + + try: + cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) + except Exception: + # encryption_key is not in 32 url-safe base64-encoded format + frappe.throw(_('Encryption key is in invalid format!')) + cipher_text = cstr(cipher_suite.encrypt(encode(txt))) return cipher_text def decrypt(txt, encryption_key=None): + # Only use encryption_key value generated with Fernet.generate_key().decode() + try: cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) plain_text = cstr(cipher_suite.decrypt(encode(txt))) From 8e3b358b9fbeb18cdbf04aeff1359e3bbaf694cb Mon Sep 17 00:00:00 2001 From: leela Date: Mon, 31 May 2021 22:15:57 +0530 Subject: [PATCH 27/29] refactor: email queue builder is added replaced queue.send function with Queue builder. Queue builder builds a queue from sendmail request and sends the mail if requested to send mail immediately. --- frappe/__init__.py | 10 +- .../email/doctype/email_queue/email_queue.py | 307 +++++++++++++++++- frappe/email/doctype/newsletter/newsletter.py | 1 - frappe/email/queue.py | 252 +------------- frappe/email/test_email_body.py | 21 +- frappe/tests/test_email.py | 13 +- frappe/utils/error.py | 2 +- 7 files changed, 327 insertions(+), 279 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 9b208f7c2d..880e5d48a4 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -528,16 +528,20 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message if not delayed: now = True - from frappe.email import queue - queue.send(recipients=recipients, sender=sender, + from frappe.email.doctype.email_queue.email_queue import QueueBuilder + builder = QueueBuilder(recipients=recipients, sender=sender, subject=subject, message=message, text_content=text_content, reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link, unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message, attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to, send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately, - communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification, + communication=communication, read_receipt=read_receipt, is_notification=is_notification, inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container) + # build email queue and send the email if send_now is True. + builder.process(send_now=now) + + whitelisted = [] guest_methods = [] xss_safe_methods = [] diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 394fe50468..dad473b8aa 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -9,14 +9,18 @@ from rq.timeouts import JobTimeoutException import smtplib import quopri from email.parser import Parser +from email.policy import SMTPUTF8 +from html2text import html2text +from six.moves import html_parser as HTMLParser import frappe from frappe import _, safe_encode, task from frappe.model.document import Document -from frappe.email.queue import get_unsubcribed_url -from frappe.email.email_body import add_attachment -from frappe.utils import cint -from email.policy import SMTPUTF8 +from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message +from frappe.email.email_body import add_attachment, get_formatted_html, get_email +from frappe.utils import cint, split_emails, add_days, nowdate, cstr +from frappe.email.doctype.email_account.email_account import EmailAccount + MAX_RETRY_COUNT = 3 class EmailQueue(Document): @@ -41,6 +45,19 @@ class EmailQueue(Document): duplicate.set_recipients(recipients) return duplicate + @classmethod + def new(cls, doc_data, ignore_permissions=False): + data = doc_data.copy() + if not data.get('recipients'): + return + + recipients = data.pop('recipients') + doc = frappe.new_doc(cls.DOCTYPE) + doc.update(data) + doc.set_recipients(recipients) + doc.insert(ignore_permissions=ignore_permissions) + return doc + @classmethod def find(cls, name): return frappe.get_doc(cls.DOCTYPE, name) @@ -74,8 +91,6 @@ class EmailQueue(Document): return json.loads(self.attachments) if self.attachments else [] def get_email_account(self): - from frappe.email.doctype.email_account.email_account import EmailAccount - if self.email_account: return frappe.get_doc('Email Account', self.email_account) @@ -300,3 +315,283 @@ def send_now(name): def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" frappe.db.add_index('Email Queue', ('status', 'send_after', 'priority', 'creation'), 'index_bulk_flush') + +class QueueBuilder: + """Builds Email Queue from the given data + """ + def __init__(self, recipients=None, sender=None, subject=None, message=None, + text_content=None, reference_doctype=None, reference_name=None, + unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, + attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, + send_after=None, expose_recipients=None, send_priority=1, communication=None, + read_receipt=None, queue_separately=False, is_notification=False, + add_unsubscribe_link=1, inline_images=None, header=None, + print_letterhead=False, with_container=False): + """Add email to sending queue (Email Queue) + + :param recipients: List of recipients. + :param sender: Email sender. + :param subject: Email subject. + :param message: Email message. + :param text_content: Text version of email message. + :param reference_doctype: Reference DocType of caller document. + :param reference_name: Reference name of caller document. + :param send_priority: Priority for Email Queue, default 1. + :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. + :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email + :param attachments: Attachments to be sent. + :param reply_to: Reply to be captured here (default inbox) + :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. + :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. + :param communication: Communication link to be set in Email Queue record + :param queue_separately: Queue each email separately + :param is_notification: Marks email as notification so will not trigger notifications from system + :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. + :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id + :param header: Append header in email (boolean) + :param with_container: Wraps email inside styled container + """ + + self._unsubscribe_method = unsubscribe_method + self._recipients = recipients + self._cc = cc + self._bcc = bcc + self._send_after = send_after + self._sender = sender + self._text_content = text_content + self._message = message + self._add_unsubscribe_link = add_unsubscribe_link + self._unsubscribe_message = unsubscribe_message + self._attachments = attachments + + self._unsubscribed_user_emails = None + self._email_account = None + + self.unsubscribe_params = unsubscribe_params + self.subject = subject + self.reference_doctype = reference_doctype + self.reference_name = reference_name + self.expose_recipients = expose_recipients + self.with_container = with_container + self.header = header + self.reply_to = reply_to + self.message_id = message_id + self.in_reply_to = in_reply_to + self.send_priority = send_priority + self.communication = communication + self.read_receipt = read_receipt + self.queue_separately = queue_separately + self.is_notification = is_notification + self.inline_images = inline_images + self.print_letterhead = print_letterhead + + @property + def unsubscribe_method(self): + return self._unsubscribe_method or '/api/method/frappe.email.queue.unsubscribe' + + def _get_emails_list(self, emails=None): + emails = split_emails(emails) if isinstance(emails, str) else (emails or []) + return [each for each in set(emails) if each] + + @property + def recipients(self): + return self._get_emails_list(self._recipients) + + @property + def cc(self): + return self._get_emails_list(self._cc) + + @property + def bcc(self): + return self._get_emails_list(self._bcc) + + @property + def send_after(self): + if isinstance(self._send_after, int): + return add_days(nowdate(), self._send_after) + return self._send_after + + @property + def sender(self): + if not self._sender or self._sender == "Administrator": + email_account = self.get_outgoing_email_account() + return email_account.default_sender + return self._sender + + def email_text_content(self): + unsubscribe_msg = self.unsubscribe_message() + unsubscribe_text_message = (unsubscribe_msg and unsubscribe_msg.text) or '' + + if self._text_content: + return self._text_content + unsubscribe_text_message + + try: + text_content = html2text(self._message) + except HTMLParser.HTMLParseError: + text_content = "See html attachment" + return text_content + unsubscribe_text_message + + def email_html_content(self): + email_account = self.get_outgoing_email_account() + return get_formatted_html(self.subject, self._message, header=self.header, + email_account=email_account, unsubscribe_link=self.unsubscribe_message(), + with_container=self.with_container) + + def should_include_unsubscribe_link(self): + return (self._add_unsubscribe_link == 1 + and self.reference_doctype + and (self._unsubscribe_message or self.reference_doctype=="Newsletter")) + + def unsubscribe_message(self): + if self.should_include_unsubscribe_link(): + return get_unsubscribe_message(self._unsubscribe_message, self.expose_recipients) + + def get_outgoing_email_account(self): + if self._email_account: + return self._email_account + + self._email_account = EmailAccount.find_outgoing( + match_by_doctype=self.reference_doctype, match_by_email=self._sender, _raise_error=True) + return self._email_account + + def get_unsubscribed_user_emails(self): + if self._unsubscribed_user_emails is not None: + return self._unsubscribed_user_emails + + all_ids = tuple(set(self.recipients + self.cc)) + + unsubscribed = frappe.db.sql_list(''' + SELECT + distinct email + from + `tabEmail Unsubscribe` + where + email in %(all_ids)s + and ( + ( + reference_doctype = %(reference_doctype)s + and reference_name = %(reference_name)s + ) + or global_unsubscribe = 1 + ) + ''', { + 'all_ids': all_ids, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + }) + + self._unsubscribed_user_emails = unsubscribed or [] + return self._unsubscribed_user_emails + + def final_recipients(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.recipients if mail_id not in unsubscribed_emails] + + def final_cc(self): + unsubscribed_emails = self.get_unsubscribed_user_emails() + return [mail_id for mail_id in self.cc if mail_id not in unsubscribed_emails] + + def get_attachments(self): + attachments = [] + if self._attachments: + # store attachments with fid or print format details, to be attached on-demand later + for att in self._attachments: + if att.get('fid'): + attachments.append(att) + elif att.get("print_format_attachment") == 1: + if not att.get('lang', None): + att['lang'] = frappe.local.lang + att['print_letterhead'] = self.print_letterhead + attachments.append(att) + return attachments + + def prepare_email_content(self): + mail = get_email(recipients=self.final_recipients(), + sender=self.sender, + subject=self.subject, + formatted=self.email_html_content(), + text_content=self.email_text_content(), + attachments=self._attachments, + reply_to=self.reply_to, + cc=self.final_cc(), + bcc=self.bcc, + email_account=self.get_outgoing_email_account(), + expose_recipients=self.expose_recipients, + inline_images=self.inline_images, + header=self.header) + + mail.set_message_id(self.message_id, self.is_notification) + if self.read_receipt: + mail.msg_root["Disposition-Notification-To"] = self.sender + if self.in_reply_to: + mail.set_in_reply_to(self.in_reply_to) + return mail + + def process(self, send_now=False): + """Build and return the email queues those are created. + + Sends email incase if it is requested to send now. + """ + final_recipients = self.final_recipients() + queue_separately = (final_recipients and self.queue_separately) or len(final_recipients) > 20 + if not (final_recipients + self.final_cc()): + return [] + + email_queues = [] + queue_data = self.as_dict(include_recipients=False) + if not queue_data: + return [] + + if not queue_separately: + recipients = list(set(final_recipients + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + else: + for r in final_recipients: + recipients = [r] if email_queues else list(set([r] + self.final_cc() + self.bcc)) + q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + email_queues.append(q) + + if send_now: + for doc in email_queues: + doc.send() + return email_queues + + def as_dict(self, include_recipients=True): + email_account = self.get_outgoing_email_account() + email_account_name = email_account and email_account.is_exists_in_db() and email_account.name + + mail = self.prepare_email_content() + try: + mail_to_string = cstr(mail.as_string()) + except frappe.InvalidEmailAddressError: + # bad Email Address - don't add to queue + frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ' + .format(self.sender, ', '.join(self.final_recipients()), traceback.format_exc()), + 'Email Not Sent' + ) + return + + d = { + 'priority': self.send_priority, + 'attachments': json.dumps(self.get_attachments()), + 'message_id': mail.msg_root["Message-Id"].strip(" <>"), + 'message': mail_to_string, + 'sender': self.sender, + 'reference_doctype': self.reference_doctype, + 'reference_name': self.reference_name, + 'add_unsubscribe_link': self._add_unsubscribe_link, + 'unsubscribe_method': self.unsubscribe_method, + 'unsubscribe_params': self.unsubscribe_params, + 'expose_recipients': self.expose_recipients, + 'communication': self.communication, + 'send_after': self.send_after, + 'show_as_cc': ",".join(self.final_cc()), + 'show_as_bcc': ','.join(self.bcc), + 'email_account': email_account_name or None + } + + if include_recipients: + d['recipients'] = self.final_recipients() + + return d diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 6412338e96..469390322c 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -8,7 +8,6 @@ import frappe.utils from frappe import throw, _ from frappe.website.website_generator import WebsiteGenerator from frappe.utils.verified_command import get_signed_params, verify_request -from frappe.email.queue import send from frappe.email.doctype.email_group.email_group import add_subscribers from frappe.utils import parse_addr, now_datetime, markdown, validate_email_address diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 52c91baf1c..451d1c970e 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -3,257 +3,9 @@ from __future__ import unicode_literals import frappe -import sys -from six.moves import html_parser as HTMLParser -import smtplib, quopri, json -from frappe import msgprint, _, safe_decode, safe_encode, enqueue -from frappe.email.smtp import SMTPServer -from frappe.email.doctype.email_account.email_account import EmailAccount -from frappe.email.email_body import get_email, get_formatted_html, add_attachment +from frappe import msgprint, _ from frappe.utils.verified_command import get_signed_params, verify_request -from html2text import html2text -from frappe.utils import get_url, nowdate, now_datetime, add_days, split_emails, cstr, cint -from rq.timeouts import JobTimeoutException -from six import text_type, string_types, PY3 -from email.parser import Parser - - -class EmailLimitCrossedError(frappe.ValidationError): pass - -def send(recipients=None, sender=None, subject=None, message=None, text_content=None, reference_doctype=None, - reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, - attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, send_after=None, - expose_recipients=None, send_priority=1, communication=None, now=False, read_receipt=None, - queue_separately=False, is_notification=False, add_unsubscribe_link=1, inline_images=None, - header=None, print_letterhead=False, with_container=False): - """Add email to sending queue (Email Queue) - - :param recipients: List of recipients. - :param sender: Email sender. - :param subject: Email subject. - :param message: Email message. - :param text_content: Text version of email message. - :param reference_doctype: Reference DocType of caller document. - :param reference_name: Reference name of caller document. - :param send_priority: Priority for Email Queue, default 1. - :param unsubscribe_method: URL method for unsubscribe. Default is `/api/method/frappe.email.queue.unsubscribe`. - :param unsubscribe_params: additional params for unsubscribed links. default are name, doctype, email - :param attachments: Attachments to be sent. - :param reply_to: Reply to be captured here (default inbox) - :param in_reply_to: Used to send the Message-Id of a received email back as In-Reply-To. - :param send_after: Send this email after the given datetime. If value is in integer, then `send_after` will be the automatically set to no of days from current date. - :param communication: Communication link to be set in Email Queue record - :param now: Send immediately (don't send in the background) - :param queue_separately: Queue each email separately - :param is_notification: Marks email as notification so will not trigger notifications from system - :param add_unsubscribe_link: Send unsubscribe link in the footer of the Email, default 1. - :param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id - :param header: Append header in email (boolean) - :param with_container: Wraps email inside styled container - """ - if not unsubscribe_method: - unsubscribe_method = "/api/method/frappe.email.queue.unsubscribe" - - if not recipients and not cc: - return - - if not cc: - cc = [] - if not bcc: - bcc = [] - - if isinstance(recipients, string_types): - recipients = split_emails(recipients) - - if isinstance(cc, string_types): - cc = split_emails(cc) - - if isinstance(bcc, string_types): - bcc = split_emails(bcc) - - if isinstance(send_after, int): - send_after = add_days(nowdate(), send_after) - - email_account = EmailAccount.find_outgoing( - match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True) - - if not sender or sender == "Administrator": - sender = email_account.default_sender - - if not text_content: - try: - text_content = html2text(message) - except HTMLParser.HTMLParseError: - text_content = "See html attachment" - - recipients = list(set(recipients)) - cc = list(set(cc)) - - all_ids = tuple(recipients + cc) - - unsubscribed = frappe.db.sql_list(''' - SELECT - distinct email - from - `tabEmail Unsubscribe` - where - email in %(all_ids)s - and ( - ( - reference_doctype = %(reference_doctype)s - and reference_name = %(reference_name)s - ) - or global_unsubscribe = 1 - ) - ''', { - 'all_ids': all_ids, - 'reference_doctype': reference_doctype, - 'reference_name': reference_name, - }) - - recipients = [r for r in recipients if r and r not in unsubscribed] - - if cc: - cc = [r for r in cc if r and r not in unsubscribed] - - if not recipients and not cc: - # Recipients may have been unsubscribed, exit quietly - return - - email_text_context = text_content - - should_append_unsubscribe = (add_unsubscribe_link - and reference_doctype - and (unsubscribe_message or reference_doctype=="Newsletter") - and add_unsubscribe_link==1) - - unsubscribe_link = None - if should_append_unsubscribe: - unsubscribe_link = get_unsubscribe_message(unsubscribe_message, expose_recipients) - email_text_context += unsubscribe_link.text - - email_content = get_formatted_html(subject, message, - email_account=email_account, header=header, - unsubscribe_link=unsubscribe_link, with_container=with_container) - - # add to queue - add(recipients, sender, subject, - formatted=email_content, - text_content=email_text_context, - reference_doctype=reference_doctype, - reference_name=reference_name, - attachments=attachments, - reply_to=reply_to, - cc=cc, - bcc=bcc, - message_id=message_id, - in_reply_to=in_reply_to, - send_after=send_after, - send_priority=send_priority, - email_account=email_account, - communication=communication, - add_unsubscribe_link=add_unsubscribe_link, - unsubscribe_method=unsubscribe_method, - unsubscribe_params=unsubscribe_params, - expose_recipients=expose_recipients, - read_receipt=read_receipt, - queue_separately=queue_separately, - is_notification = is_notification, - inline_images = inline_images, - header=header, - now=now, - print_letterhead=print_letterhead) - - -def add(recipients, sender, subject, **kwargs): - """Add to Email Queue""" - if kwargs.get('queue_separately') or len(recipients) > 20: - email_queue = None - for r in recipients: - if not email_queue: - email_queue = get_email_queue([r], sender, subject, **kwargs) - if kwargs.get('now'): - email_queue.send() - else: - duplicate = email_queue.get_duplicate([r]) - duplicate.insert(ignore_permissions=True) - - if kwargs.get('now'): - duplicate.send() - - frappe.db.commit() - else: - email_queue = get_email_queue(recipients, sender, subject, **kwargs) - if kwargs.get('now'): - email_queue.send() - -def get_email_queue(recipients, sender, subject, **kwargs): - '''Make Email Queue object''' - e = frappe.new_doc('Email Queue') - e.priority = kwargs.get('send_priority') - attachments = kwargs.get('attachments') - if attachments: - # store attachments with fid or print format details, to be attached on-demand later - _attachments = [] - for att in attachments: - if att.get('fid'): - _attachments.append(att) - elif att.get("print_format_attachment") == 1: - if not att.get('lang', None): - att['lang'] = frappe.local.lang - att['print_letterhead'] = kwargs.get('print_letterhead') - _attachments.append(att) - e.attachments = json.dumps(_attachments) - - try: - mail = get_email(recipients, - sender=sender, - subject=subject, - formatted=kwargs.get('formatted'), - text_content=kwargs.get('text_content'), - attachments=kwargs.get('attachments'), - reply_to=kwargs.get('reply_to'), - cc=kwargs.get('cc'), - bcc=kwargs.get('bcc'), - email_account=kwargs.get('email_account'), - expose_recipients=kwargs.get('expose_recipients'), - inline_images=kwargs.get('inline_images'), - header=kwargs.get('header')) - - mail.set_message_id(kwargs.get('message_id'),kwargs.get('is_notification')) - if kwargs.get('read_receipt'): - mail.msg_root["Disposition-Notification-To"] = sender - if kwargs.get('in_reply_to'): - mail.set_in_reply_to(kwargs.get('in_reply_to')) - - e.message_id = mail.msg_root["Message-Id"].strip(" <>") - e.message = cstr(mail.as_string()) - e.sender = mail.sender - - except frappe.InvalidEmailAddressError: - # bad Email Address - don't add to queue - import traceback - frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} '.format(mail.sender, - ', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent') - - recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', []))) - email_account = kwargs.get('email_account') - email_account_name = email_account and email_account.is_exists_in_db() and email_account.name - - e.set_recipients(recipients) - e.reference_doctype = kwargs.get('reference_doctype') - e.reference_name = kwargs.get('reference_name') - e.add_unsubscribe_link = kwargs.get("add_unsubscribe_link") - e.unsubscribe_method = kwargs.get('unsubscribe_method') - e.unsubscribe_params = kwargs.get('unsubscribe_params') - e.expose_recipients = kwargs.get('expose_recipients') - e.communication = kwargs.get('communication') - e.send_after = kwargs.get('send_after') - e.show_as_cc = ",".join(kwargs.get('cc', [])) - e.show_as_bcc = ",".join(kwargs.get('bcc', [])) - e.email_account = email_account_name or None - e.insert(ignore_permissions=True) - return e +from frappe.utils import get_url, now_datetime, cint def get_emails_sent_this_month(): return frappe.db.sql(""" diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 33668cddba..5b37102a17 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -7,8 +7,7 @@ from frappe import safe_decode from frappe.email.receive import Email from frappe.email.email_body import (replace_filename_with_cid, get_email, inline_style_in_html, get_header) -from frappe.email.queue import get_email_queue -from frappe.email.doctype.email_queue.email_queue import SendMailContext +from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder from six import PY3 class TestEmailBody(unittest.TestCase): @@ -50,27 +49,25 @@ This is the text version of this email uni_chr1 = unichr(40960) uni_chr2 = unichr(1972) - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - formatted='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - text_content='whatever') - mail_ctx = SendMailContext(queue_doc = email) + message='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', + text_content='whatever').process()[0] + mail_ctx = SendMailContext(queue_doc = queue_doc) result = mail_ctx.build_message(recipient_email = 'test@test.com') self.assertTrue(b"

=EA=80=80abcd=DE=B4

" in result) def test_prepare_message_returns_cr_lf(self): - email = get_email_queue( + queue_doc = QueueBuilder( recipients=['test@example.com'], sender='me@example.com', subject='Test Subject', - content='

\n this is a test of newlines\n' + '

', - formatted='

\n this is a test of newlines\n' + '

', - text_content='whatever') + message='

\n this is a test of newlines\n' + '

', + text_content='whatever').process()[0] - mail_ctx = SendMailContext(queue_doc = email) + mail_ctx = SendMailContext(queue_doc = queue_doc) result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com')) if PY3: diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 74962dd84c..58ffe7531d 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -143,7 +143,8 @@ class TestEmail(unittest.TestCase): self.assertEqual(len(queue_recipients), 2) def test_unsubscribe(self): - from frappe.email.queue import unsubscribe, send + from frappe.email.queue import unsubscribe + from frappe.email.doctype.email_queue.email_queue import QueueBuilder unsubscribe(doctype="User", name="Administrator", email="test@example.com") self.assertTrue(frappe.db.get_value("Email Unsubscribe", @@ -152,11 +153,11 @@ class TestEmail(unittest.TestCase): before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[0][0] - send(recipients=['test@example.com', 'test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") - + builder = QueueBuilder(recipients=['test@example.com', 'test1@example.com'], + sender="admin@example.com", + reference_doctype='User', reference_name="Administrator", + subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") + builder.process() # this is sent async (?) email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 2d8d6491a5..68577e47e5 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -215,7 +215,7 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None): >>> @raise_error_on_no_output("Ingradients missing") ... def get_indradients(_raise_error=1): return ... - >>> get_indradients() + >>> get_ingradients() `Exception Name`: Ingradients missing """ def decorator_raise_error_on_no_output(func): From e489c1b594697931972cf819c44a41b25311a75c Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 2 Jun 2021 13:05:16 +0530 Subject: [PATCH 28/29] fix: Added select permission for Module Def doctype to All role --- frappe/core/doctype/module_def/module_def.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/module_def/module_def.json b/frappe/core/doctype/module_def/module_def.json index 7a8bfd76a7..4de046bbb6 100644 --- a/frappe/core/doctype/module_def/module_def.json +++ b/frappe/core/doctype/module_def/module_def.json @@ -55,7 +55,7 @@ "link_fieldname": "module" } ], - "modified": "2020-08-06 12:39:30.740379", + "modified": "2021-06-02 13:04:53.118716", "modified_by": "Administrator", "module": "Core", "name": "Module Def", @@ -69,6 +69,7 @@ "read": 1, "report": 1, "role": "Administrator", + "select": 1, "share": 1, "write": 1 }, @@ -78,7 +79,14 @@ "read": 1, "report": 1, "role": "System Manager", + "select": 1, "write": 1 + }, + { + "read": 1, + "report": 1, + "role": "All", + "select": 1 } ], "show_name_in_global_search": 1, From a457501422e06b22293ab2bfd19d8c2a8f509ace Mon Sep 17 00:00:00 2001 From: walstanb Date: Wed, 2 Jun 2021 16:20:51 +0530 Subject: [PATCH 29/29] fix: removed duplicate fields from Customize Form --- .../doctype/customize_form/customize_form.json | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 1807678673..b9dde88126 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -288,16 +288,6 @@ "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name" - }, - { - "fieldname": "default_email_template", - "fieldtype": "Link", - "label": "Default Email Template", - "options": "Email Template" - }, - { - "fieldname": "column_break_26", - "fieldtype": "Column Break" } ], "hide_toolbar": 1, @@ -306,7 +296,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-04-29 21:21:06.476372", + "modified": "2021-06-02 06:49:16.782806", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form",