diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 1742e813c6..1c7655528c 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -3,6 +3,8 @@ name: Server on: pull_request: workflow_dispatch: + push: + branches: [ develop ] jobs: test: diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index d9ccb07da0..f2f43f10f8 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -3,6 +3,8 @@ name: UI on: pull_request: workflow_dispatch: + push: + branches: [ develop ] jobs: test: diff --git a/README.md b/README.md index e00bea7857..11343a632a 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,21 @@
- - - - - - + + + + + + + + + - - - + + +
diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index ecf0d49511..5154adb634 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -343,12 +343,7 @@ async function write_assets_json(metafile) { } } - let assets_json_path = path.resolve( - assets_path, - "frappe", - "dist", - "assets.json" - ); + let assets_json_path = path.resolve(assets_path, "assets.json"); let assets_json; try { assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); diff --git a/frappe/__init__.py b/frappe/__init__.py index 9b208f7c2d..5793b224a3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1693,6 +1693,23 @@ def safe_eval(code, eval_globals=None, eval_locals=None): "round": round } + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + # Format Attributes + "format", "format_map", + } + + for attribute in UNSAFE_ATTRIBUTES: + if attribute in code: + throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) + if '__' in code: throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) diff --git a/frappe/api.py b/frappe/api.py index 9039ae0e5f..6427cbfbd8 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -11,6 +11,7 @@ import frappe.client import frappe.handler from frappe import _ from frappe.utils.response import build_response +from frappe.utils.data import sbool def handle(): @@ -108,25 +109,40 @@ def handle(): elif doctype: if frappe.local.request.method == "GET": - if frappe.local.form_dict.get('fields'): - frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields']) - frappe.local.form_dict.setdefault('limit_page_length', 20) - frappe.local.response.update({ - "data": frappe.call( - frappe.client.get_list, - doctype, - **frappe.local.form_dict - ) - }) + # set fields for frappe.get_list + if frappe.local.form_dict.get("fields"): + frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"]) + + # set limit of records for frappe.get_list + frappe.local.form_dict.setdefault( + "limit_page_length", + frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20, + ) + + # convert strings to native types - only as_dict and debug accept bool + for param in ["as_dict", "debug"]: + param_val = frappe.local.form_dict.get(param) + if param_val is not None: + frappe.local.form_dict[param] = sbool(param_val) + + # evaluate frappe.get_list + data = frappe.call(frappe.client.get_list, doctype, **frappe.local.form_dict) + + # set frappe.get_list result to response + frappe.local.response.update({"data": data}) if frappe.local.request.method == "POST": + # fetch data from from dict data = get_request_form_data() - data.update({ - "doctype": doctype - }) - frappe.local.response.update({ - "data": frappe.get_doc(data).insert().as_dict() - }) + data.update({"doctype": doctype}) + + # insert document from request data + doc = frappe.get_doc(data).insert() + + # set response data + frappe.local.response.update({"data": doc.as_dict()}) + + # commit for POST requests frappe.db.commit() else: raise frappe.DoesNotExistError diff --git a/frappe/build.py b/frappe/build.py index c970ae3a28..1df42ca2e6 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -50,7 +50,7 @@ def build_missing_files(): development = frappe.local.conf.developer_mode or frappe.local.dev_server build_mode = "development" if development else "production" - assets_json = frappe.read_file(frappe.get_app_path('frappe', 'public', 'dist', 'assets.json')) + assets_json = frappe.read_file("assets/assets.json") if assets_json: assets_json = frappe.parse_json(assets_json) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 5ebf714645..9879807033 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -21,9 +21,11 @@ from frappe.automation.doctype.assignment_rule.assignment_rule import apply as a exclude_from_linked_with = True class Communication(Document): + """Communication represents an external communication like Email. + """ no_feed_on_delete = True + DOCTYPE = 'Communication' - """Communication represents an external communication like Email.""" def onload(self): """create email flag queue""" if self.communication_type == "Communication" and self.communication_medium == "Email" \ @@ -149,6 +151,23 @@ class Communication(Document): self.email_status = "Spam" + @classmethod + def find(cls, name, ignore_error=False): + try: + return frappe.get_doc(cls.DOCTYPE, name) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @classmethod + def find_one_by_filters(cls, *, order_by=None, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs, order_by=order_by) + return cls.find(name) if name else None + + def update_db(self, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + def set_sender_full_name(self): if not self.sender_full_name and self.sender: if self.sender == "Administrator": @@ -485,4 +504,4 @@ def set_avg_response_time(parent, communication): response_times.append(response_time) if response_times: avg_response_time = sum(response_times) / len(response_times) - parent.db_set("avg_response_time", avg_response_time) \ No newline at end of file + parent.db_set("avg_response_time", avg_response_time) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 079bdaa09c..216db53c72 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -91,7 +91,7 @@ frappe.ui.form.on('Data Import', { if (frm.doc.status.includes('Success')) { frm.add_custom_button( - __('Go to {0} List', [frm.doc.reference_doctype]), + __('Go to {0} List', [__(frm.doc.reference_doctype)]), () => frappe.set_route('List', frm.doc.reference_doctype) ); } diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 1a173f7252..b4d3fb9a89 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -33,11 +33,11 @@ frappe.ui.form.on('DocType', { if (!frm.is_new() && !frm.doc.istable) { if (frm.doc.issingle) { - frm.add_custom_button(__('Go to {0}', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } else { - frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => { + frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.js b/frappe/core/doctype/document_naming_rule/document_naming_rule.js index 56b5c2fdf4..097a4e9a6e 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.js +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.js @@ -4,6 +4,7 @@ frappe.ui.form.on('Document Naming Rule', { refresh: function(frm) { frm.trigger('document_type'); + if (!frm.doc.__islocal) frm.trigger("add_update_counter_button"); }, document_type: (frm) => { // update the select field options with fieldnames @@ -20,5 +21,44 @@ frappe.ui.form.on('Document Naming Rule', { ); }); } + }, + add_update_counter_button: (frm) => { + frm.add_custom_button(__('Update Counter'), function() { + + const fields = [{ + fieldtype: 'Data', + fieldname: 'new_counter', + label: __('New Counter'), + default: frm.doc.counter, + reqd: 1, + description: __('Warning: Updating counter may lead to document name conflicts if not done properly') + }]; + + let primary_action_label = __('Save'); + + let primary_action = (fields) => { + frappe.call({ + method: 'frappe.core.doctype.document_naming_rule.document_naming_rule.update_current', + args: { + name: frm.doc.name, + new_counter: fields.new_counter + }, + callback: function() { + frm.set_value("counter", fields.new_counter); + dialog.hide(); + } + }); + }; + + const dialog = new frappe.ui.Dialog({ + title: __('Update Counter Value for Prefix: {0}', [frm.doc.prefix]), + fields, + primary_action_label, + primary_action + }); + + dialog.show(); + + }); } }); diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 4b34293af6..653c056caa 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -30,3 +30,8 @@ class DocumentNamingRule(Document): counter = frappe.db.get_value(self.doctype, self.name, 'counter', for_update=True) or 0 doc.name = self.prefix + ('%0'+str(self.prefix_digits)+'d') % (counter + 1) frappe.db.set_value(self.doctype, self.name, 'counter', counter + 1) + +@frappe.whitelist() +def update_current(name, new_counter): + frappe.only_for('System Manager') + frappe.db.set_value('Document Naming Rule', name, 'counter', new_counter) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index d9d8ae196e..4e00456f0d 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -117,7 +117,7 @@ frappe.ui.form.on("Customize Form", { frappe.customize_form.set_primary_action(frm); frm.add_custom_button( - __("Go to {0} List", [frm.doc.doc_type]), + __("Go to {0} List", [__(frm.doc.doc_type)]), function() { frappe.set_route("List", frm.doc.doc_type); }, diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index 6f1cd8eebd..91ca518e67 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -245,6 +245,7 @@ def send_monthly(): def make_links(columns, data): for row in data: + doc_name = row.get('name') for col in columns: if col.fieldtype == "Link" and col.options != "Currency": if col.options and row.get(col.fieldname): @@ -253,8 +254,9 @@ def make_links(columns, data): if col.options and row.get(col.fieldname) and row.get(col.options): row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname]) elif col.fieldtype == "Currency" and row.get(col.fieldname): - row[col.fieldname] = frappe.format_value(row[col.fieldname], col) - + doc = frappe.get_doc(col.parent, doc_name) if doc_name else None + # Pass the Document to get the currency based on docfield option + row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data def update_field_types(columns): @@ -262,4 +264,4 @@ def update_field_types(columns): if col.fieldtype in ("Link", "Dynamic Link", "Currency") and col.options != "Currency": col.fieldtype = "Data" col.options = "" - return columns \ No newline at end of file + return columns diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 36b662bb39..6ee106bb12 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -19,7 +19,7 @@ from frappe.utils import (validate_email_address, cint, cstr, get_datetime, from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, Email +from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from poplib import error_proto from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta @@ -430,89 +430,76 @@ class EmailAccount(Document): def receive(self, test_mails=None): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" - def get_seen(status): - if not status: - return None - seen = 1 if status == "SEEN" else 0 - return seen + exceptions = [] + inbound_mails = self.get_inbound_mails(test_mails=test_mails) + for mail in inbound_mails: + try: + communication = mail.process() + frappe.db.commit() + # If email already exists in the system + # then do not send notifications for the same email. + if communication and mail.flags.is_new_communication: + # notify all participants of this thread + if self.enable_auto_reply: + self.send_auto_reply(communication, mail) - if self.enable_incoming: - uid_list = [] - exceptions = [] - seen_status = [] - uid_reindexed = False - email_server = None + attachments = [] + if hasattr(communication, '_attachments'): + attachments = [d.file_name for d in communication._attachments] + communication.notify(attachments=attachments, fetched_from_email_account=True) + except SentEmailInInboxError: + frappe.db.rollback() + except Exception: + frappe.db.rollback() + frappe.log_error('email_account.receive') + if self.use_imap: + self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback()) + exceptions.append(frappe.get_traceback()) - if frappe.local.flags.in_test: - incoming_mails = test_mails or [] - else: - email_sync_rule = self.build_email_sync_rule() + #notify if user is linked to account + if len(inbound_mails)>0 and not frappe.local.flags.in_test: + frappe.publish_realtime('new_email', + {"account":self.email_account_name, "number":len(inbound_mails)} + ) - try: - email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) - except Exception: - frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + if exceptions: + raise Exception(frappe.as_json(exceptions)) - if not email_server: - return + def get_inbound_mails(self, test_mails=None): + """retrive and return inbound mails. - emails = email_server.get_messages() - if not emails: - return + """ + if frappe.local.flags.in_test: + return [InboundMail(msg, self) for msg in test_mails or []] - incoming_mails = emails.get("latest_messages", []) - uid_list = emails.get("uid_list", []) - seen_status = emails.get("seen_status", []) - uid_reindexed = emails.get("uid_reindexed", False) + if not self.enable_incoming: + return [] - for idx, msg in enumerate(incoming_mails): - uid = None if not uid_list else uid_list[idx] - self.flags.notify = True + email_sync_rule = self.build_email_sync_rule() + try: + email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) + messages = email_server.get_messages() or {} + except Exception: + raise + frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) + return [] - try: - args = { - "uid": uid, - "seen": None if not seen_status else get_seen(seen_status.get(uid, None)), - "uid_reindexed": uid_reindexed - } - communication = self.insert_communication(msg, args=args) + mails = [] + for index, message in enumerate(messages.get("latest_messages", [])): + uid = messages['uid_list'][index] + seen_status = 1 if messages['seen_status'][uid]=='SEEN' else 0 + mails.append(InboundMail(message, self, uid, seen_status)) - except SentEmailInInbox: - frappe.db.rollback() + return mails - except Exception: - frappe.db.rollback() - frappe.log_error('email_account.receive') - if self.use_imap: - self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback()) - exceptions.append(frappe.get_traceback()) - - else: - frappe.db.commit() - if communication and self.flags.notify: - - # If email already exists in the system - # then do not send notifications for the same email. - - attachments = [] - - if hasattr(communication, '_attachments'): - attachments = [d.file_name for d in communication._attachments] - - communication.notify(attachments=attachments, fetched_from_email_account=True) - - #notify if user is linked to account - if len(incoming_mails)>0 and not frappe.local.flags.in_test: - frappe.publish_realtime('new_email', {"account":self.email_account_name, "number":len(incoming_mails)}) - - if exceptions: - raise Exception(frappe.as_json(exceptions)) - - def handle_bad_emails(self, email_server, uid, raw, reason): - if email_server and cint(email_server.settings.use_imap): + def handle_bad_emails(self, uid, raw, reason): + if cint(self.use_imap): import email try: - mail = email.message_from_string(raw) + if isinstance(raw, bytes): + mail = email.message_from_bytes(raw) + else: + mail = email.message_from_string(raw) message_id = mail.get('Message-ID') except Exception: @@ -524,275 +511,18 @@ class EmailAccount(Document): "reason":reason, "message_id": message_id, "doctype": "Unhandled Email", - "email_account": email_server.settings.email_account + "email_account": self.name }) unhandled_email.insert(ignore_permissions=True) frappe.db.commit() - def insert_communication(self, msg, args=None): - if isinstance(msg, list): - raw, uid, seen = msg - else: - raw = msg - uid = -1 - seen = 0 - if isinstance(args, dict): - if args.get("uid", -1): uid = args.get("uid", -1) - if args.get("seen", 0): seen = args.get("seen", 0) - - email = Email(raw) - - if email.from_email == self.email_id and not email.mail.get("Reply-To"): - # gmail shows sent emails in inbox - # and we don't want emails sent by us to be pulled back into the system again - # dont count emails sent by the system get those - if frappe.flags.in_test: - print('WARN: Cannot pull email. Sender sames as recipient inbox') - raise SentEmailInInbox - - if email.message_id: - # https://stackoverflow.com/a/18367248 - names = frappe.db.sql("""SELECT DISTINCT `name`, `creation` FROM `tabCommunication` - WHERE `message_id`='{message_id}' - ORDER BY `creation` DESC LIMIT 1""".format( - message_id=email.message_id - ), as_dict=True) - - if names: - name = names[0].get("name") - # email is already available update communication uid instead - frappe.db.set_value("Communication", name, "uid", uid, update_modified=False) - - self.flags.notify = False - - return frappe.get_doc("Communication", name) - - if email.content_type == 'text/html': - email.content = clean_email_html(email.content) - - communication = frappe.get_doc({ - "doctype": "Communication", - "subject": email.subject, - "content": email.content, - 'text_content': email.text_content, - "sent_or_received": "Received", - "sender_full_name": email.from_real_name, - "sender": email.from_email, - "recipients": email.mail.get("To"), - "cc": email.mail.get("CC"), - "email_account": self.name, - "communication_medium": "Email", - "uid": int(uid or -1), - "message_id": email.message_id, - "communication_date": email.date, - "has_attachment": 1 if email.attachments else 0, - "seen": seen or 0 - }) - - self.set_thread(communication, email) - if communication.seen: - # get email account user and set communication as seen - users = frappe.get_all("User Email", filters={ "email_account": self.name }, - fields=["parent"]) - users = list(set([ user.get("parent") for user in users ])) - communication._seen = json.dumps(users) - - communication.flags.in_receive = True - communication.insert(ignore_permissions=True) - - # save attachments - communication._attachments = email.save_attachments_in_doc(communication) - - # replace inline images - dirty = False - for file in communication._attachments: - if file.name in email.cid_map and email.cid_map[file.name]: - dirty = True - - email.content = email.content.replace("cid:{0}".format(email.cid_map[file.name]), - file.file_url) - - if dirty: - # not sure if using save() will trigger anything - communication.db_set("content", sanitize_html(email.content)) - - # notify all participants of this thread - if self.enable_auto_reply and getattr(communication, "is_first", False): - self.send_auto_reply(communication, email) - - return communication - - def set_thread(self, communication, email): - """Appends communication to parent based on thread ID. Will extract - parent communication and will link the communication to the reference of that - communication. Also set the status of parent transaction to Open or Replied. - - If no thread id is found and `append_to` is set for the email account, - it will create a new parent transaction (e.g. Issue)""" - parent = None - - parent = self.find_parent_from_in_reply_to(communication, email) - - if not parent and self.append_to: - self.set_sender_field_and_subject_field() - - if not parent and self.append_to: - parent = self.find_parent_based_on_subject_and_sender(communication, email) - - if not parent and self.append_to and self.append_to!="Communication": - parent = self.create_new_parent(communication, email) - - if parent: - communication.reference_doctype = parent.doctype - communication.reference_name = parent.name - - # check if message is notification and disable notifications for this message - isnotification = email.mail.get("isnotification") - if isnotification: - if "notification" in isnotification: - communication.unread_notification_sent = 1 - - def set_sender_field_and_subject_field(self): - '''Identify the sender and subject fields from the `append_to` DocType''' - # set subject_field and sender_field - meta = frappe.get_meta(self.append_to) - self.subject_field = None - self.sender_field = None - - if hasattr(meta, "subject_field"): - self.subject_field = meta.subject_field - - if hasattr(meta, "sender_field"): - self.sender_field = meta.sender_field - - def find_parent_based_on_subject_and_sender(self, communication, email): - '''Find parent document based on subject and sender match''' - parent = None - - if self.append_to and self.sender_field: - if self.subject_field: - if '#' in email.subject: - # try and match if ID is found - # document ID is appended to subject - # example "Re: Your email (#OPP-2020-2334343)" - parent_id = email.subject.rsplit('#', 1)[-1].strip(' ()') - if parent_id: - parent = frappe.db.get_all(self.append_to, filters = dict(name = parent_id), - fields = 'name') - - if not parent: - # try and match by subject and sender - # if sent by same sender with same subject, - # append it to old coversation - subject = frappe.as_unicode(strip(re.sub(r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*", - "", email.subject, 0, flags=re.IGNORECASE))) - - parent = frappe.db.get_all(self.append_to, filters={ - self.sender_field: email.from_email, - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - if not parent and len(subject) > 10 and is_system_user(email.from_email): - # match only subject field - # when the from_email is of a user in the system - # and subject is atleast 10 chars long - parent = frappe.db.get_all(self.append_to, filters={ - self.subject_field: ("like", "%{0}%".format(subject)), - "creation": (">", (get_datetime() - relativedelta(days=60)).strftime(DATE_FORMAT)) - }, fields = "name", limit = 1) - - - - if parent: - parent = frappe._dict(doctype=self.append_to, name=parent[0].name) - return parent - - def create_new_parent(self, communication, email): - '''If no parent found, create a new reference document''' - - # no parent found, but must be tagged - # insert parent type doc - parent = frappe.new_doc(self.append_to) - - if self.subject_field: - parent.set(self.subject_field, frappe.as_unicode(email.subject)[:140]) - - if self.sender_field: - parent.set(self.sender_field, frappe.as_unicode(email.from_email)) - - if parent.meta.has_field("email_account"): - parent.email_account = self.name - - parent.flags.ignore_mandatory = True - - try: - parent.insert(ignore_permissions=True) - except frappe.DuplicateEntryError: - # try and find matching parent - parent_name = frappe.db.get_value(self.append_to, {self.sender_field: email.from_email}) - if parent_name: - parent.name = parent_name - else: - parent = None - - # NOTE if parent isn't found and there's no subject match, it is likely that it is a new conversation thread and hence is_first = True - communication.is_first = True - - return parent - - def find_parent_from_in_reply_to(self, communication, email): - '''Returns parent reference if embedded in In-Reply-To header - - Message-ID is formatted as `{message_id}@{site}`''' - parent = None - in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>") - - if in_reply_to: - if "@{0}".format(frappe.local.site) in in_reply_to: - # reply to a communication sent from the system - email_queue = frappe.db.get_value('Email Queue', dict(message_id=in_reply_to), ['communication','reference_doctype', 'reference_name']) - if email_queue: - parent_communication, parent_doctype, parent_name = email_queue - if parent_communication: - communication.in_reply_to = parent_communication - else: - reference, domain = in_reply_to.split("@", 1) - parent_doctype, parent_name = 'Communication', reference - - if frappe.db.exists(parent_doctype, parent_name): - parent = frappe._dict(doctype=parent_doctype, name=parent_name) - - # set in_reply_to of current communication - if parent_doctype=='Communication': - # communication.in_reply_to = email_queue.communication - - if parent.reference_name: - # the true parent is the communication parent - parent = frappe.get_doc(parent.reference_doctype, - parent.reference_name) - else: - comm = frappe.db.get_value('Communication', - dict( - message_id=in_reply_to, - creation=['>=', add_days(get_datetime(), -30)]), - ['reference_doctype', 'reference_name'], as_dict=1) - if comm: - parent = frappe._dict(doctype=comm.reference_doctype, name=comm.reference_name) - - return parent - def send_auto_reply(self, communication, email): """Send auto reply if set.""" from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts - if self.enable_auto_reply: set_incoming_outgoing_accounts(communication) - if self.send_unsubscribe_message: - unsubscribe_message = _("Leave this conversation") - else: - unsubscribe_message = "" + unsubscribe_message = (self.send_unsubscribe_message and _("Leave this conversation")) or "" frappe.sendmail(recipients = [email.from_email], sender = self.email_id, diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index f87ee32bb1..35cacac45a 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -1,45 +1,56 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # See license.txt -from __future__ import unicode_literals -import frappe, os -import unittest, email +import os +import email +import unittest +from datetime import datetime, timedelta +from frappe.email.receive import InboundMail, SentEmailInInboxError, Email +from frappe.email.email_body import get_message_id +import frappe from frappe.test_runner import make_test_records +from frappe.core.doctype.communication.email import make +from frappe.desk.form.load import get_attachments +from frappe.email.doctype.email_account.email_account import notify_unreplied make_test_records("User") make_test_records("Email Account") -from frappe.core.doctype.communication.email import make -from frappe.desk.form.load import get_attachments -from frappe.email.doctype.email_account.email_account import notify_unreplied -from datetime import datetime, timedelta + class TestEmailAccount(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + email_account.db_set("enable_auto_reply", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + def setUp(self): frappe.flags.mute_emails = False frappe.flags.sent_mail = None - - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 1) frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabUnhandled Email`') - def tearDown(self): - email_account = frappe.get_doc("Email Account", "_Test Email Account 1") - email_account.db_set("enable_incoming", 0) + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() def test_incoming(self): cleanup("test_sender@example.com") - with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-1.raw"), "r") as f: - test_mails = [f.read()] + test_mails = [self.get_test_mail('incoming-1.raw')] email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) self.assertTrue("test_receiver@example.com" in comm.recipients) - # check if todo is created self.assertTrue(frappe.db.get_value(comm.reference_doctype, comm.reference_name, "name")) @@ -88,7 +99,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_incoming_attached_email_from_outlook_layers(self): @@ -101,7 +112,7 @@ class TestEmailAccount(unittest.TestCase): email_account.receive(test_mails=test_mails) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: \"Microsoft Outlook\" <test_sender@example.com>" in comm.content) + self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) def test_outgoing(self): @@ -166,7 +177,6 @@ class TestEmailAccount(unittest.TestCase): comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, fields=["name", "reference_doctype", "reference_name"]) - # both communications attached to the same reference self.assertEqual(comm_list[0].reference_doctype, comm_list[1].reference_doctype) self.assertEqual(comm_list[0].reference_name, comm_list[1].reference_name) @@ -199,6 +209,215 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(comm_list[0].reference_doctype, event.doctype) self.assertEqual(comm_list[0].reference_name, event.name) + def test_auto_reply(self): + cleanup("test_sender@example.com") + + test_mails = [self.get_test_mail('incoming-1.raw')] + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.receive(test_mails=test_mails) + + comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) + self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, + "reference_name": comm.reference_name})) + + def test_handle_bad_emails(self): + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).mail.get('Message-ID') + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing") + self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id})) + +class TestInboundMail(unittest.TestCase): + @classmethod + def setUpClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 1) + + @classmethod + def tearDownClass(cls): + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.db_set("enable_incoming", 0) + + def setUp(self): + cleanup() + frappe.db.sql('delete from `tabEmail Queue`') + frappe.db.sql('delete from `tabToDo`') + + def get_test_mail(self, fname): + with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f: + return f.read() + + def new_doc(self, doctype, **data): + doc = frappe.new_doc(doctype) + for field, value in data.items(): + setattr(doc, field, value) + doc.insert() + return doc + + def new_communication(self, **kwargs): + defaults = { + 'subject': "Test Subject" + } + d = {**defaults, **kwargs} + return self.new_doc('Communication', **d) + + def new_email_queue(self, **kwargs): + defaults = { + 'message_id': get_message_id().strip(" <>") + } + d = {**defaults, **kwargs} + return self.new_doc('Email Queue', **d) + + def new_todo(self, **kwargs): + defaults = { + 'description': "Description" + } + d = {**defaults, **kwargs} + return self.new_doc('ToDo', **d) + + def test_self_sent_mail(self): + """Check that we raise SentEmailInInboxError if the inbound mail is self sent mail. + """ + mail_content = self.get_test_mail(fname="incoming-self-sent.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 1, 1) + with self.assertRaises(SentEmailInInboxError): + inbound_mail.process() + + def test_mail_exist_validation(self): + """Do not create communication record if the mail is already downloaded into the system. + """ + mail_content = self.get_test_mail(fname="incoming-1.raw") + message_id = Email(mail_content).message_id + # Create new communication record in DB + communication = self.new_communication(message_id=message_id) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + new_communiction = inbound_mail.process() + + # Make sure that uid is changed to new uid + self.assertEqual(new_communiction.uid, 12345) + self.assertEqual(communication.name, new_communiction.name) + + def test_find_parent_email_queue(self): + """If the mail is reply to the already sent mail, there will be a email queue record. + """ + # Create email queue record + queue_record = self.new_email_queue() + + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_queue = inbound_mail.parent_email_queue() + self.assertEqual(queue_record.name, parent_queue.name) + + def test_find_parent_communication_through_queue(self): + """Find parent communication of an inbound mail. + Cases where parent communication does exist: + 1. No parent communication is the mail is not a reply. + + Cases where parent communication does not exist: + 2. If mail is not a reply to system sent mail, then there can exist co + """ + # Create email queue record + communication = self.new_communication() + queue_record = self.new_email_queue(communication=communication.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_for_self_reply(self): + """If the inbound email is a reply but not reply to system sent mail. + + Ex: User replied to his/her mail. + """ + message_id = "new-message-id" + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertFalse(parent_communication) + + communication = self.new_communication(message_id=message_id) + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_find_parent_communication_from_header(self): + """Incase of header contains parent communication name + """ + communication = self.new_communication() + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", f"<{communication.name}@{frappe.local.site}>" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + parent_communication = inbound_mail.parent_communication() + self.assertEqual(parent_communication.name, communication.name) + + def test_reference_document(self): + # Create email queue record + todo = self.new_todo() + # communication = self.new_communication(reference_doctype='ToDo', reference_name=todo.name) + queue_record = self.new_email_queue(reference_doctype='ToDo', reference_name=todo.name) + mail_content = self.get_test_mail(fname="reply-4.raw").replace( + "{{ message_id }}", queue_record.message_id + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_record_name_in_subject(self): + # Create email queue record + todo = self.new_todo() + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: (#{todo.name})" + ) + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_reference_document_by_subject_match(self): + subject = "New todo" + todo = self.new_todo(sender='test_sender@example.com', description=subject) + + mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( + "{{ subject }}", f"RE: {subject}" + ) + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + reference_doc = inbound_mail.reference_document() + self.assertEqual(todo.name, reference_doc.name) + + def test_create_communication_from_mail(self): + # Create email queue record + mail_content = self.get_test_mail(fname="incoming-2.raw") + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + inbound_mail = InboundMail(mail_content, email_account, 12345, 1) + communication = inbound_mail.process() + self.assertTrue(communication.is_first) + self.assertTrue(communication._attachments) + def cleanup(sender=None): filters = {} if sender: @@ -207,4 +426,4 @@ def cleanup(sender=None): names = frappe.get_list("Communication", filters=filters, fields=["name"]) for name in names: frappe.delete_doc_if_exists("Communication", name.name) - frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) \ No newline at end of file + frappe.delete_doc_if_exists("Communication Link", {"parent": name.name}) diff --git a/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw new file mode 100644 index 0000000000..a16eecccd5 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-self-sent.raw @@ -0,0 +1,91 @@ +Delivered-To: test_receiver@example.com +Received: by 10.96.153.227 with SMTP id vj3csp416144qdb; + Mon, 15 Sep 2014 03:35:07 -0700 (PDT) +X-Received: by 10.66.119.103 with SMTP id kt7mr36981968pab.95.1410777306321; + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Return-Path: +Received: from mail-pa0-x230.google.com (mail-pa0-x230.google.com [2607:f8b0:400e:c03::230]) + by mx.google.com with ESMTPS id dg10si22178346pdb.115.2014.09.15.03.35.06 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +Received-SPF: pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) client-ip=2607:f8b0:400e:c03::230; +Authentication-Results: mx.google.com; + spf=pass (google.com: domain of test@example.com designates 2607:f8b0:400e:c03::230 as permitted sender) smtp.mail=test@example.com; + dkim=pass header.i=@gmail.com; + dmarc=pass (p=NONE dis=NONE) header.from=gmail.com +Received: by mail-pa0-f48.google.com with SMTP id hz1so6118714pad.21 + for ; Mon, 15 Sep 2014 03:35:06 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=gmail.com; s=20120113; + h=from:content-type:subject:message-id:date:to:mime-version; + bh=rwiLijtF3lfy9M6cP/7dv2Hm7NJuBwFZn1OFsN8Tlvs=; + b=x7U4Ny3Kz2ULRJ7a04NDBrBTVhP2ImIB9n3LVNGQDnDonPUM5Ro/wZcxPTVnBWZ2L1 + o1bGfP+lhBrvYUlHsd5r4FYC0Uvpad6hbzLr0DGUQgPTxW4cGKbtDEAq+BR2JWd9f803 + vdjSWdGk8w2dt2qbngTqIZkm5U2XWjICDOAYuPIseLUgCFwi9lLyOSARFB7mjAa2YL7Q + Nswk7mbWU1hbnHP6jaBb0m8QanTc7Up944HpNDRxIrB1ZHgKzYhXtx8nhnOx588ZGIAe + E6tyG8IwogR11vLkkrBhtMaOme9PohYx4F1CSTiwspmDCadEzJFGRe//lEXKmZHAYH6g + 90Zg== +X-Received: by 10.70.38.135 with SMTP id g7mr22078275pdk.100.1410777305744; + Mon, 15 Sep 2014 03:35:05 -0700 (PDT) +Return-Path: +Received: from [192.168.0.100] ([27.106.4.70]) + by mx.google.com with ESMTPSA id zr6sm11025126pbc.50.2014.09.15.03.35.02 + for + (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); + Mon, 15 Sep 2014 03:35:04 -0700 (PDT) +From: Rushabh Mehta +Content-Type: multipart/alternative; boundary="Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA" +Subject: test mail 🦄🌈😎 +Message-Id: <9143999C-8456-4399-9CF1-4A2DA9DD7711@gmail.com> +Date: Mon, 15 Sep 2014 16:04:57 +0530 +To: Rushabh Mehta +Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\)) +X-Mailer: Apple Mail (2.1878.6) + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +test mail + + + +@rushabh_mehta +https://erpnext.org + + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; + charset=us-ascii + +test = +mail
+



@rushabh_mehta
+
+
= + +--Apple-Mail=_57F71261-5C3A-43F6-918B-4438B96F61AA-- diff --git a/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw new file mode 100644 index 0000000000..35ddf06b01 --- /dev/null +++ b/frappe/email/doctype/email_account/test_mails/incoming-subject-placeholder.raw @@ -0,0 +1,183 @@ +Return-path: +Envelope-to: test_receiver@example.com +Delivery-date: Wed, 27 Jan 2016 16:24:20 +0800 +Received: from 23-59-23-10.perm.iinet.net.au ([23.59.23.10]:62191 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtp (Exim 4.86) + (envelope-from ) + id 1aOLOj-002xFL-CP + for test_receiver@example.com; Wed, 27 Jan 2016 16:24:20 +0800 +From: +To: +References: +In-Reply-To: +Subject: RE: {{ subject }} +Date: Wed, 27 Jan 2016 16:24:09 +0800 +Message-ID: <000001d158dc$1b8363a0$528a2ae0$@example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_NextPart_000_0001_01D1591F.29A7DC20" +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AQJZfZxrgcB9KnMqoZ+S4Qq9hcoSeZ3+vGiQ +Content-Language: en-au + +This is a multipart message in MIME format. + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: multipart/alternative; + boundary="----=_NextPart_001_0002_01D1591F.29A7DC20" + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Test purely for testing with the debugger has email attached + +=20 + +From: Notification [mailto:test_receiver@example.com]=20 +Sent: Wednesday, 27 January 2016 9:30 AM +To: test_receiver@example.com +Subject: Sales Invoice: SINV-12276 + +=20 + +test no 6 sent from bench to outlook to be replied to with messaging + + + + +------=_NextPart_001_0002_01D1591F.29A7DC20 +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +hi there

Test purely for testing with the debugger has email = +attached

 

From:= + = +Notification [mailto:test_receiver@example.com]
Sent: Wednesday, 27 = +January 2016 9:30 AM
To: = +test_receiver@example.com
Subject: Sales Invoice: = +SINV-12276

 

test no 3 sent from bench to outlook to be replied to with = +messaging

fizz buzz

This email was sent to test_receiver@example.= +com and copied to SuperUser

Leave this conversation = +

hi

+------=_NextPart_001_0002_01D1591F.29A7DC20-- + +------=_NextPart_000_0001_01D1591F.29A7DC20 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment + +Received: from 203-59-223-10.perm.iinet.net.au ([23.59.23.10]:49772 helo=DESKTOP7C66I2M) + by webcloud85.au.syrahost.com with esmtpsa (TLSv1.2:DHE-RSA-AES256-GCM-SHA384:256) + (Exim 4.86) + (envelope-from ) + id 1aOEtO-003tI4-Kv + for test_receiver@example.com; Wed, 27 Jan 2016 09:27:30 +0800 +Return-Path: +From: "Microsoft Outlook" +To: +Subject: Microsoft Outlook Test Message +MIME-Version: 1.0 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable +X-Mailer: Microsoft Outlook 14.0 +Thread-Index: AdFYoeN8x8wUI/+QSoCJkp33NKPVmw== + +This is an e-mail message sent automatically by Microsoft Outlook while = +testing the settings for your account. diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py index b19a134713..b1c1295f75 100755 --- a/frappe/email/doctype/email_group/email_group.py +++ b/frappe/email/doctype/email_group/email_group.py @@ -105,6 +105,6 @@ def send_welcome_email(welcome_email, email, email_group): email=email, email_group=email_group ) - - message = frappe.render_template(welcome_email.response, args) + email_message = welcome_email.response or welcome_email.response_html + message = frappe.render_template(email_message, args) frappe.sendmail(email, subject=welcome_email.subject, message=message) diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 9e2fe32250..394fe50468 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -45,6 +45,11 @@ class EmailQueue(Document): def find(cls, name): return frappe.get_doc(cls.DOCTYPE, name) + @classmethod + def find_one_by_filters(cls, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs) + return cls.find(name) if name else None + def update_db(self, commit=False, **kwargs): frappe.db.set_value(self.DOCTYPE, self.name, kwargs) if commit: diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 3b03c42b95..9f085aa94f 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -353,9 +353,7 @@ def add_attachment(fname, fcontent, content_type=None, def get_message_id(): '''Returns Message ID created from doctype and name''' - return "<{unique}@{site}>".format( - site=frappe.local.site, - unique=email.utils.make_msgid(random_string(10)).split('@')[0].split('<')[1]) + return email.utils.make_msgid(domain=frappe.local.site) def get_signature(email_account): if email_account and email_account.add_signature and email_account.signature: diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 6d60007cdb..7b3623df21 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -8,6 +8,7 @@ import imaplib import poplib import re import time +import json from email.header import decode_header import _socket @@ -20,13 +21,26 @@ from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file.file import (MaxFileSizeReachedError, get_random_filename) from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, - extract_email_id, markdown, now, parse_addr, strip) + extract_email_id, markdown, now, parse_addr, strip, get_datetime, + add_days, sanitize_html) +from frappe.utils.user import is_system_user +from frappe.utils.html_utils import clean_email_html + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 +imaplib._MAXLINE = 20480 class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass class TotalSizeExceededError(frappe.ValidationError): pass class LoginLimitExceeded(frappe.ValidationError): pass +class SentEmailInInboxError(Exception): + pass class EmailServer: """Wrapper for POP server to pull emails.""" @@ -100,14 +114,11 @@ class EmailServer: def get_messages(self): """Returns new email messages in a list.""" - if not self.check_mails(): - return # nothing to do + if not (self.check_mails() or self.connect()): + return [] frappe.db.commit() - if not self.connect(): - return - uid_list = [] try: @@ -116,7 +127,6 @@ class EmailServer: self.latest_messages = [] self.seen_status = {} self.uid_reindexed = False - uid_list = email_list = self.get_new_mails() if not email_list: @@ -132,11 +142,7 @@ class EmailServer: self.max_email_size = cint(frappe.local.conf.get("max_email_size")) self.max_total_size = 5 * self.max_email_size - for i, message_meta in enumerate(email_list): - # do not pull more than NUM emails - if (i+1) > num: - break - + for i, message_meta in enumerate(email_list[:num]): try: self.retrieve_message(message_meta, i+1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): @@ -152,7 +158,6 @@ class EmailServer: except Exception as e: if self.has_login_limit_exceeded(e): pass - else: raise @@ -369,6 +374,7 @@ class Email: else: self.mail = email.message_from_string(content) + self.raw_message = content self.text_content = '' self.html_content = '' self.attachments = [] @@ -391,6 +397,10 @@ class Email: if self.date > now(): self.date = now() + @property + def in_reply_to(self): + return (self.mail.get("In-Reply-To") or "").strip(" <>") + def parse(self): """Walk and process multi-part email.""" for part in self.mail.walk(): @@ -558,10 +568,330 @@ class Email: l = re.findall(r'(?<=\[)[\w/-]+', self.subject) return l and l[0] or None + def is_reply(self): + return bool(self.in_reply_to) -# fix due to a python bug in poplib that limits it to 2048 -poplib._MAXLINE = 20480 -imaplib._MAXLINE = 20480 +class InboundMail(Email): + """Class representation of incoming mail along with mail handlers. + """ + def __init__(self, content, email_account, uid=None, seen_status=None): + super().__init__(content) + self.email_account = email_account + self.uid = uid or -1 + self.seen_status = seen_status or 0 + + # System documents related to this mail + self._parent_email_queue = None + self._parent_communication = None + self._reference_document = None + + self.flags = frappe._dict() + + def get_content(self): + if self.content_type == 'text/html': + return clean_email_html(self.content) + + def process(self): + """Create communication record from email. + """ + if self.is_sender_same_as_receiver() and not self.is_reply(): + if frappe.flags.in_test: + print('WARN: Cannot pull email. Sender same as recipient inbox') + raise SentEmailInInboxError + + communication = self.is_exist_in_system() + if communication: + communication.update_db(uid=self.uid) + communication.reload() + return communication + + self.flags.is_new_communication = True + return self._build_communication_doc() + + def _build_communication_doc(self): + data = self.as_dict() + data['doctype'] = "Communication" + + if self.parent_communication(): + data['in_reply_to'] = self.parent_communication().name + + if self.reference_document(): + data['reference_doctype'] = self.reference_document().doctype + data['reference_name'] = self.reference_document().name + elif self.email_account.append_to and self.email_account.append_to != 'Communication': + reference_doc = self._create_reference_document(self.email_account.append_to) + if reference_doc: + data['reference_doctype'] = reference_doc.doctype + data['reference_name'] = reference_doc.name + data['is_first'] = True + + if self.is_notification(): + # Disable notifications for notification. + data['unread_notification_sent'] = 1 + + if self.seen_status: + data['_seen'] = json.dumps(self.get_users_linked_to_account(self.email_account)) + + communication = frappe.get_doc(data) + communication.flags.in_receive = True + communication.insert(ignore_permissions=True) + + # save attachments + communication._attachments = self.save_attachments_in_doc(communication) + communication.content = sanitize_html(self.replace_inline_images(communication._attachments)) + communication.save() + return communication + + def replace_inline_images(self, attachments): + # replace inline images + content = self.content + for file in attachments: + if file.name in self.cid_map and self.cid_map[file.name]: + content = content.replace("cid:{0}".format(self.cid_map[file.name]), + file.file_url) + return content + + def is_notification(self): + isnotification = self.mail.get("isnotification") + return isnotification and ("notification" in isnotification) + + def is_exist_in_system(self): + """Check if this email already exists in the system(as communication document). + """ + from frappe.core.doctype.communication.communication import Communication + if not self.message_id: + return + + return Communication.find_one_by_filters(message_id = self.message_id, + order_by = 'creation DESC') + + def is_sender_same_as_receiver(self): + return self.from_email == self.email_account.email_id + + def is_reply_to_system_sent_mail(self): + """Is it a reply to already sent mail. + """ + return self.is_reply() and frappe.local.site in self.in_reply_to + + def parent_email_queue(self): + """Get parent record from `Email Queue`. + + If it is a reply to already sent mail, then there will be a parent record in EMail Queue. + """ + from frappe.email.doctype.email_queue.email_queue import EmailQueue + + if self._parent_email_queue is not None: + return self._parent_email_queue + + parent_email_queue = '' + if self.is_reply_to_system_sent_mail(): + parent_email_queue = EmailQueue.find_one_by_filters(message_id=self.in_reply_to) + + self._parent_email_queue = parent_email_queue or '' + return self._parent_email_queue + + def parent_communication(self): + """Find a related communication so that we can prepare a mail thread. + + The way it happens is by using in-reply-to header, and we can't make thread if it does not exist. + + Here are the cases to handle: + 1. If mail is a reply to already sent mail, then we can get parent communicaion from + Email Queue record. + 2. Sometimes we send communication name in message-ID directly, use that to get parent communication. + 3. Sender sent a reply but reply is on top of what (s)he sent before, + then parent record exists directly in communication. + """ + from frappe.core.doctype.communication.communication import Communication + if self._parent_communication is not None: + return self._parent_communication + + if not self.is_reply(): + return '' + + if not self.is_reply_to_system_sent_mail(): + communication = Communication.find_one_by_filters(message_id=self.in_reply_to, + creation = ['>=', self.get_relative_dt(-30)]) + elif self.parent_email_queue() and self.parent_email_queue().communication: + communication = Communication.find(self.parent_email_queue().communication, ignore_error=True) + else: + reference = self.in_reply_to + if '@' in self.in_reply_to: + reference, _ = self.in_reply_to.split("@", 1) + communication = Communication.find(reference, ignore_error=True) + + self._parent_communication = communication or '' + return self._parent_communication + + def reference_document(self): + """Reference document is a document to which mail relate to. + + We can get reference document from Parent record(EmailQueue | Communication) if exists. + Otherwise we do subject match to find reference document if we know the reference(append_to) doctype. + """ + if self._reference_document is not None: + return self._reference_document + + reference_document = "" + parent = self.parent_email_queue() or self.parent_communication() + + if parent and parent.reference_doctype: + reference_doctype, reference_name = parent.reference_doctype, parent.reference_name + reference_document = self.get_doc(reference_doctype, reference_name, ignore_error=True) + + if not reference_document and self.email_account.append_to: + reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to) + + # if not reference_document: + # reference_document = Create_reference_document(self.email_account.append_to) + + self._reference_document = reference_document or '' + return self._reference_document + + def get_reference_name_from_subject(self): + """ + Ex: "Re: Your email (#OPP-2020-2334343)" + """ + return self.subject.rsplit('#', 1)[-1].strip(' ()') + + def match_record_by_subject_and_sender(self, doctype): + """Find a record in the given doctype that matches with email subject and sender. + + Cases: + 1. Sometimes record name is part of subject. We can get document by parsing name from subject + 2. Find by matching sender and subject + 3. Find by matching subject alone (Special case) + Ex: when a System User is using Outlook and replies to an email from their own client, + it reaches the Email Account with the threading info lost and the (sender + subject match) + doesn't work because the sender in the first communication was someone different to whom + the system user is replying to via the common email account in Frappe. This fix bypasses + the sender match when the sender is a system user and subject is atleast 10 chars long + (for additional safety) + + NOTE: We consider not to match by subject if match record is very old. + """ + name = self.get_reference_name_from_subject() + email_fields = self.get_email_fields(doctype) + + record = self.get_doc(doctype, name, ignore_error=True) if name else None + + if not record: + subject = self.clean_subject(self.subject) + filters = { + email_fields.subject_field: ("like", f"%{subject}%"), + "creation": (">", self.get_relative_dt(days=-60)) + } + + # Sender check is not needed incase mail is from system user. + if not (len(subject) > 10 and is_system_user(self.from_email)): + filters[email_fields.sender_field] = self.from_email + + name = frappe.db.get_value(self.email_account.append_to, filters = filters) + record = self.get_doc(doctype, name, ignore_error=True) if name else None + return record + + def _create_reference_document(self, doctype): + """ Create reference document if it does not exist in the system. + """ + parent = frappe.new_doc(doctype) + email_fileds = self.get_email_fields(doctype) + + if email_fileds.subject_field: + parent.set(email_fileds.subject_field, frappe.as_unicode(self.subject)[:140]) + + if email_fileds.sender_field: + parent.set(email_fileds.sender_field, frappe.as_unicode(self.from_email)) + + parent.flags.ignore_mandatory = True + + try: + parent.insert(ignore_permissions=True) + except frappe.DuplicateEntryError: + # try and find matching parent + parent_name = frappe.db.get_value(self.email_account.append_to, + {email_fileds.sender_field: email.from_email} + ) + if parent_name: + parent.name = parent_name + else: + parent = None + return parent + + + @staticmethod + def get_doc(doctype, docname, ignore_error=False): + try: + return frappe.get_doc(doctype, docname) + except frappe.DoesNotExistError: + if ignore_error: + return + raise + + @staticmethod + def get_relative_dt(days): + """Get relative to current datetime. Only relative days are supported. + """ + return add_days(get_datetime(), days) + + @staticmethod + def get_users_linked_to_account(email_account): + """Get list of users who linked to Email account. + """ + users = frappe.get_all("User Email", filters={"email_account": email_account.name}, + fields=["parent"]) + return list(set([user.get("parent") for user in users])) + + @staticmethod + def clean_subject(subject): + """Remove Prefixes like 'fw', FWD', 're' etc from subject. + """ + # Match strings like "fw:", "re :" etc. + regex = r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*" + return frappe.as_unicode(strip(re.sub(regex, "", subject, 0, flags=re.IGNORECASE))) + + @staticmethod + def get_email_fields(doctype): + """Returns Email related fields of a doctype. + """ + fields = frappe._dict() + + email_fields = ['subject_field', 'sender_field'] + meta = frappe.get_meta(doctype) + + for field in email_fields: + if hasattr(meta, field): + fields[field] = getattr(meta, field) + return fields + + @staticmethod + def get_document(self, doctype, name): + """Is same as frappe.get_doc but suppresses the DoesNotExist error. + """ + try: + return frappe.get_doc(doctype, name) + except frappe.DoesNotExistError: + return None + + def as_dict(self): + """ + """ + return { + "subject": self.subject, + "content": self.get_content(), + 'text_content': self.text_content, + "sent_or_received": "Received", + "sender_full_name": self.from_real_name, + "sender": self.from_email, + "recipients": self.mail.get("To"), + "cc": self.mail.get("CC"), + "email_account": self.email_account.name, + "communication_medium": "Email", + "uid": self.uid, + "message_id": self.message_id, + "communication_date": self.date, + "has_attachment": 1 if self.attachments else 0, + "seen": self.seen_status or 0 + } class TimerMixin(object): def __init__(self, *args, **kwargs): diff --git a/frappe/model/document.py b/frappe/model/document.py index 623916597e..a3f8ad0cfa 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -17,6 +17,7 @@ from frappe.model.workflow import set_workflow_state_on_action from frappe.utils.global_search import update_global_search from frappe.integrations.doctype.webhook import run_webhooks from frappe.desk.form.document_follow import follow_document +from frappe.desk.utils import slug from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event # once_only validation @@ -1202,8 +1203,8 @@ class Document(BaseDocument): doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield))) def get_url(self): - """Returns Desk URL for this document. `/app/Form/{doctype}/{name}`""" - return "/app/Form/{doctype}/{name}".format(doctype=self.doctype, name=self.name) + """Returns Desk URL for this document. `/app/{doctype}/{name}`""" + return f"/app/{slug(self.doctype)}/{self.name}" def add_comment(self, comment_type='Comment', text=None, comment_email=None, link_doctype=None, link_name=None, comment_by=None): """Add a comment to this document. diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index dee4839b34..03e6288856 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -72,8 +72,8 @@ frappe.data_import.DataExporter = class DataExporter { let child_fieldname = df.fieldname; let label = df.reqd ? // prettier-ignore - __('{0} ({1}) (1 row mandatory)', [df.label || df.fieldname, doctype]) - : __('{0} ({1})', [df.label || df.fieldname, doctype]); + __('{0} ({1}) (1 row mandatory)', [__(df.label || df.fieldname), __(doctype)]) + : __('{0} ({1})', [__(df.label || df.fieldname), __(doctype)]); return { label, fieldname: child_fieldname, diff --git a/frappe/public/js/frappe/form/controls/check.js b/frappe/public/js/frappe/form/controls/check.js index 658ef640a9..1209f52e6d 100644 --- a/frappe/public/js/frappe/form/controls/check.js +++ b/frappe/public/js/frappe/form/controls/check.js @@ -14,8 +14,10 @@ frappe.ui.form.ControlCheck = class ControlCheck extends frappe.ui.form.ControlD `).appendTo(this.parent); } set_input_areas() { - this.label_area = this.label_span = this.$wrapper.find(".label-area").get(0); this.input_area = this.$wrapper.find(".input-area").get(0); + if (this.only_input) return; + + this.label_area = this.label_span = this.$wrapper.find(".label-area").get(0); this.disp_area = this.$wrapper.find(".disp-area").get(0); } make_input() { diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index 43bd7443ab..af92f3b7a9 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -200,10 +200,11 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat if(frappe.model.can_create(doctype)) { // new item r.results.push({ - label: "" + html: "" + " " + __("Create a new {0}", [__(me.get_options())]) + "", + label: __("Create a new {0}", [__(me.get_options())]), value: "create_new__link_option", action: me.new_doc }); @@ -213,10 +214,11 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat if (locals && locals['DocType']) { // not applicable in web forms r.results.push({ - label: "" + html: "" + " " + __("Advanced Search") + "", + label: __("Advanced Search"), value: "advanced_search__link_option", action: me.open_advanced_search }); diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 88d7ceaa94..35ebf9274d 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -175,6 +175,7 @@ frappe.ui.form.Form = class FrappeForm { field && ["Link", "Dynamic Link"].includes(field.df.fieldtype) && field.validate && field.validate(value); me.layout.refresh_dependency(); + me.layout.refresh_sections(); let object = me.script_manager.trigger(fieldname, doc.doctype, doc.name); return object; } @@ -1068,7 +1069,7 @@ frappe.ui.form.Form = class FrappeForm { if(!this.doc.__islocal) { frappe.model.remove_from_locals(this.doctype, this.docname); - frappe.model.with_doc(this.doctype, this.docname, () => { + return frappe.model.with_doc(this.doctype, this.docname, () => { this.refresh(); }); } @@ -1078,6 +1079,7 @@ frappe.ui.form.Form = class FrappeForm { if (this.fields_dict[fname] && this.fields_dict[fname].refresh) { this.fields_dict[fname].refresh(); this.layout.refresh_dependency(); + this.layout.refresh_sections(); } } diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index dd7c339395..a77791d0a2 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -196,7 +196,7 @@ export default class Grid { tasks.push(() => { if (dirty) { this.refresh(); - this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); + this.frm && this.frm.script_manager.trigger(this.df.fieldname + "_delete", this.doctype); } }); @@ -345,6 +345,9 @@ export default class Grid { if (d.idx === undefined) { d.idx = ri + 1; } + if (d.name === undefined) { + d.name = "row " + d.idx; + } if (this.grid_rows[ri] && !append_row) { var grid_row = this.grid_rows[ri]; grid_row.doc = d; diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 7eea578f8c..33db00dede 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -529,7 +529,7 @@ export default class GridRow { // hide other var open_row = this.get_open_form(); - if (show===undefined) show = !!!open_row; + if (show === undefined) show = !open_row; // call blur document.activeElement && document.activeElement.blur(); @@ -594,19 +594,42 @@ export default class GridRow { this.wrapper.removeClass("grid-row-open"); } open_prev() { - const row_index = this.wrapper.index(); - if (this.grid.grid_rows[row_index - 1]) { - this.grid.grid_rows[row_index - 1].toggle_view(true); - } + if (!this.doc) return; + this.open_row_at_index(this.doc.idx - 2); } open_next() { - const row_index = this.wrapper.index(); - if (this.grid.grid_rows[row_index + 1]) { - this.grid.grid_rows[row_index + 1].toggle_view(true); - } else { + if (!this.doc) return; + + if (!this.open_row_at_index(this.doc.idx)) { this.grid.add_new_row(null, null, true); } } + open_row_at_index(row_index) { + if (!this.grid.data[row_index]) return; + + this.change_page_if_reqd(row_index); + this.grid.grid_rows[row_index].toggle_view(true); + return true; + } + change_page_if_reqd(row_index) { + const { + page_index, + page_length + } = this.grid.grid_pagination; + + row_index++; + let new_page; + + if (row_index <= (page_index - 1) * page_length) { + new_page = page_index - 1; + } else if (row_index > page_index * page_length) { + new_page = page_index + 1; + } + + if (new_page) { + this.grid.grid_pagination.go_to_page(new_page); + } + } refresh_field(fieldname, txt) { let df = this.docfields.find(col => { return col.fieldname === fieldname; diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index f65f1a83c2..e2aaec553d 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -724,9 +724,14 @@ frappe.views.CommunicationComposer = class { } message += await this.get_signature(); - if (this.real_name && !message.includes("")) { - message = `

${__('Dear')} ${this.real_name},

-
${message}`; + + const SALUTATION_END_COMMENT = ""; + if (this.real_name && !message.includes(SALUTATION_END_COMMENT)) { + this.message = ` +

${__('Dear')} ${this.real_name},

+ ${SALUTATION_END_COMMENT}
+ ${message} + `; } if (this.is_a_reply) { diff --git a/frappe/public/scss/common/awesomeplete.scss b/frappe/public/scss/common/awesomeplete.scss index 096da1e2fd..619d6a9e3e 100644 --- a/frappe/public/scss/common/awesomeplete.scss +++ b/frappe/public/scss/common/awesomeplete.scss @@ -31,6 +31,7 @@ margin: 0; padding: var(--padding-xs); z-index: 1; + min-width: 250px; &> li { cursor: pointer; diff --git a/frappe/public/scss/common/buttons.scss b/frappe/public/scss/common/buttons.scss index 591dc5bba6..de3a4cfc20 100644 --- a/frappe/public/scss/common/buttons.scss +++ b/frappe/public/scss/common/buttons.scss @@ -48,13 +48,13 @@ $active-border: darken($primary-light, 12.5%) ); - color: var(--blue-500); + color: var(--primary); &:hover, &:active { - color: var(--blue-500); + color: var(--primary); } &:focus { - box-shadow: 0 0 0 0.2rem var(--blue-50) + box-shadow: 0 0 0 0.2rem var(--primary-light); } } @@ -77,11 +77,11 @@ } .btn.btn-primary { - background-color: var(--primary-color); + background-color: var(--primary); color: var(--white); white-space: nowrap; --icon-stroke: currentColor; - --icon-fill-bg: var(--primary-color); + --icon-fill-bg: var(--primary); } .btn.btn-danger { diff --git a/frappe/public/scss/common/grid.scss b/frappe/public/scss/common/grid.scss index 3cc5139d9e..aac949b1bf 100644 --- a/frappe/public/scss/common/grid.scss +++ b/frappe/public/scss/common/grid.scss @@ -140,7 +140,7 @@ .checkbox { margin: 0px; text-align: center; - margin-top: 9px; + margin-top: var(--padding-sm); } textarea { diff --git a/frappe/public/scss/website/css_variables.scss b/frappe/public/scss/website/css_variables.scss index 2aeb842802..463a0db49e 100644 --- a/frappe/public/scss/website/css_variables.scss +++ b/frappe/public/scss/website/css_variables.scss @@ -28,4 +28,6 @@ --font-size-4xl: #{$font-size-4xl}; --font-size-5xl: #{$font-size-5xl}; --font-size-6xl: #{$font-size-6xl}; + + --card-border-radius: #{$card-border-radius}; } diff --git a/frappe/public/scss/website/sidebar.scss b/frappe/public/scss/website/sidebar.scss index b13eaf2a74..76956c9136 100644 --- a/frappe/public/scss/website/sidebar.scss +++ b/frappe/public/scss/website/sidebar.scss @@ -10,7 +10,7 @@ margin-top: 0.25rem; border-radius: 0.375rem; font-size: $font-size-sm; - color: $gray-600; + color: var(--text-color); text-decoration: none; font-weight: 500; @include transition(); @@ -26,8 +26,8 @@ } .sidebar-item a.active { - color: $primary; - background-color: $primary-light; + color: var(--primary); + background-color: var(--primary-light); } .sidebar-item-icon { diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index 6453062877..7e77aab779 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -1,178 +1,170 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt -from __future__ import unicode_literals - -import unittest, frappe, os -from frappe.core.doctype.user.user import generate_keys -from frappe.frappeclient import FrappeClient, FrappeException -from frappe.utils.data import get_url +import unittest +from random import choice import requests -import base64 +from semantic_version import Version -class TestAPI(unittest.TestCase): - def test_insert_many(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')") +import frappe +from frappe.utils import get_site_url + + +def maintain_state(f): + def wrapper(*args, **kwargs): + frappe.db.rollback() + r = f(*args, **kwargs) frappe.db.commit() + return r - server.insert_many([ - {"doctype": "Note", "public": True, "title": "Sing"}, - {"doctype": "Note", "public": True, "title": "a"}, - {"doctype": "Note", "public": True, "title": "song"}, - {"doctype": "Note", "public": True, "title": "of"}, - {"doctype": "Note", "public": True, "title": "sixpence"}, - ]) + return wrapper - self.assertTrue(frappe.db.get_value('Note', {'title': 'Sing'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'a'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'song'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'of'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'sixpence'})) - def test_create_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title = 'test_create'") - frappe.db.commit() +class TestResourceAPI(unittest.TestCase): + SITE_URL = get_site_url(frappe.local.site) + RESOURCE_URL = f"{SITE_URL}/api/resource" + DOCTYPE = "ToDo" + GENERATED_DOCUMENTS = [] - server.insert({"doctype": "Note", "public": True, "title": "test_create"}) + @classmethod + @maintain_state + def setUpClass(self): + for _ in range(10): + doc = frappe.get_doc( + {"doctype": "ToDo", "description": frappe.mock("paragraph")} + ).insert() + self.GENERATED_DOCUMENTS.append(doc.name) - self.assertTrue(frappe.db.get_value('Note', {'title': 'test_create'})) + @classmethod + @maintain_state + def tearDownClass(self): + for name in self.GENERATED_DOCUMENTS: + frappe.delete_doc_if_exists(self.DOCTYPE, name) - def test_list_docs(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - doc_list = server.get_list("Note") + @property + def sid(self): + if not getattr(self, "_sid", None): + self._sid = requests.post( + f"{self.SITE_URL}/api/method/login", + data={ + "usr": "Administrator", + "pwd": frappe.conf.admin_password or "admin", + }, + ).cookies.get("sid") - self.assertTrue(len(doc_list)) + return self._sid - def test_get_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title = 'get_this'") - frappe.db.commit() + def get(self, path, params=""): + return requests.get(f"{self.RESOURCE_URL}/{path}?sid={self.sid}{params}") - server.insert_many([ - {"doctype": "Note", "public": True, "title": "get_this"}, - ]) - doc = server.get_doc("Note", "get_this") - self.assertTrue(doc) - - def test_get_value(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title = 'get_value'") - frappe.db.commit() - - test_content = "test get value" - - server.insert_many([ - {"doctype": "Note", "public": True, "title": "get_value", "content": test_content}, - ]) - self.assertEqual(server.get_value("Note", "content", {"title": "get_value"}).get('content'), test_content) - name = server.get_value("Note", "name", {"title": "get_value"}).get('name') - - # test by name - self.assertEqual(server.get_value("Note", "content", name).get('content'), test_content) - - self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"}) - - def test_get_single(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix') - self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix') - self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix') - frappe.db.set_value('Website Settings', None, 'title_prefix', '') - - def test_update_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title in ('Sing','sing')") - frappe.db.commit() - - server.insert({"doctype":"Note", "public": True, "title": "Sing"}) - doc = server.get_doc("Note", 'Sing') - changed_title = "sing" - doc["title"] = changed_title - doc = server.update(doc) - self.assertTrue(doc["title"] == changed_title) - - def test_update_child_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabContact` where first_name = 'George' and last_name = 'Steevens'") - frappe.db.sql("delete from `tabContact` where first_name = 'William' and last_name = 'Shakespeare'") - frappe.db.sql("delete from `tabCommunication` where reference_doctype = 'Event'") - frappe.db.sql("delete from `tabCommunication Link` where link_doctype = 'Contact'") - frappe.db.sql("delete from `tabEvent` where subject = 'Sing a song of sixpence'") - frappe.db.sql("delete from `tabEvent Participants` where reference_doctype = 'Contact'") - frappe.db.commit() - - # create multiple contacts - server.insert_many([ - {"doctype": "Contact", "first_name": "George", "last_name": "Steevens"}, - {"doctype": "Contact", "first_name": "William", "last_name": "Shakespeare"} - ]) - - # create an event with one of the created contacts - event = server.insert({ - "doctype": "Event", - "subject": "Sing a song of sixpence", - "event_participants": [{ - "reference_doctype": "Contact", - "reference_docname": "George Steevens" - }] - }) - - # update the event's contact to the second contact - server.update({ - "doctype": "Event Participants", - "name": event.get("event_participants")[0].get("name"), - "reference_docname": "William Shakespeare" - }) - - # the change should run the parent document's validations and - # create a Communication record with the new contact - self.assertTrue(frappe.db.exists("Communication Link", {"link_name": "William Shakespeare"})) - - def test_delete_doc(self): - server = FrappeClient(get_url(), "Administrator", "admin", verify=False) - frappe.db.sql("delete from `tabNote` where title = 'delete'") - frappe.db.commit() - - server.insert_many([ - {"doctype": "Note", "public": True, "title": "delete"}, - ]) - server.delete("Note", "delete") - - self.assertFalse(frappe.db.get_value('Note', {'title': 'delete'})) - - def test_auth_via_api_key_secret(self): - # generate API key and API secret for administrator - keys = generate_keys("Administrator") - frappe.db.commit() - generated_secret = frappe.utils.password.get_decrypted_password( - "User", "Administrator", fieldname='api_secret' + def post(self, path, data): + return requests.post( + f"{self.RESOURCE_URL}/{path}?sid={self.sid}", data=frappe.as_json(data) ) - api_key = frappe.db.get_value("User", "Administrator", "api_key") - header = {"Authorization": "token {}:{}".format(api_key, generated_secret)} - res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) + def put(self, path, data): + return requests.put( + f"{self.RESOURCE_URL}/{path}?sid={self.sid}", data=frappe.as_json(data) + ) - self.assertEqual(res.status_code, 200) - self.assertEqual("Administrator", res.json()["message"]) - self.assertEqual(keys['api_secret'], generated_secret) + def delete(self, path): + return requests.delete(f"{self.RESOURCE_URL}/{path}?sid={self.sid}") - header = {"Authorization": "Basic {}".format(base64.b64encode(frappe.safe_encode("{}:{}".format(api_key, generated_secret))).decode())} - res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) - self.assertEqual(res.status_code, 200) - self.assertEqual("Administrator", res.json()["message"]) + def test_unauthorized_call(self): + # test 1: fetch documents without auth + response = requests.get(f"{self.RESOURCE_URL}/{self.DOCTYPE}") + self.assertEqual(response.status_code, 403) - # Valid api key, invalid api secret - api_secret = "ksk&93nxoe3os" - header = {"Authorization": "token {}:{}".format(api_key, api_secret)} - res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) - self.assertEqual(res.status_code, 403) + def test_get_list(self): + # test 2: fetch documents without params + response = self.get(self.DOCTYPE) + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response.json(), dict) + self.assertIn("data", response.json()) + + def test_get_list_limit(self): + # test 3: fetch data with limit + response = self.get(self.DOCTYPE, "&limit=2") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()["data"]), 2) + + def test_get_list_dict(self): + # test 4: fetch response as (not) dict + response = self.get(self.DOCTYPE, "&as_dict=True") + json = frappe._dict(response.json()) + self.assertEqual(response.status_code, 200) + self.assertIsInstance(json.data, list) + self.assertIsInstance(json.data[0], dict) + + response = self.get(self.DOCTYPE, "&as_dict=False") + json = frappe._dict(response.json()) + self.assertEqual(response.status_code, 200) + self.assertIsInstance(json.data, list) + self.assertIsInstance(json.data[0], list) + + def test_get_list_debug(self): + # test 5: fetch response with debug + response = self.get(self.DOCTYPE, "&debug=true") + self.assertEqual(response.status_code, 200) + self.assertIn("exc", response.json()) + self.assertIsInstance(response.json()["exc"], str) + self.assertIsInstance(eval(response.json()["exc"]), list) + + def test_get_list_fields(self): + # test 6: fetch response with fields + response = self.get(self.DOCTYPE, r'&fields=["description"]') + self.assertEqual(response.status_code, 200) + json = frappe._dict(response.json()) + self.assertIn("description", json.data[0]) + + def test_create_document(self): + # test 7: POST method on /api/resource to create doc + data = {"description": frappe.mock("paragraph")} + response = self.post(self.DOCTYPE, data) + self.assertEqual(response.status_code, 200) + docname = response.json()["data"]["name"] + self.assertIsInstance(docname, str) + self.GENERATED_DOCUMENTS.append(docname) + + def test_update_document(self): + # test 8: PUT method on /api/resource to update doc + generated_desc = frappe.mock("paragraph") + data = {"description": generated_desc} + random_doc = choice(self.GENERATED_DOCUMENTS) + desc_before_update = frappe.db.get_value(self.DOCTYPE, random_doc, "description") + + response = self.put(f"{self.DOCTYPE}/{random_doc}", data=data) + self.assertEqual(response.status_code, 200) + self.assertNotEqual(response.json()["data"]["description"], desc_before_update) + self.assertEqual(response.json()["data"]["description"], generated_desc) + + def test_delete_document(self): + # test 9: DELETE method on /api/resource + doc_to_delete = choice(self.GENERATED_DOCUMENTS) + response = self.delete(f"{self.DOCTYPE}/{doc_to_delete}") + self.assertEqual(response.status_code, 202) + self.assertDictEqual(response.json(), {"message": "ok"}) + + non_existent_doc = frappe.generate_hash(length=12) + response = self.delete(f"{self.DOCTYPE}/{non_existent_doc}") + self.assertEqual(response.status_code, 404) + self.assertDictEqual(response.json(), {}) - # random api key and api secret - api_key = "@3djdk3kld" - api_secret = "ksk&93nxoe3os" - header = {"Authorization": "token {}:{}".format(api_key, api_secret)} - res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) - self.assertEqual(res.status_code, 401) +class TestMethodAPI(unittest.TestCase): + METHOD_URL = f"{get_site_url(frappe.local.site)}/api/method" + + def test_version(self): + # test 1: test for /api/method/version + response = requests.get(f"{self.METHOD_URL}/version") + json = frappe._dict(response.json()) + + self.assertEqual(response.status_code, 200) + self.assertIsInstance(json, dict) + self.assertIsInstance(json.message, str) + self.assertEqual(Version(json.message), Version(frappe.__version__)) + + def test_ping(self): + # test 2: test for /api/method/ping + response = requests.get(f"{self.METHOD_URL}/ping") + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response.json(), dict) + self.assertEqual(response.json()['message'], "pong") diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index 8340db4852..74962dd84c 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -1,8 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - import unittest, frappe, re, email from six import PY3 @@ -178,7 +176,8 @@ class TestEmail(unittest.TestCase): frappe.db.sql('''delete from `tabCommunication` where sender = 'sukh@yyy.com' ''') with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: - communication = email_account.insert_communication(raw.read()) + mails = email_account.get_inbound_mails(test_mails=[raw.read()]) + communication = mails[0].process() self.assertTrue(re.search(''']*src=["']/private/files/rtco1.png[^>]*>''', communication.content)) self.assertTrue(re.search(''']*src=["']/private/files/rtco2.png[^>]*>''', communication.content)) diff --git a/frappe/tests/test_frappe_client.py b/frappe/tests/test_frappe_client.py new file mode 100644 index 0000000000..e1cdbb6ccd --- /dev/null +++ b/frappe/tests/test_frappe_client.py @@ -0,0 +1,177 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import unittest, frappe +from frappe.core.doctype.user.user import generate_keys +from frappe.frappeclient import FrappeClient, FrappeException +from frappe.utils.data import get_url + +import requests +import base64 + +class TestFrappeClient(unittest.TestCase): + def test_insert_many(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title in ('Sing','a','song','of','sixpence')") + frappe.db.commit() + + server.insert_many([ + {"doctype": "Note", "public": True, "title": "Sing"}, + {"doctype": "Note", "public": True, "title": "a"}, + {"doctype": "Note", "public": True, "title": "song"}, + {"doctype": "Note", "public": True, "title": "of"}, + {"doctype": "Note", "public": True, "title": "sixpence"}, + ]) + + self.assertTrue(frappe.db.get_value('Note', {'title': 'Sing'})) + self.assertTrue(frappe.db.get_value('Note', {'title': 'a'})) + self.assertTrue(frappe.db.get_value('Note', {'title': 'song'})) + self.assertTrue(frappe.db.get_value('Note', {'title': 'of'})) + self.assertTrue(frappe.db.get_value('Note', {'title': 'sixpence'})) + + def test_create_doc(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title = 'test_create'") + frappe.db.commit() + + server.insert({"doctype": "Note", "public": True, "title": "test_create"}) + + self.assertTrue(frappe.db.get_value('Note', {'title': 'test_create'})) + + def test_list_docs(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + doc_list = server.get_list("Note") + + self.assertTrue(len(doc_list)) + + def test_get_doc(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title = 'get_this'") + frappe.db.commit() + + server.insert_many([ + {"doctype": "Note", "public": True, "title": "get_this"}, + ]) + doc = server.get_doc("Note", "get_this") + self.assertTrue(doc) + + def test_get_value(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title = 'get_value'") + frappe.db.commit() + + test_content = "test get value" + + server.insert_many([ + {"doctype": "Note", "public": True, "title": "get_value", "content": test_content}, + ]) + self.assertEqual(server.get_value("Note", "content", {"title": "get_value"}).get('content'), test_content) + name = server.get_value("Note", "name", {"title": "get_value"}).get('name') + + # test by name + self.assertEqual(server.get_value("Note", "content", name).get('content'), test_content) + + self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"}) + + def test_get_single(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix') + self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix') + self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix') + frappe.db.set_value('Website Settings', None, 'title_prefix', '') + + def test_update_doc(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title in ('Sing','sing')") + frappe.db.commit() + + server.insert({"doctype":"Note", "public": True, "title": "Sing"}) + doc = server.get_doc("Note", 'Sing') + changed_title = "sing" + doc["title"] = changed_title + doc = server.update(doc) + self.assertTrue(doc["title"] == changed_title) + + def test_update_child_doc(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabContact` where first_name = 'George' and last_name = 'Steevens'") + frappe.db.sql("delete from `tabContact` where first_name = 'William' and last_name = 'Shakespeare'") + frappe.db.sql("delete from `tabCommunication` where reference_doctype = 'Event'") + frappe.db.sql("delete from `tabCommunication Link` where link_doctype = 'Contact'") + frappe.db.sql("delete from `tabEvent` where subject = 'Sing a song of sixpence'") + frappe.db.sql("delete from `tabEvent Participants` where reference_doctype = 'Contact'") + frappe.db.commit() + + # create multiple contacts + server.insert_many([ + {"doctype": "Contact", "first_name": "George", "last_name": "Steevens"}, + {"doctype": "Contact", "first_name": "William", "last_name": "Shakespeare"} + ]) + + # create an event with one of the created contacts + event = server.insert({ + "doctype": "Event", + "subject": "Sing a song of sixpence", + "event_participants": [{ + "reference_doctype": "Contact", + "reference_docname": "George Steevens" + }] + }) + + # update the event's contact to the second contact + server.update({ + "doctype": "Event Participants", + "name": event.get("event_participants")[0].get("name"), + "reference_docname": "William Shakespeare" + }) + + # the change should run the parent document's validations and + # create a Communication record with the new contact + self.assertTrue(frappe.db.exists("Communication Link", {"link_name": "William Shakespeare"})) + + def test_delete_doc(self): + server = FrappeClient(get_url(), "Administrator", "admin", verify=False) + frappe.db.sql("delete from `tabNote` where title = 'delete'") + frappe.db.commit() + + server.insert_many([ + {"doctype": "Note", "public": True, "title": "delete"}, + ]) + server.delete("Note", "delete") + + self.assertFalse(frappe.db.get_value('Note', {'title': 'delete'})) + + def test_auth_via_api_key_secret(self): + # generate API key and API secret for administrator + keys = generate_keys("Administrator") + frappe.db.commit() + generated_secret = frappe.utils.password.get_decrypted_password( + "User", "Administrator", fieldname='api_secret' + ) + + api_key = frappe.db.get_value("User", "Administrator", "api_key") + header = {"Authorization": "token {}:{}".format(api_key, generated_secret)} + res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) + + self.assertEqual(res.status_code, 200) + self.assertEqual("Administrator", res.json()["message"]) + self.assertEqual(keys['api_secret'], generated_secret) + + header = {"Authorization": "Basic {}".format(base64.b64encode(frappe.safe_encode("{}:{}".format(api_key, generated_secret))).decode())} + res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) + self.assertEqual(res.status_code, 200) + self.assertEqual("Administrator", res.json()["message"]) + + # Valid api key, invalid api secret + api_secret = "ksk&93nxoe3os" + header = {"Authorization": "token {}:{}".format(api_key, api_secret)} + res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) + self.assertEqual(res.status_code, 403) + + + # random api key and api secret + api_key = "@3djdk3kld" + api_secret = "ksk&93nxoe3os" + header = {"Authorization": "token {}:{}".format(api_key, api_secret)} + res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) + self.assertEqual(res.status_code, 401) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 436dcba028..1470616ca9 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -809,7 +809,7 @@ def get_assets_json(): assets_json = None if not assets_json: - assets_json = frappe.read_file("assets/frappe/dist/assets.json") + assets_json = frappe.read_file("assets/assets.json") cache.set_value("assets_json", assets_json, shared=True) frappe.local.assets_json = frappe.safe_decode(assets_json) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 9cbac2a570..a9af30ab2c 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -622,6 +622,26 @@ def ceil(s): def cstr(s, encoding='utf-8'): return frappe.as_unicode(s, encoding) +def sbool(x): + """Converts str object to Boolean if possible. + Example: + "true" becomes True + "1" becomes True + "{}" remains "{}" + + Args: + x (str): String to be converted to Bool + + Returns: + object: Returns Boolean or type(x) + """ + from distutils.util import strtobool + + try: + return bool(strtobool(x)) + except Exception: + return x + def rounded(num, precision=0): """round method for round halfs to nearest even algorithm aka banker's rounding - compatible with python3""" precision = cint(precision) diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 347d52dc57..2881a21aff 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -2,7 +2,6 @@ # MIT License. See license.txt from __future__ import unicode_literals -from frappe.utils.jinja import get_jenv def resolve_class(classes): @@ -22,6 +21,8 @@ def resolve_class(classes): def inspect(var, render=True): + from frappe.utils.jinja import get_jenv + context = {"var": var} if render: html = "
{{ var | pprint | e }}
" diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 643812b226..632fc5df23 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -61,7 +61,9 @@ def get_safe_globals(): out = NamespaceDict( # make available limited methods of frappe - json=json, + json=NamespaceDict( + loads = json.loads, + dumps = json.dumps), dict=dict, log=frappe.log, _dict=frappe._dict, @@ -148,6 +150,7 @@ def get_safe_globals(): # default writer allows write access out._write_ = _write out._getitem_ = _getitem + out._getattr_ = _getattr # allow iterators and list comprehension out._getiter_ = iter @@ -174,6 +177,27 @@ def _getitem(obj, key): raise SyntaxError('Key starts with _') return obj[key] +def _getattr(object, name, default=None): + # guard function for RestrictedPython + # allow any key to be accessed as long as + # 1. it does not start with an underscore (safer_getattr) + # 2. it is not an UNSAFE_ATTRIBUTES + + UNSAFE_ATTRIBUTES = { + # Generator Attributes + "gi_frame", "gi_code", + # Coroutine Attributes + "cr_frame", "cr_code", "cr_origin", + # Async Generator Attributes + "ag_code", "ag_frame", + # Traceback Attributes + "tb_frame", "tb_next", + } + + if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES): + raise SyntaxError("{name} is an unsafe attribute".format(name=name)) + return RestrictedPython.Guards.safer_getattr(object, name, default=default) + def _write(obj): # guard function for RestrictedPython # allow writing to any object diff --git a/frappe/website/doctype/website_theme/website_theme_template.scss b/frappe/website/doctype/website_theme/website_theme_template.scss index fbd640690b..34cd66a4fe 100644 --- a/frappe/website/doctype/website_theme/website_theme_template.scss +++ b/frappe/website/doctype/website_theme/website_theme_template.scss @@ -1,13 +1,13 @@ {% if google_font %} @import url("https://fonts.googleapis.com/css2?family={{ google_font.replace(' ', '+') }}:{{ font_properties }}&display=swap"); $font-family-sans-serif: "{{ google_font }}", -apple-system, BlinkMacSystemFont, - "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", - "Droid Sans", "Helvetica Neue", sans-serif; + "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", + "Droid Sans", "Helvetica Neue", sans-serif; {% endif -%} {% if primary_color %}$primary: {{ frappe.db.get_value('Color', primary_color, 'color') }};{% endif -%} {% if dark_color %}$dark: {{ frappe.db.get_value('Color', dark_color, 'color') }};{% endif -%} -{% if text_color %}$body-color: {{ frappe.db.get_value('Color', text_color, 'color') }};{% endif -%} +{% if text_color %}$body-text-color: {{ frappe.db.get_value('Color', text_color, 'color') }};{% endif -%} {% if background_color %}$body-bg: {{ frappe.db.get_value('Color', background_color, 'color') }};{% endif -%} $enable-shadows: {{ button_shadows and "true" or "false" }}; @@ -24,9 +24,24 @@ $enable-rounded: {{ button_rounded_corners and "true" or "false" }}; {% if font_size -%} body { - font-size: {{ font_size }}; + font-size: {{ font_size }}; } {%- endif %} // Custom Theme {{ custom_scss or '' }} + +:root { + {% if primary_color %} + --primary: #{$primary}; + --primary-color: #{$primary}; + {% endif -%} + {% if background_color %} + --bg-color: #{$body-bg}; + {% endif -%} + {% if text_color %} + --text-color: #{$body-text-color}; + --text-light: #{$body-text-color}; + {% endif -%} +} + diff --git a/yarn.lock b/yarn.lock index 86719d81f4..298c424b72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7608,9 +7608,9 @@ write-file-atomic@^3.0.0: typedarray-to-buffer "^3.1.5" ws@~7.4.2: - version "7.4.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.2.tgz#782100048e54eb36fe9843363ab1c68672b261dd" - integrity sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA== + version "7.4.6" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" + integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== xdg-basedir@^4.0.0: version "4.0.0"