From 07a577af867fe7c4c2486106a7e559b5e9c9717a Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 27 May 2022 11:56:34 +0530 Subject: [PATCH] feat: google oauth for google emails * used unique constraint on email_id in Email Account Doctype --- frappe/core/doctype/communication/mixins.py | 4 +- frappe/core/doctype/user/user.py | 10 +-- .../doctype/email_account/email_account.js | 17 +++++ .../doctype/email_account/email_account.json | 36 ++++++++- .../doctype/email_account/email_account.py | 74 ++++++++++++++----- frappe/email/receive.py | 27 ++++++- frappe/email/smtp.py | 31 +++++++- frappe/email/utils.py | 58 +++++++++++++++ frappe/integrations/google_oauth.py | 21 ++++-- .../website_settings/website_settings.py | 1 - 10 files changed, 233 insertions(+), 46 deletions(-) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 695b8bebae..ad74b47026 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -92,7 +92,7 @@ class CommunicationEmailMixin: cc_list = self.mail_cc( is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender ) - return [self.get_email_with_displayname(email) for email in cc_list] + return [self.get_email_with_displayname(email) for email in cc_list if email] def mail_bcc(self, is_inbound_mail_communcation=False): """ @@ -120,7 +120,7 @@ class CommunicationEmailMixin: def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) - return [self.get_email_with_displayname(email) for email in bcc_list] + return [self.get_email_with_displayname(email) for email in bcc_list if email] def mail_sender(self): email_account = self.get_outgoing_email_account() diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index d6fe883fae..eb3c11a4db 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -763,19 +763,11 @@ def has_email_account(email): @frappe.whitelist(allow_guest=False) def get_email_awaiting(user): - waiting = frappe.get_all( + return frappe.get_all( "User Email", fields=["email_account", "email_id"], filters={"awaiting_password": 1, "parent": user}, ) - if waiting: - return waiting - else: - user_email_table = DocType("User Email") - frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where( - user_email_table.parent == user - ).run() - return False def ask_pass_update(): diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index a357126a48..af6947c07c 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -1,6 +1,7 @@ frappe.email_defaults = { "GMail": { "email_server": "imap.gmail.com", + "incoming_port": 993, "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.gmail.com", @@ -142,6 +143,22 @@ frappe.ui.form.on("Email Account", { } }, + authorize_google_api_access: function(frm) { + frappe.call({ + method: "frappe.email.doctype.email_account.email_account.authorize_google_access", + args: { + "email_account": frm.doc.name, + "reauthorize": frm.doc.refresh_token ? 1 : 0 + }, + callback: function(r) { + if (!r.exc) { + frm.save(); + window.open(r.message.url); + } + } + }); + }, + email_id:function(frm) { //pull domain and if no matching domain go create one frm.events.update_domain(frm); diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 1e3e995669..34572c600d 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -18,6 +18,10 @@ "awaiting_password", "ascii_encode_password", "column_break_10", + "use_google_oauth", + "authorize_google_api_access", + "google_refresh_token", + "google_access_token", "login_id_is_different", "login_id", "mailbox_settings", @@ -79,7 +83,8 @@ "in_list_view": 1, "label": "Email Address", "options": "Email", - "reqd": 1 + "reqd": 1, + "unique": 1 }, { "default": "0", @@ -576,12 +581,39 @@ "fieldname": "section_break_25", "fieldtype": "Section Break", "label": "IMAP Details" + }, + { + "default": "0", + "depends_on": "eval: doc.service === \"GMail\"", + "fieldname": "use_google_oauth", + "fieldtype": "Check", + "label": "Use Google OAuth" + }, + { + "depends_on": "eval: doc.service === \"GMail\" && doc.use_google_oauth", + "fieldname": "authorize_google_api_access", + "fieldtype": "Button", + "label": "Authorize API Access" + }, + { + "fieldname": "google_refresh_token", + "fieldtype": "Data", + "hidden": 1, + "label": "Google Refresh Token", + "read_only": 1 + }, + { + "fieldname": "google_access_token", + "fieldtype": "Small Text", + "hidden": 1, + "label": "Google Access Token", + "read_only": 1 } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-11-30 09:03:25.728637", + "modified": "2022-05-29 18:11:06.463553", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 02afe4f4b5..f73aa02183 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -8,6 +8,7 @@ import socket import time from datetime import datetime, timedelta from poplib import error_proto +from urllib.parse import quote import frappe from frappe import _, are_emails_muted, safe_encode @@ -15,8 +16,16 @@ from frappe.desk.form import assign_to from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from frappe.email.smtp import SMTPServer from frappe.email.utils import get_port +from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document -from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address +from frappe.utils import ( + cint, + comma_or, + cstr, + get_request_site_address, + parse_addr, + validate_email_address, +) from frappe.utils.background_jobs import enqueue, get_jobs from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template @@ -63,6 +72,7 @@ class EmailAccount(Document): def validate(self): """Validate Email Address and check POP3/IMAP and SMTP connections is enabled.""" + if self.email_id: validate_email_address(self.email_id, True) @@ -76,25 +86,11 @@ class EmailAccount(Document): if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0: frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id))) - duplicate_email_account = frappe.get_all( - "Email Account", filters={"email_id": self.email_id, "name": ("!=", self.name)} - ) - if duplicate_email_account: - frappe.throw( - _("Email ID must be unique, Email Account already exists for {0}").format( - frappe.bold(self.email_id) - ) - ) - if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - if ( - not self.awaiting_password - and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch - ): - if self.password or self.smtp_server in ("127.0.0.1", "localhost"): + if not frappe.local.flags.in_install and (self.use_google_oauth or not self.awaiting_password): + if self.google_refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -103,7 +99,10 @@ class EmailAccount(Document): self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - frappe.throw(_("Password is required or select Awaiting Password")) + if self.use_google_oauth: + frappe.throw(_("Please Authorize Google by using `Authorize API access` button")) + else: + frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: if not self.send_notification_to: @@ -208,6 +207,9 @@ class EmailAccount(Document): "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, + "use_google_oauth": self.use_google_oauth or 0, + "google_refresh_token": getattr(self, "google_refresh_token", None), + "google_access_token": getattr(self, "google_access_token", None), } ) @@ -274,7 +276,9 @@ class EmailAccount(Document): @property def _password(self): - raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test) + raise_exception = not ( + self.use_google_oauth or self.no_smtp_authentication or frappe.flags.in_test + ) return self.get_password(raise_exception=raise_exception) @property @@ -405,12 +409,16 @@ class EmailAccount(Document): def sendmail_config(self): return { + "email_account": self.name, "server": self.smtp_server, "port": cint(self.smtp_port), "login": getattr(self, "login_id", None) or self.email_id, "password": self._password, "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), + "use_google_oauth": self.use_google_oauth or 0, + "google_refresh_token": getattr(self, "google_refresh_token", None), + "google_access_token": getattr(self, "google_access_token", None), } def get_smtp_server(self): @@ -491,6 +499,7 @@ class EmailAccount(Document): def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages["uid_list"][index] if messages.get("uid_list") else None + uid = uid.decode() if isinstance(uid, bytes) else uid seen_status = messages.get("seen_status", {}).get(uid) if self.email_sync_option != "UNSEEN" or seen_status != "SEEN": # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' @@ -771,14 +780,18 @@ def notify_unreplied(): def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" + if frappe.cache().get_value("workers:no-internet") == True: if test_internet(): frappe.cache().set_value("workers:no-internet", False) else: return + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] for email_account in frappe.get_list( - "Email Account", filters={"enable_incoming": 1, "awaiting_password": 0} + "Email Account", + filters={"enable_incoming": 1}, + or_filters={"awaiting_password": 0, "use_google_oauth": 1}, ): if now: pull_from_email_account(email_account.name) @@ -906,3 +919,24 @@ def set_email_password(email_account, user, password): return False return True + + +@frappe.whitelist(methods=["POST"]) +def authorize_google_access(email_account, reauthorize=False, code=None): + doctype = "Email Account" + refresh_token = frappe.db.get_value(doctype, email_account, "google_refresh_token") + oauth_obj = GoogleOAuth("mail") + + if not (refresh_token or code) or reauthorize: + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.email.doctype.email_account.email_account.authorize_google_access", + "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), + "email_account": email_account, + }, + ) + + res = oauth_obj.authorize(code, get_request_site_address(True)) + frappe.db.set_value(doctype, email_account, "google_refresh_token", res.get("refresh_token")) + frappe.db.set_value(doctype, email_account, "google_access_token", res.get("access_token")) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 93e1a68285..81ef7927c1 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -18,6 +18,7 @@ from email_reply_parser import EmailReplyParser import frappe from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file import MaxFileSizeReachedError, get_random_filename +from frappe.email.utils import connect_google_oauth from frappe.utils import ( add_days, cint, @@ -98,7 +99,18 @@ class EmailServer: self.imap = Timed_IMAP4( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - self.imap.login(self.settings.username, self.settings.password) + + if self.settings.use_google_oauth and self.settings.google_refresh_token: + connect_google_oauth( + self.imap, + self.settings.email_account, + self.settings.username, + self.settings.google_access_token, + self.settings.google_refresh_token, + ) + else: + self.imap.login(self.settings.username, self.settings.password) + # connection established! return True @@ -119,8 +131,17 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - self.pop.user(self.settings.username) - self.pop.pass_(self.settings.password) + if self.settings.use_google_oauth and self.settings.google_refresh_token: + connect_google_oauth( + self.pop, + self.settings.email_account, + self.settings.username, + self.settings.google_access_token, + self.settings.google_refresh_token, + ) + else: + self.pop.user(self.settings.username) + self.pop.pass_(self.settings.password) # connection established! return True diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 1211419de1..cd31aa0d5c 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -5,6 +5,7 @@ import smtplib import frappe from frappe import _ +from frappe.email.utils import connect_google_oauth from frappe.utils import cint, cstr @@ -43,13 +44,29 @@ def send(email, append_to=None, retry=1): class SMTPServer: - def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None): + def __init__( + self, + server, + login=None, + email_account=None, + password=None, + port=None, + use_tls=None, + use_ssl=None, + use_google_oauth=0, + google_refresh_token=None, + google_access_token=None, + ): self.login = login + self.email_account = email_account self.password = password self._server = server self._port = port self.use_tls = use_tls self.use_ssl = use_ssl + self.use_google_oauth = use_google_oauth + self.google_refresh_token = google_refresh_token + self.google_access_token = google_access_token self._session = None if not self.server: @@ -91,7 +108,17 @@ class SMTPServer: ) self.secure_session(_session) - if self.login and self.password: + + if self.use_google_oauth and self.google_refresh_token: + connect_google_oauth( + _session, + self.email_account, + self.login, + self.google_access_token, + self.google_refresh_token, + ) + + elif self.password: res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly diff --git a/frappe/email/utils.py b/frappe/email/utils.py index 147284a625..97cfaaa1e7 100644 --- a/frappe/email/utils.py +++ b/frappe/email/utils.py @@ -1,8 +1,13 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +import base64 import imaplib import poplib +import smtplib +from typing import Union +from frappe import _, db, log_error, throw +from frappe.integrations.google_oauth import GoogleAuthenticationError, GoogleOAuth from frappe.utils import cint @@ -15,3 +20,56 @@ def get_port(doc): doc.incoming_port = poplib.POP3_SSL_PORT if doc.use_ssl else poplib.POP3_PORT return cint(doc.incoming_port) + + +def connect_google_oauth( + connection_obj: Union[imaplib.IMAP4, poplib.POP3, smtplib.SMTP], + email_account: str, + email: str, + google_access_token: str, + google_refresh_token: str, + retry: int = 0, +) -> None: + auth_string = "user=%s\1auth=Bearer %s\1\1" % (email, google_access_token) + mechanism = "XOAUTH2" + _func = lambda x: auth_string # noqa: E731 + + try: + if isinstance(connection_obj, poplib.POP3): + # poplib doesn't have AUTH command implementation + res = connection_obj._shortcmd( + "AUTH {0} {1}".format(mechanism, base64.b64encode(bytes(auth_string, "utf-8")).decode("utf-8")) + ) + + if not res.startswith(b"+OK"): + raise + + elif isinstance(connection_obj, imaplib.IMAP4): + connection_obj.authenticate(mechanism, _func) + + else: + # SMTP + connection_obj.auth(mechanism, _func, initial_response_ok=False) + + except Exception: + # maybe the access token expired - refreshing + access_token = refresh_google_access_token(email_account, google_refresh_token) + + if not access_token or retry > 0: + throw( + _("Google Authentication Failed. Please Check and Update the credentials."), + GoogleAuthenticationError, + _("Google Authentication Error"), + ) + + connect_google_oauth( + connection_obj, email_account, email, access_token, google_refresh_token, retry + 1 + ) + + +def refresh_google_access_token(email_account: str, google_refresh_token: str) -> str: + oauth_obj = GoogleOAuth("mail") + res = oauth_obj.refresh_access_token(google_refresh_token) + db.set_value("Email Account", email_account, "google_access_token", res.get("access_token")) + + return res.get("access_token") diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index f8731a1e6a..da67f2ccf5 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -3,7 +3,7 @@ from typing import Dict, Union from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from requests import post, get +from requests import get, post import frappe @@ -12,22 +12,30 @@ _SCOPES = { "mail": ("https://mail.google.com/"), "contacts": ("https://www.googleapis.com/auth/contacts"), "drive": ("https://www.googleapis.com/auth/drive"), - "indexing": ("https://www.googleapis.com/auth/indexing") + "indexing": ("https://www.googleapis.com/auth/indexing"), } _SERVICES = { "contacts": ("people", "v1"), "drive": ("drive", "v3"), - "indexing": ("indexing", "v3") + "indexing": ("indexing", "v3"), } +class GoogleAuthenticationError(Exception): + pass + + class GoogleOAuth: OAUTH_URL = "https://oauth2.googleapis.com/token" - def __init__(self, domain: str, validate: bool=True): + def __init__(self, domain: str, validate: bool = True): self.google_settings = frappe.get_single("Google Settings") self.domain = domain.lower() - self.scopes = " ".join(_SCOPES[self.domain]) if isinstance(_SCOPES[self.domain], (list, tuple)) else _SCOPES[self.domain] + self.scopes = ( + " ".join(_SCOPES[self.domain]) + if isinstance(_SCOPES[self.domain], (list, tuple)) + else _SCOPES[self.domain] + ) if validate: self.validate_google_settings() @@ -132,8 +140,7 @@ def handle_response( def is_valid_access_token(access_token: str) -> bool: response = get( - "https://oauth2.googleapis.com/tokeninfo", - params={'access_token': access_token} + "https://oauth2.googleapis.com/tokeninfo", params={"access_token": access_token} ).json() if "error" in response: diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index 31e452c03f..bd634b4f32 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -95,7 +95,6 @@ class WebsiteSettings(Document): res = oauth_obj.refresh_access_token( self.get_password(fieldname="indexing_refresh_token", raise_exception=False) ) - print(res) return res.get("access_token")