From d3900485a61c45d34e1bffe7307f8dba42737607 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 01:03:57 +0530 Subject: [PATCH 001/201] refactor: Use mariadb client for creating connections Move to use MariaDB's official Python client written in C instead of the PyMySQL library. This change doesn't rid Frappe of the PyMySQL library. Instead, it continues to utilize it for the ER module and converter methods until the MariaDB library adds support for the same. Ticket: https://jira.mariadb.org/projects/CONPY/issues/CONPY-203 --- frappe/database/database.py | 1 - frappe/database/mariadb/database.py | 105 +++++++++--------- frappe/geo/utils.py | 5 +- .../patches/v12_0/delete_duplicate_indexes.py | 4 +- requirements.txt | 1 + 5 files changed, 55 insertions(+), 61 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 42135f3cd5..ab1d8ea4fa 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -183,7 +183,6 @@ class Database(object): except Exception as e: if self.is_syntax_error(e): - # only for mariadb frappe.errprint("Syntax error in query:") frappe.errprint(query) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 7505ef3a7f..57e9576143 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,8 +1,9 @@ from typing import List, Tuple, Union -import pymysql -from pymysql.constants import ER, FIELD_TYPE -from pymysql.converters import conversions, escape_string +import mariadb +from mariadb.constants import FIELD_TYPE +from pymysql.constants import ER +from pymysql.converters import escape_string import frappe from frappe.database.database import Database @@ -11,12 +12,12 @@ from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name class MariaDBDatabase(Database): - ProgrammingError = pymysql.err.ProgrammingError - TableMissingError = pymysql.err.ProgrammingError - OperationalError = pymysql.err.OperationalError - InternalError = pymysql.err.InternalError - SQLError = pymysql.err.ProgrammingError - DataError = pymysql.err.DataError + ProgrammingError = mariadb.ProgrammingError + TableMissingError = mariadb.ProgrammingError + OperationalError = mariadb.OperationalError + InternalError = mariadb.InternalError + SQLError = mariadb.ProgrammingError + DataError = mariadb.DataError REGEX_CHARACTER = "regexp" # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, @@ -68,42 +69,34 @@ class MariaDBDatabase(Database): } def get_connection(self): - usessl = 0 - if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: - usessl = 1 - ssl_params = { - "ca": frappe.conf.db_ssl_ca, - "cert": frappe.conf.db_ssl_cert, - "key": frappe.conf.db_ssl_key, - } - - conversions.update( - { + conn_settings = { + "host": self.host, + "user": self.user, + "password": self.password, + "database": self.user, + "converter": { FIELD_TYPE.NEWDECIMAL: float, FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[str], + UnicodeWithAttrs: escape_string, + }, + } + + if self.port: + conn_settings["port"] = int(self.port) + + if frappe.conf.local_infile: + conn_settings["local_infile"] = frappe.conf.local_infile + + if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: + ssl_params = { + "ssl": True, + "ssl_ca": frappe.conf.db_ssl_ca, + "ssl_cert": frappe.conf.db_ssl_cert, + "ssl_key": frappe.conf.db_ssl_key, } - ) + conn_settings.update(ssl_params) - conn = pymysql.connect( - user=self.user or "", - password=self.password or "", - host=self.host, - port=self.port, - charset="utf8mb4", - use_unicode=True, - ssl=ssl_params if usessl else None, - conv=conversions, - local_infile=frappe.conf.local_infile, - ) - - # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 - # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) - - if self.user != "root": - conn.select_db(self.user) - - return conn + return mariadb.connect(**conn_settings) def get_database_size(self): """'Returns database size in MB""" @@ -122,6 +115,10 @@ class MariaDBDatabase(Database): @staticmethod def escape(s, percent=True): """Excape quotes and percent in given string.""" + # Update: We've scrapped PyMySQL in favour of MariaDB's official Python client + # Also, given we're promoting use of the PyPika builder via frappe.qb, the use + # of this method should be limited. + # pymysql expects unicode argument to escape_string with Python 3 s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") @@ -138,11 +135,11 @@ class MariaDBDatabase(Database): # column type @staticmethod def is_type_number(code): - return code == pymysql.NUMBER + return code == mariadb.NUMBER @staticmethod def is_type_datetime(code): - return code in (pymysql.DATE, pymysql.DATETIME) + return code == mariadb.DATETIME def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: old_name = get_table_name(old_name) @@ -163,15 +160,15 @@ class MariaDBDatabase(Database): # exception types @staticmethod def is_deadlocked(e): - return e.args[0] == ER.LOCK_DEADLOCK + return getattr(e, "errno", 0) == ER.LOCK_DEADLOCK @staticmethod def is_timedout(e): - return e.args[0] == ER.LOCK_WAIT_TIMEOUT + return getattr(e, "errno", 0) == ER.LOCK_WAIT_TIMEOUT @staticmethod def is_table_missing(e): - return e.args[0] == ER.NO_SUCH_TABLE + return getattr(e, "errno", 0) == ER.NO_SUCH_TABLE @staticmethod def is_missing_table(e): @@ -179,35 +176,37 @@ class MariaDBDatabase(Database): @staticmethod def is_missing_column(e): - return e.args[0] == ER.BAD_FIELD_ERROR + return getattr(e, "errno", 0) == ER.BAD_FIELD_ERROR @staticmethod def is_duplicate_fieldname(e): - return e.args[0] == ER.DUP_FIELDNAME + return getattr(e, "errno", 0) == ER.DUP_FIELDNAME @staticmethod def is_duplicate_entry(e): - return e.args[0] == ER.DUP_ENTRY + return getattr(e, "errno", 0) == ER.DUP_ENTRY @staticmethod def is_access_denied(e): - return e.args[0] == ER.ACCESS_DENIED_ERROR + return getattr(e, "errno", 0) == ER.ACCESS_DENIED_ERROR @staticmethod def cant_drop_field_or_key(e): - return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY + return getattr(e, "errno", 0) == ER.CANT_DROP_FIELD_OR_KEY @staticmethod def is_syntax_error(e): - return e.args[0] == ER.PARSE_ERROR + return getattr(e, "errno", 0) == ER.PARSE_ERROR @staticmethod def is_data_too_long(e): - return e.args[0] == ER.DATA_TOO_LONG + return getattr(e, "errno", 0) == ER.DATA_TOO_LONG + @staticmethod def is_primary_key_violation(self, e): return self.is_duplicate_entry(e) and "PRIMARY" in cstr(e.args[1]) + @staticmethod def is_unique_key_violation(self, e): return self.is_duplicate_entry(e) and "Duplicate" in cstr(e.args[1]) diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py index 577c5de2ff..9340e28a36 100644 --- a/frappe/geo/utils.py +++ b/frappe/geo/utils.py @@ -1,9 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE -from pymysql import InternalError - import frappe @@ -67,7 +64,7 @@ def return_location(doctype, filters_sql): coords = frappe.db.sql( """SELECT name, location FROM `tab{}` WHERE {}""".format(doctype, filters_sql), as_dict=True ) - except InternalError: + except frappe.db.InternalError: frappe.msgprint(frappe._("This Doctype does not contain location fields"), raise_exception=True) return else: diff --git a/frappe/patches/v12_0/delete_duplicate_indexes.py b/frappe/patches/v12_0/delete_duplicate_indexes.py index 6a6b0b3204..96ebd237d8 100644 --- a/frappe/patches/v12_0/delete_duplicate_indexes.py +++ b/frappe/patches/v12_0/delete_duplicate_indexes.py @@ -1,5 +1,3 @@ -from pymysql import InternalError - import frappe # This patch deletes all the duplicate indexes created for same column @@ -51,5 +49,5 @@ def execute(): for query in query_list: try: frappe.db.sql(query) - except InternalError: + except frappe.db.InternalError: pass diff --git a/requirements.txt b/requirements.txt index 88ad0562b7..040bd72498 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ html5lib~=1.1 ipython~=7.31.1 Jinja2~=3.0.1 ldap3~=2.9 +mariadb~=1.0.11 markdown2~=2.4.0 maxminddb-geolite2==2018.703 num2words~=0.5.10 From cac25f1229212e9be5a50d9e445c0ea192e37e18 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 01:08:13 +0530 Subject: [PATCH 002/201] feat(minor): Add support for `for_update` in get_last_doc --- frappe/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 17a945c875..c97dc42a9b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1112,11 +1112,11 @@ def get_doc(*args, **kwargs): return doc -def get_last_doc(doctype, filters=None, order_by="creation desc"): +def get_last_doc(doctype, filters=None, order_by="creation desc", for_update=False): """Get last created document of this type.""" d = get_all(doctype, filters=filters, limit_page_length=1, order_by=order_by, pluck="name") if d: - return get_doc(doctype, d[0]) + return get_doc(doctype, d[0], for_update=for_update) else: raise DoesNotExistError From 1fe3624c430cee9b7ee75403475ea51c1cd5dc08 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 01:26:15 +0530 Subject: [PATCH 003/201] fix: DB error detection API usage * Make all methods static * Add typing hints * Don't safe fetch attribute value for errno - MariaDB exceptions if raised will have errno attr in them. If the class doesn't have one, it's not meant to be passed in these methods. --- frappe/database/mariadb/database.py | 59 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 57e9576143..7b13f575e1 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -157,58 +157,65 @@ class MariaDBDatabase(Database): null_constraint = "NOT NULL" if not nullable else "" return self.sql_ddl(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") - # exception types @staticmethod - def is_deadlocked(e): - return getattr(e, "errno", 0) == ER.LOCK_DEADLOCK + def is_deadlocked(e: mariadb.Error) -> bool: + return e.errno == ER.LOCK_DEADLOCK @staticmethod - def is_timedout(e): - return getattr(e, "errno", 0) == ER.LOCK_WAIT_TIMEOUT + def is_timedout(e: mariadb.Error) -> bool: + return e.errno == ER.LOCK_WAIT_TIMEOUT @staticmethod - def is_table_missing(e): - return getattr(e, "errno", 0) == ER.NO_SUCH_TABLE + def is_table_missing(e: mariadb.Error) -> bool: + return e.errno == ER.NO_SUCH_TABLE @staticmethod - def is_missing_table(e): + def is_missing_table(e: mariadb.Error) -> bool: return MariaDBDatabase.is_table_missing(e) @staticmethod - def is_missing_column(e): - return getattr(e, "errno", 0) == ER.BAD_FIELD_ERROR + def is_missing_column(e: mariadb.Error) -> bool: + return e.errno == ER.BAD_FIELD_ERROR @staticmethod - def is_duplicate_fieldname(e): - return getattr(e, "errno", 0) == ER.DUP_FIELDNAME + def is_duplicate_fieldname(e: mariadb.Error) -> bool: + return e.errno == ER.DUP_FIELDNAME @staticmethod - def is_duplicate_entry(e): - return getattr(e, "errno", 0) == ER.DUP_ENTRY + def is_duplicate_entry(e: mariadb.Error) -> bool: + return e.errno == ER.DUP_ENTRY @staticmethod - def is_access_denied(e): - return getattr(e, "errno", 0) == ER.ACCESS_DENIED_ERROR + def is_access_denied(e: mariadb.Error) -> bool: + return e.errno == ER.ACCESS_DENIED_ERROR @staticmethod - def cant_drop_field_or_key(e): - return getattr(e, "errno", 0) == ER.CANT_DROP_FIELD_OR_KEY + def cant_drop_field_or_key(e: mariadb.Error) -> bool: + return e.errno == ER.CANT_DROP_FIELD_OR_KEY @staticmethod - def is_syntax_error(e): - return getattr(e, "errno", 0) == ER.PARSE_ERROR + def is_syntax_error(e: mariadb.Error) -> bool: + return e.errno == ER.PARSE_ERROR @staticmethod - def is_data_too_long(e): - return getattr(e, "errno", 0) == ER.DATA_TOO_LONG + def is_data_too_long(e: mariadb.Error) -> bool: + return e.errno == ER.DATA_TOO_LONG @staticmethod - def is_primary_key_violation(self, e): - return self.is_duplicate_entry(e) and "PRIMARY" in cstr(e.args[1]) + def is_primary_key_violation(e: mariadb.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "PRIMARY" in e.errmsg + and isinstance(e, mariadb.IntegrityError) + ) @staticmethod - def is_unique_key_violation(self, e): - return self.is_duplicate_entry(e) and "Duplicate" in cstr(e.args[1]) + def is_unique_key_violation(e: mariadb.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "Duplicate" in e.errmsg + and isinstance(e, mariadb.IntegrityError) + ) def create_auth_table(self): self.sql_ddl( From 756e385362e67bf32923731762e01940cc097ad4 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 01:42:05 +0530 Subject: [PATCH 004/201] refactor: Move exception detection & abstraction in separate class This change has only been done to separate and club only like methods / utils together --- frappe/database/mariadb/database.py | 127 ++++++++++++++-------------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 7b13f575e1..304ea1e470 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -8,16 +8,79 @@ from pymysql.converters import escape_string import frappe from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable -from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name +from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name -class MariaDBDatabase(Database): +class MariaDBExceptionUtil: ProgrammingError = mariadb.ProgrammingError TableMissingError = mariadb.ProgrammingError OperationalError = mariadb.OperationalError InternalError = mariadb.InternalError SQLError = mariadb.ProgrammingError DataError = mariadb.DataError + + @staticmethod + def is_deadlocked(e: mariadb.Error) -> bool: + return e.errno == ER.LOCK_DEADLOCK + + @staticmethod + def is_timedout(e: mariadb.Error) -> bool: + return e.errno == ER.LOCK_WAIT_TIMEOUT + + @staticmethod + def is_table_missing(e: mariadb.Error) -> bool: + return e.errno == ER.NO_SUCH_TABLE + + @staticmethod + def is_missing_table(e: mariadb.Error) -> bool: + return MariaDBDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e: mariadb.Error) -> bool: + return e.errno == ER.BAD_FIELD_ERROR + + @staticmethod + def is_duplicate_fieldname(e: mariadb.Error) -> bool: + return e.errno == ER.DUP_FIELDNAME + + @staticmethod + def is_duplicate_entry(e: mariadb.Error) -> bool: + return e.errno == ER.DUP_ENTRY + + @staticmethod + def is_access_denied(e: mariadb.Error) -> bool: + return e.errno == ER.ACCESS_DENIED_ERROR + + @staticmethod + def cant_drop_field_or_key(e: mariadb.Error) -> bool: + return e.errno == ER.CANT_DROP_FIELD_OR_KEY + + @staticmethod + def is_syntax_error(e: mariadb.Error) -> bool: + return e.errno == ER.PARSE_ERROR + + @staticmethod + def is_data_too_long(e: mariadb.Error) -> bool: + return e.errno == ER.DATA_TOO_LONG + + @staticmethod + def is_primary_key_violation(e: mariadb.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "PRIMARY" in e.errmsg + and isinstance(e, mariadb.IntegrityError) + ) + + @staticmethod + def is_unique_key_violation(e: mariadb.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "Duplicate" in e.errmsg + and isinstance(e, mariadb.IntegrityError) + ) + + +class MariaDBDatabase(Database, MariaDBExceptionUtil): REGEX_CHARACTER = "regexp" # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, @@ -157,66 +220,6 @@ class MariaDBDatabase(Database): null_constraint = "NOT NULL" if not nullable else "" return self.sql_ddl(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") - @staticmethod - def is_deadlocked(e: mariadb.Error) -> bool: - return e.errno == ER.LOCK_DEADLOCK - - @staticmethod - def is_timedout(e: mariadb.Error) -> bool: - return e.errno == ER.LOCK_WAIT_TIMEOUT - - @staticmethod - def is_table_missing(e: mariadb.Error) -> bool: - return e.errno == ER.NO_SUCH_TABLE - - @staticmethod - def is_missing_table(e: mariadb.Error) -> bool: - return MariaDBDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e: mariadb.Error) -> bool: - return e.errno == ER.BAD_FIELD_ERROR - - @staticmethod - def is_duplicate_fieldname(e: mariadb.Error) -> bool: - return e.errno == ER.DUP_FIELDNAME - - @staticmethod - def is_duplicate_entry(e: mariadb.Error) -> bool: - return e.errno == ER.DUP_ENTRY - - @staticmethod - def is_access_denied(e: mariadb.Error) -> bool: - return e.errno == ER.ACCESS_DENIED_ERROR - - @staticmethod - def cant_drop_field_or_key(e: mariadb.Error) -> bool: - return e.errno == ER.CANT_DROP_FIELD_OR_KEY - - @staticmethod - def is_syntax_error(e: mariadb.Error) -> bool: - return e.errno == ER.PARSE_ERROR - - @staticmethod - def is_data_too_long(e: mariadb.Error) -> bool: - return e.errno == ER.DATA_TOO_LONG - - @staticmethod - def is_primary_key_violation(e: mariadb.Error) -> bool: - return ( - MariaDBDatabase.is_duplicate_entry(e) - and "PRIMARY" in e.errmsg - and isinstance(e, mariadb.IntegrityError) - ) - - @staticmethod - def is_unique_key_violation(e: mariadb.Error) -> bool: - return ( - MariaDBDatabase.is_duplicate_entry(e) - and "Duplicate" in e.errmsg - and isinstance(e, mariadb.IntegrityError) - ) - def create_auth_table(self): self.sql_ddl( """create table if not exists `__Auth` ( From a17c978065515c264901207f9320f8d2048fa203 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 01:43:17 +0530 Subject: [PATCH 005/201] refactor!: frappe.db.get_database_list * Drop mandatory unused arg in method call * Use pluck instead of list comprehension + subscripting --- frappe/database/mariadb/database.py | 4 ++-- frappe/database/postgres/database.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 304ea1e470..7afa265810 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -362,5 +362,5 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): db_table.sync() self.begin() - def get_database_list(self, target): - return [d[0] for d in self.sql("SHOW DATABASES;")] + def get_database_list(self): + return self.sql("SHOW DATABASES", pluck=True) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 8bd4113823..84a237e702 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -375,8 +375,8 @@ class PostgresDatabase(Database): as_dict=1, ) - def get_database_list(self, target): - return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] + def get_database_list(self): + return self.sql("SELECT datname FROM pg_database", pluck=True) def modify_query(query): From 0d80f6ac52e34a4005c75c2e4fef7729083e6480 Mon Sep 17 00:00:00 2001 From: gavin Date: Sat, 21 May 2022 02:37:51 +0530 Subject: [PATCH 006/201] feat: mariadb connection pooling Start initial pool of 4 (_POOL_SIZE) connections for a given site. When the pool is exhausted, generate new connections upon request and add them to the pool. Max allowed size for each pool is 64 (_MAX_POOL_SIZE) connections. However, you may have more than 64 active connections, just that they won't be pooled but destroyed upon job completion/end of request/similar cycle. --- frappe/database/mariadb/database.py | 56 +++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 7afa265810..f43f068e40 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, Tuple, Union import mariadb from mariadb.constants import FIELD_TYPE @@ -10,6 +10,19 @@ from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name +if TYPE_CHECKING: + from mariadb import ConnectionPool + +_SITE_POOLS: Dict[str, "ConnectionPool"] = {} +_MAX_POOL_SIZE = 64 # max pool size supported by MariaDB client +_POOL_SIZE = ( + 4 # selected arbitrarily to avoid overloading the server and being mindful of multitenancy +) + +# init size of connection pool will be _POOL_SIZE for each site. This pool may expand up to _MAX_POOL_SIZE +# as per requirement. This cannot be a function of @@global.max_connections, # of sites since there may be +# multiple processes holding connections; and this defines the size for each of those processes/workers + class MariaDBExceptionUtil: ProgrammingError = mariadb.ProgrammingError @@ -132,6 +145,44 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): } def get_connection(self): + # get pooled connection + global _SITE_POOLS + + if frappe.local.site not in _SITE_POOLS: + pool = mariadb.ConnectionPool( + pool_name=f"{frappe.local.site}_conn_pool", + pool_size=_MAX_POOL_SIZE, + pool_reset_connection=False, + ) + pool.set_config(**self.get_connection_settings()) + + for _ in range(_POOL_SIZE): + pool.add_connection() + + _SITE_POOLS[frappe.local.site] = pool + + site_pool = _SITE_POOLS[frappe.local.site] + + try: + conn = site_pool.get_connection() + except mariadb.PoolError: + # PoolError is raised when the pool is exhausted + conn = self.create_connection() + try: + site_pool.add_connection(conn) + # log this via frappe.logger & continue - site needs bigger pool...over _POOL_SIZE + except mariadb.PoolError: + # PoolError is raised when size limit is reached + # log this via frappe.logger & continue - site needs a much bigger pool...over _MAX_POOL_SIZE + pass + + return conn + + def create_connection(self): + # get new connection + return mariadb.connect(**self.get_connection_settings()) + + def get_connection_settings(self) -> Dict: conn_settings = { "host": self.host, "user": self.user, @@ -158,8 +209,7 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): "ssl_key": frappe.conf.db_ssl_key, } conn_settings.update(ssl_params) - - return mariadb.connect(**conn_settings) + return conn_settings def get_database_size(self): """'Returns database size in MB""" From dae57d0de2dabf22dbc876a3819dcdbe7e397d1e Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 12:45:42 +0530 Subject: [PATCH 007/201] fix(db)!: Base methods should throw NotImplementedError --- frappe/database/database.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index ab1d8ea4fa..c6cb162fa0 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -82,10 +82,11 @@ class Database(object): self._conn.select_db(db_name) def get_connection(self): - pass + """Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects""" + raise NotImplementedError def get_database_size(self): - pass + raise NotImplementedError def sql( self, From 363708d7f2e161b0becb41e90c03bfb31a5f822d Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 12:46:56 +0530 Subject: [PATCH 008/201] fix: frappe.conf.disable_database_connection_pooling For the times you don't want to use pooling ;) Calling get_connection/connect will close all pooled connections for the current process / worker! --- frappe/database/mariadb/database.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index f43f068e40..3f2580490d 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -145,9 +145,24 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): } def get_connection(self): + """Return MariaDB connection object. + + If frappe.conf.disable_database_connection_pooling is set, return a new connection + object and close existing pool if exists. Else, return a connection from the pool. + """ # get pooled connection global _SITE_POOLS + if frappe.conf.disable_database_connection_pooling: + if frappe.local.site in _SITE_POOLS: + pool = _SITE_POOLS[frappe.local.site] + try: + pool.close() + except Exception: + pass + _SITE_POOLS.pop(frappe.local.site, None) + return self.create_connection() + if frappe.local.site not in _SITE_POOLS: pool = mariadb.ConnectionPool( pool_name=f"{frappe.local.site}_conn_pool", From 993b935097a6289b8325b982d980b6dd2c861bb2 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 13:57:24 +0530 Subject: [PATCH 009/201] fix: Separate pool for replica setup _SITE_POOLS[frappe.local.site].read_only will hold ConnectionPool for replica database connections. _SITE_POOLS[frappe.local.site].default will hold pool for connections that allow read+writes --- frappe/database/mariadb/database.py | 76 +++++++++++++++++++---------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 3f2580490d..e5cf1cd34a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import TYPE_CHECKING, Dict, List, Tuple, Union import mariadb @@ -13,12 +14,11 @@ from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name if TYPE_CHECKING: from mariadb import ConnectionPool -_SITE_POOLS: Dict[str, "ConnectionPool"] = {} -_MAX_POOL_SIZE = 64 # max pool size supported by MariaDB client -_POOL_SIZE = ( - 4 # selected arbitrarily to avoid overloading the server and being mindful of multitenancy -) +_SITE_POOLS = defaultdict(frappe._dict) +_MAX_POOL_SIZE = 64 +_POOL_SIZE = 4 +# _POOL_SIZE is selected "arbitrarily" to avoid overloading the server and being mindful of multitenancy # init size of connection pool will be _POOL_SIZE for each site. This pool may expand up to _MAX_POOL_SIZE # as per requirement. This cannot be a function of @@global.max_connections, # of sites since there may be # multiple processes holding connections; and this defines the size for each of those processes/workers @@ -154,29 +154,15 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): global _SITE_POOLS if frappe.conf.disable_database_connection_pooling: - if frappe.local.site in _SITE_POOLS: - pool = _SITE_POOLS[frappe.local.site] - try: - pool.close() - except Exception: - pass - _SITE_POOLS.pop(frappe.local.site, None) + self.close_connection_pools() return self.create_connection() + is_read_only_conn = hasattr(frappe.local, "primary_db") + if frappe.local.site not in _SITE_POOLS: - pool = mariadb.ConnectionPool( - pool_name=f"{frappe.local.site}_conn_pool", - pool_size=_MAX_POOL_SIZE, - pool_reset_connection=False, - ) - pool.set_config(**self.get_connection_settings()) - - for _ in range(_POOL_SIZE): - pool.add_connection() - - _SITE_POOLS[frappe.local.site] = pool - - site_pool = _SITE_POOLS[frappe.local.site] + site_pool = self.create_connection_pool(read_only=is_read_only_conn) + else: + site_pool = self.get_connection_pool(read_only=is_read_only_conn) try: conn = site_pool.get_connection() @@ -193,8 +179,46 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): return conn + def close_connection_pools(self): + if frappe.local.site in _SITE_POOLS: + pools = _SITE_POOLS[frappe.local.site] + for pool in pools.values(): + try: + pool.close() + except Exception: + pass + _SITE_POOLS.pop(frappe.local.site, None) + + def get_pool_name(self, read_only=False) -> str: + pool_type = "read-only" if read_only else "default" + return f"{frappe.local.site}-{pool_type}" + + def get_connection_pool(self, read_only=False) -> "ConnectionPool": + """Return MariaDB connection pool object. + + If `read_only` is True, return a read only pool. + """ + return _SITE_POOLS[frappe.local.site]["read_only" if read_only else "default"] + + def create_connection_pool(self, read_only=False): + pool = mariadb.ConnectionPool( + pool_name=self.get_pool_name(read_only=read_only), + pool_size=_MAX_POOL_SIZE, + pool_reset_connection=False, + ) + pool.set_config(**self.get_connection_settings()) + + for _ in range(_POOL_SIZE): + pool.add_connection() + + if read_only: + _SITE_POOLS[frappe.local.site].read_only = pool + else: + _SITE_POOLS[frappe.local.site].default = pool + + return pool + def create_connection(self): - # get new connection return mariadb.connect(**self.get_connection_settings()) def get_connection_settings(self) -> Dict: From f02125030882b28404229401ed341e24143aa60f Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 14:07:34 +0530 Subject: [PATCH 010/201] refactor: Move connection & pool management out, inherit instead --- frappe/database/mariadb/database.py | 112 ++++++++++++++-------------- 1 file changed, 58 insertions(+), 54 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index e5cf1cd34a..f9616761b1 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -19,9 +19,11 @@ _MAX_POOL_SIZE = 64 _POOL_SIZE = 4 # _POOL_SIZE is selected "arbitrarily" to avoid overloading the server and being mindful of multitenancy -# init size of connection pool will be _POOL_SIZE for each site. This pool may expand up to _MAX_POOL_SIZE -# as per requirement. This cannot be a function of @@global.max_connections, # of sites since there may be -# multiple processes holding connections; and this defines the size for each of those processes/workers +# init size of connection pool will be _POOL_SIZE for each site. Replica setups will have separate pool. +# This means each site with a replica setup can have 2 active pools of size _POOL_SIZE each. Each pool may +# expand up to _MAX_POOL_SIZE as per requirement. This cannot be a function of @@global.max_connections, +# no. of sites since there may be multiple processes holding connections; and this defines the size for each +# of those processes/workers. Check MariaDBConnectionUtil for connection & pool management. class MariaDBExceptionUtil: @@ -93,57 +95,7 @@ class MariaDBExceptionUtil: ) -class MariaDBDatabase(Database, MariaDBExceptionUtil): - REGEX_CHARACTER = "regexp" - - # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, - # it drops the cache and uses the next non cached value in setval query and - # puts that in the backup file, which will start the counter - # from that value when inserting any new record in the doctype. - # By default the cache is 1000 which will mess up the sequence when - # using the system after a restore. - # issue link: https://jira.mariadb.org/browse/MDEV-21786 - SEQUENCE_CACHE = 50 - - def setup_type_map(self): - self.db_type = "mariadb" - self.type_map = { - "Currency": ("decimal", "21,9"), - "Int": ("int", "11"), - "Long Int": ("bigint", "20"), - "Float": ("decimal", "21,9"), - "Percent": ("decimal", "21,9"), - "Check": ("int", "1"), - "Small Text": ("text", ""), - "Long Text": ("longtext", ""), - "Code": ("longtext", ""), - "Text Editor": ("longtext", ""), - "Markdown Editor": ("longtext", ""), - "HTML Editor": ("longtext", ""), - "Date": ("date", ""), - "Datetime": ("datetime", "6"), - "Time": ("time", "6"), - "Text": ("text", ""), - "Data": ("varchar", self.VARCHAR_LEN), - "Link": ("varchar", self.VARCHAR_LEN), - "Dynamic Link": ("varchar", self.VARCHAR_LEN), - "Password": ("text", ""), - "Select": ("varchar", self.VARCHAR_LEN), - "Rating": ("decimal", "3,2"), - "Read Only": ("varchar", self.VARCHAR_LEN), - "Attach": ("text", ""), - "Attach Image": ("text", ""), - "Signature": ("longtext", ""), - "Color": ("varchar", self.VARCHAR_LEN), - "Barcode": ("longtext", ""), - "Geolocation": ("longtext", ""), - "Duration": ("decimal", "21,9"), - "Icon": ("varchar", self.VARCHAR_LEN), - "Phone": ("varchar", self.VARCHAR_LEN), - "Autocomplete": ("varchar", self.VARCHAR_LEN), - "JSON": ("json", ""), - } - +class MariaDBConnectionUtil: def get_connection(self): """Return MariaDB connection object. @@ -250,6 +202,58 @@ class MariaDBDatabase(Database, MariaDBExceptionUtil): conn_settings.update(ssl_params) return conn_settings + +class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): + REGEX_CHARACTER = "regexp" + + # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, + # it drops the cache and uses the next non cached value in setval query and + # puts that in the backup file, which will start the counter + # from that value when inserting any new record in the doctype. + # By default the cache is 1000 which will mess up the sequence when + # using the system after a restore. + # issue link: https://jira.mariadb.org/browse/MDEV-21786 + SEQUENCE_CACHE = 50 + + def setup_type_map(self): + self.db_type = "mariadb" + self.type_map = { + "Currency": ("decimal", "21,9"), + "Int": ("int", "11"), + "Long Int": ("bigint", "20"), + "Float": ("decimal", "21,9"), + "Percent": ("decimal", "21,9"), + "Check": ("int", "1"), + "Small Text": ("text", ""), + "Long Text": ("longtext", ""), + "Code": ("longtext", ""), + "Text Editor": ("longtext", ""), + "Markdown Editor": ("longtext", ""), + "HTML Editor": ("longtext", ""), + "Date": ("date", ""), + "Datetime": ("datetime", "6"), + "Time": ("time", "6"), + "Text": ("text", ""), + "Data": ("varchar", self.VARCHAR_LEN), + "Link": ("varchar", self.VARCHAR_LEN), + "Dynamic Link": ("varchar", self.VARCHAR_LEN), + "Password": ("text", ""), + "Select": ("varchar", self.VARCHAR_LEN), + "Rating": ("decimal", "3,2"), + "Read Only": ("varchar", self.VARCHAR_LEN), + "Attach": ("text", ""), + "Attach Image": ("text", ""), + "Signature": ("longtext", ""), + "Color": ("varchar", self.VARCHAR_LEN), + "Barcode": ("longtext", ""), + "Geolocation": ("longtext", ""), + "Duration": ("decimal", "21,9"), + "Icon": ("varchar", self.VARCHAR_LEN), + "Phone": ("varchar", self.VARCHAR_LEN), + "Autocomplete": ("varchar", self.VARCHAR_LEN), + "JSON": ("json", ""), + } + def get_database_size(self): """'Returns database size in MB""" db_size = self.sql( From aaef732581f470c2264a0f07168bfe498d88a2f6 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 14:09:09 +0530 Subject: [PATCH 011/201] fix!: Remove frappe.db.create_help_table Help table has been deprecated for a while. It's safe to get rid of the API too. --- frappe/database/mariadb/database.py | 16 ---------------- frappe/database/postgres/database.py | 11 ----------- 2 files changed, 27 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index f9616761b1..fd9ecd1f96 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -354,22 +354,6 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): ) ENGINE=InnoDB DEFAULT CHARSET=utf8""" ) - def create_help_table(self): - self.sql( - """create table help( - path varchar(255), - content text, - title text, - intro text, - full_path text, - fulltext(title), - fulltext(content), - index (path)) - COLLATE=utf8mb4_unicode_ci - ENGINE=MyISAM - CHARACTER SET=utf8mb4""" - ) - @staticmethod def get_on_duplicate_update(key=None): return "ON DUPLICATE key UPDATE " diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 84a237e702..d60a6d1918 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -267,17 +267,6 @@ class PostgresDatabase(Database): )""" ) - def create_help_table(self): - self.sql( - """CREATE TABLE "help"( - "path" varchar(255), - "content" text, - "title" text, - "intro" text, - "full_path" text)""" - ) - self.sql("""CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")""") - def updatedb(self, doctype, meta=None): """ Syncs a `DocType` to the table From 0fe764c9b8564205e40ae91b22aa4e88c1e6bb03 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 14:24:26 +0530 Subject: [PATCH 012/201] fix: Depend on replica details being there for Replica ConnectionPool This logic mirror how replica connections are handled --- frappe/database/mariadb/database.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index fd9ecd1f96..595a02db6f 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -102,19 +102,18 @@ class MariaDBConnectionUtil: If frappe.conf.disable_database_connection_pooling is set, return a new connection object and close existing pool if exists. Else, return a connection from the pool. """ - # get pooled connection global _SITE_POOLS if frappe.conf.disable_database_connection_pooling: self.close_connection_pools() return self.create_connection() - is_read_only_conn = hasattr(frappe.local, "primary_db") + read_only = frappe.conf.read_from_replica and frappe.conf.replica_host if frappe.local.site not in _SITE_POOLS: - site_pool = self.create_connection_pool(read_only=is_read_only_conn) + site_pool = self.create_connection_pool(read_only=read_only) else: - site_pool = self.get_connection_pool(read_only=is_read_only_conn) + site_pool = self.get_connection_pool(read_only=read_only) try: conn = site_pool.get_connection() From 9100e8f0bb53a9afb19437b456a11d2077dd0054 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 18:36:59 +0530 Subject: [PATCH 013/201] ci: Install system dependency - libmariadb-dev --- .github/helper/install_dependencies.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index f0e8016860..666af13882 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -21,3 +21,5 @@ sudo apt-get install libcups2-dev # install redis sudo apt-get install redis-server +# install redis +sudo apt-get install libmariadb-dev From 607a99f50bdee0a906bca8f87b00617d6293a374 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 23 May 2022 18:43:19 +0530 Subject: [PATCH 014/201] fix: F821 undefined name 'InternalError' tx flake8 --- frappe/geo/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py index 9340e28a36..d4b225b055 100644 --- a/frappe/geo/utils.py +++ b/frappe/geo/utils.py @@ -80,7 +80,7 @@ def return_coordinates(doctype, filters_sql): """SELECT name, latitude, longitude FROM `tab{}` WHERE {}""".format(doctype, filters_sql), as_dict=True, ) - except InternalError: + except frappe.db.InternalError: frappe.msgprint( frappe._("This Doctype does not contain latitude and longitude fields"), raise_exception=True ) From 6637a4ebbe58d677095b5c031a57cde5cfdf5d4c Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 12:34:14 +0530 Subject: [PATCH 015/201] fix: Allow non-blocking checks via MariaDBExceptionUtils --- frappe/database/mariadb/database.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 595a02db6f..9bd2d0bcd2 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -36,15 +36,15 @@ class MariaDBExceptionUtil: @staticmethod def is_deadlocked(e: mariadb.Error) -> bool: - return e.errno == ER.LOCK_DEADLOCK + return getattr(e, "errno", None) == ER.LOCK_DEADLOCK @staticmethod def is_timedout(e: mariadb.Error) -> bool: - return e.errno == ER.LOCK_WAIT_TIMEOUT + return getattr(e, "errno", None) == ER.LOCK_WAIT_TIMEOUT @staticmethod def is_table_missing(e: mariadb.Error) -> bool: - return e.errno == ER.NO_SUCH_TABLE + return getattr(e, "errno", None) == ER.NO_SUCH_TABLE @staticmethod def is_missing_table(e: mariadb.Error) -> bool: @@ -52,31 +52,31 @@ class MariaDBExceptionUtil: @staticmethod def is_missing_column(e: mariadb.Error) -> bool: - return e.errno == ER.BAD_FIELD_ERROR + return getattr(e, "errno", None) == ER.BAD_FIELD_ERROR @staticmethod def is_duplicate_fieldname(e: mariadb.Error) -> bool: - return e.errno == ER.DUP_FIELDNAME + return getattr(e, "errno", None) == ER.DUP_FIELDNAME @staticmethod def is_duplicate_entry(e: mariadb.Error) -> bool: - return e.errno == ER.DUP_ENTRY + return getattr(e, "errno", None) == ER.DUP_ENTRY @staticmethod def is_access_denied(e: mariadb.Error) -> bool: - return e.errno == ER.ACCESS_DENIED_ERROR + return getattr(e, "errno", None) == ER.ACCESS_DENIED_ERROR @staticmethod def cant_drop_field_or_key(e: mariadb.Error) -> bool: - return e.errno == ER.CANT_DROP_FIELD_OR_KEY + return getattr(e, "errno", None) == ER.CANT_DROP_FIELD_OR_KEY @staticmethod def is_syntax_error(e: mariadb.Error) -> bool: - return e.errno == ER.PARSE_ERROR + return getattr(e, "errno", None) == ER.PARSE_ERROR @staticmethod def is_data_too_long(e: mariadb.Error) -> bool: - return e.errno == ER.DATA_TOO_LONG + return getattr(e, "errno", None) == ER.DATA_TOO_LONG @staticmethod def is_primary_key_violation(e: mariadb.Error) -> bool: From 73c3db12ee2e87518dbb06a52720648956c95781 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 12:44:21 +0530 Subject: [PATCH 016/201] refactor(minor): Move internal util to the module where it's used --- frappe/commands/scheduler.py | 16 ---------------- frappe/installer.py | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index ed6a0dea57..70af784930 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -5,22 +5,6 @@ import click import frappe from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import cint - - -def _is_scheduler_enabled(): - enable_scheduler = False - try: - frappe.connect() - enable_scheduler = ( - cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False - ) - except: - pass - finally: - frappe.db.close() - - return enable_scheduler @click.command("trigger-scheduler-event", help="Trigger a scheduler event") diff --git a/frappe/installer.py b/frappe/installer.py index 5cd46e618d..42cc023c6a 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json @@ -11,7 +11,20 @@ import click import frappe from frappe.defaults import _clear_cache -from frappe.utils import is_git_url +from frappe.utils import cint, is_git_url + + +def _is_scheduler_enabled() -> bool: + enable_scheduler = False + try: + frappe.connect() + enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) + except Exception: + pass + finally: + frappe.db.close() + + return bool(enable_scheduler) def _new_site( @@ -30,11 +43,9 @@ def _new_site( db_type=None, db_host=None, db_port=None, - new_site=False, ): """Install a new Frappe site""" - from frappe.commands.scheduler import _is_scheduler_enabled from frappe.utils import get_site_path, scheduler, touch_file if not force and os.path.exists(site): From 639fa621386b15490c99c22c54516fab9e5a63dd Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 18:41:31 +0530 Subject: [PATCH 017/201] fix: frappe.DISABLE_DATABASE_POOLING to override frappe.conf.disable_database_connection_pooling Also, don't pool root connections --- frappe/__init__.py | 1 + frappe/commands/site.py | 2 ++ frappe/database/mariadb/database.py | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index c97dc42a9b..ec016c8b36 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -41,6 +41,7 @@ __title__ = "Frappe Framework" controllers = {} local = Local() STANDARD_USERS = ("Guest", "Administrator") +DISABLE_DATABASE_POOLING = None _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 628a10d67e..c8191daaa1 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -68,6 +68,8 @@ def new_site( "Create a new site" from frappe.installer import _new_site + frappe.DISABLE_DATABASE_POOLING = True + frappe.init(site=site, new_site=True) _new_site( diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 9bd2d0bcd2..fe43e8ffed 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -26,6 +26,16 @@ _POOL_SIZE = 4 # of those processes/workers. Check MariaDBConnectionUtil for connection & pool management. +def is_connection_pooling_enabled() -> bool: + """Set `frappe.DISABLE_CONNECTION_POOLING` to enable/disable connection pooling for all on current + process. This will override config key `disable_database_connection_pooling`. Set key + `disable_database_connection_pooling` in site config for persistent settings across workers.""" + + if frappe.DISABLE_DATABASE_POOLING is not None: + return not frappe.DISABLE_DATABASE_POOLING + return frappe.local.conf.disable_database_connection_pooling + + class MariaDBExceptionUtil: ProgrammingError = mariadb.ProgrammingError TableMissingError = mariadb.ProgrammingError @@ -104,7 +114,11 @@ class MariaDBConnectionUtil: """ global _SITE_POOLS - if frappe.conf.disable_database_connection_pooling: + # don't pool root connections + if self.user == "root": + return self.create_connection() + + if is_connection_pooling_enabled(): self.close_connection_pools() return self.create_connection() From 7e7695343502c4734dd5ce11c48704e568deea38 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 18:43:23 +0530 Subject: [PATCH 018/201] fix(drop-site): Tell user where archived site data is stored --- frappe/commands/site.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index c8191daaa1..0a97a4107f 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -87,7 +87,6 @@ def new_site( db_type=db_type, db_host=db_host, db_port=db_port, - new_site=True, ) if set_default: @@ -849,9 +848,10 @@ def _drop_site( archived_sites_path = archived_sites_path or os.path.join( frappe.get_app_path("frappe"), "..", "..", "..", "archived", "sites" ) + archived_sites_path = os.path.realpath(archived_sites_path) + click.secho(f"Moving site to archive under {archived_sites_path}", fg="green") os.makedirs(archived_sites_path, exist_ok=True) - move(archived_sites_path, site) From 32a30ce933f5146e2322894849d56957ad28a79f Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 18:44:35 +0530 Subject: [PATCH 019/201] refactor: DBManager * Simplify logic: DRY, lesser indentation & all DAT * Utilize newer APIs, f-strings & more * Cleaner namespace * Conform inconsistent behaviours --- frappe/database/db_manager.py | 58 +++++++++++++---------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 8f810fe54b..f90fb59d97 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -1,5 +1,3 @@ -import os - import frappe @@ -15,63 +13,51 @@ class DbManager: return self.db.sql("select user()")[0][0].split("@")[1] def create_user(self, user, password, host=None): - # Create user if it doesn't exist. - if not host: - host = self.get_current_host() - - if password: - self.db.sql("CREATE USER '%s'@'%s' IDENTIFIED BY '%s';" % (user, host, password)) - else: - self.db.sql("CREATE USER '%s'@'%s';" % (user, host)) + host = host or self.get_current_host() + password_predicate = f" IDENTIFIED BY '{password}'" if password else "" + self.db.sql(f"CREATE USER '{user}'@'{host}'{password_predicate}") def delete_user(self, target, host=None): - if not host: - host = self.get_current_host() - try: - self.db.sql("DROP USER '%s'@'%s';" % (target, host)) - except Exception as e: - if e.args[0] == 1396: - pass - else: - raise + host = host or self.get_current_host() + self.db.sql(f"DROP USER IF EXISTS '{target}'@'{host}'") def create_database(self, target): if target in self.get_database_list(): self.drop_database(target) - - self.db.sql("CREATE DATABASE `%s` ;" % target) + self.db.sql(f"CREATE DATABASE `{target}`") def drop_database(self, target): - self.db.sql("DROP DATABASE IF EXISTS `%s`;" % target) + self.db.sql(f"DROP DATABASE IF EXISTS `{target}`") def grant_all_privileges(self, target, user, host=None): - if not host: - host = self.get_current_host() - - if frappe.conf.get("rds_db", 0) == 1: - self.db.sql( - "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES ON `%s`.* TO '%s'@'%s';" - % (target, user, host) + host = host or self.get_current_host() + permissions = ( + ( + "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, " + "CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, " + "CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" ) - else: - self.db.sql("GRANT ALL PRIVILEGES ON `%s`.* TO '%s'@'%s';" % (target, user, host)) + if frappe.conf.rds_db + else "ALL PRIVILEGES" + ) + self.db.sql(f"GRANT {permissions} ON `{target}`.* TO '{user}'@'{host}'") def flush_privileges(self): self.db.sql("FLUSH PRIVILEGES") def get_database_list(self): - """get list of databases""" - return [d[0] for d in self.db.sql("SHOW DATABASES")] + return self.db.sql("SHOW DATABASES", pluck=True) @staticmethod def restore_database(target, source, user, password): + import os + from distutils.spawn import find_executable + from frappe.utils import make_esc esc = make_esc("$ ") - - from distutils.spawn import find_executable - pv = find_executable("pv") + if pv: pipe = "{pv} {source} |".format(pv=pv, source=source) source = "" From a7838ccca4ea8671174d3d0e74092ac5268a0022 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 18:49:12 +0530 Subject: [PATCH 020/201] fix: Remove half baked support for filters_config in db.query --- frappe/database/query.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index f608539854..c2dd076f8f 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -5,7 +5,6 @@ from typing import Any, Callable, Dict, List, Tuple, Union import frappe from frappe import _ -from frappe.boot import get_additional_filters_from_hooks from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table @@ -173,19 +172,14 @@ class Query: # default operators all_operators = OPERATOR_MAP.copy() - # update with site-specific custom operators - additional_filters_config = get_additional_filters_from_hooks() - - if additional_filters_config: + # TODO: update with site-specific custom operators / removed previous buggy implementation + if frappe.get_hooks("filters_config"): from frappe.utils.commands import warn - warn("'filters_config' hook is not completely implemented yet in frappe.db.query engine") - - for _operator, function in additional_filters_config.items(): - if callable(function): - all_operators.update({_operator.casefold(): function}) - elif isinstance(function, dict): - all_operators[_operator.casefold()] = frappe.get_attr(function.get("get_field"))()["operator"] + warn( + "The 'filters_config' hook used to add custom operators is not yet implemented" + " in frappe.db.query engine. Use db_query (frappe.get_list) instead." + ) return all_operators From 3ed808aec7db82ffec9ed9bbde13f210196fa138 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 18:52:19 +0530 Subject: [PATCH 021/201] fix: Connection Pooling * Reduce _POOL_SIZE from 4 to 1. New pools will have just one connection. They can scale up as per requirement there after. * Set auto_connect flag in MariaDB connection - https://mariadb.com/docs/connect/programming-languages/python/connect/ --- frappe/database/mariadb/database.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index fe43e8ffed..c8b365c61d 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: _SITE_POOLS = defaultdict(frappe._dict) _MAX_POOL_SIZE = 64 -_POOL_SIZE = 4 +_POOL_SIZE = 1 # _POOL_SIZE is selected "arbitrarily" to avoid overloading the server and being mindful of multitenancy # init size of connection pool will be _POOL_SIZE for each site. Replica setups will have separate pool. @@ -107,6 +107,11 @@ class MariaDBExceptionUtil: class MariaDBConnectionUtil: def get_connection(self): + conn = self._get_connection() + conn.auto_reconnect = True + return conn + + def _get_connection(self) -> "mariadb.Connection": """Return MariaDB connection object. If frappe.conf.disable_database_connection_pooling is set, return a new connection @@ -173,14 +178,14 @@ class MariaDBConnectionUtil: ) pool.set_config(**self.get_connection_settings()) - for _ in range(_POOL_SIZE): - pool.add_connection() - if read_only: _SITE_POOLS[frappe.local.site].read_only = pool else: _SITE_POOLS[frappe.local.site].default = pool + for _ in range(_POOL_SIZE): + pool.add_connection() + return pool def create_connection(self): @@ -191,7 +196,6 @@ class MariaDBConnectionUtil: "host": self.host, "user": self.user, "password": self.password, - "database": self.user, "converter": { FIELD_TYPE.NEWDECIMAL: float, FIELD_TYPE.DATETIME: get_datetime, @@ -199,6 +203,9 @@ class MariaDBConnectionUtil: }, } + if self.user != "root": + conn_settings["database"] = self.user + if self.port: conn_settings["port"] = int(self.port) From d87197f04211435bddeae04112ff1a735a8f96c6 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 19:18:22 +0530 Subject: [PATCH 022/201] fix: Correct use of is_connection_pooling_enabled check --- frappe/database/mariadb/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index c8b365c61d..a85df44dac 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -123,7 +123,7 @@ class MariaDBConnectionUtil: if self.user == "root": return self.create_connection() - if is_connection_pooling_enabled(): + if not is_connection_pooling_enabled(): self.close_connection_pools() return self.create_connection() From f7c5c27d818420392d48d0a61c98dd1c25509cf3 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 19:35:57 +0530 Subject: [PATCH 023/201] fix: Use sql_ddl instead of sql Previous attempts to run this would lead to stalls...long long stalls. And nothing blocking the query, it would stay in a "Waiting for table metadata lock" state. --- frappe/database/db_manager.py | 2 +- frappe/database/mariadb/setup_db.py | 33 +++++++++++++---------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index f90fb59d97..f69c93db0f 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -27,7 +27,7 @@ class DbManager: self.db.sql(f"CREATE DATABASE `{target}`") def drop_database(self, target): - self.db.sql(f"DROP DATABASE IF EXISTS `{target}`") + self.db.sql_ddl(f"DROP DATABASE IF EXISTS `{target}`") def grant_all_privileges(self, target, user, host=None): host = host or self.get_current_host() diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index 4399ccfa6a..c26aaea343 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -83,9 +83,9 @@ def setup_help_database(help_db_name): def drop_user_and_database(db_name, root_login, root_password): frappe.local.db = get_root_connection(root_login, root_password) dbman = DbManager(frappe.local.db) + dbman.drop_database(db_name) dbman.delete_user(db_name, host="%") dbman.delete_user(db_name) - dbman.drop_database(db_name) def bootstrap_database(db_name, verbose, source_sql=None): @@ -131,7 +131,7 @@ def check_database_settings(): else: expected_variables = expected_settings_10_3_later - mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) + mariadb_variables = frappe._dict(frappe.db.sql("show variables")) # Check each expected value vs. actuals: result = True for key, expected_value in expected_variables.items(): @@ -142,16 +142,19 @@ def check_database_settings(): ) result = False if not result: - site = frappe.local.site - msg = ( - "Creation of your site - {x} failed because MariaDB is not properly {sep}" - "configured. If using version 10.2.x or earlier, make sure you use the {sep}" - "the Barracuda storage engine. {sep}{sep}" - "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" - "then run `bench new-site {x}` again.{sep2}" - "" - ).format(x=site, sep2="\n" * 2, sep="\n") - print_db_config(msg) + print( + ( + "=" * 80 + "\n" + "Creation of your site - {x} failed because MariaDB is not properly {sep}" + "configured. If using version 10.2.x or earlier, make sure you use the {sep}" + "the Barracuda storage engine. {sep}{sep}" + "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" + "then run `bench new-site {x}` again.{sep2}" + "" + "=" * 80 + ).format(x=frappe.local.site, sep2="\n" * 2, sep="\n") + ) + return result @@ -173,9 +176,3 @@ def get_root_connection(root_login, root_password): ) return frappe.local.flags.root_connection - - -def print_db_config(explanation): - print("=" * 80) - print(explanation) - print("=" * 80) From 0a8941c58317956e0661f90fa2357e39674f63c3 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 24 May 2022 19:54:12 +0530 Subject: [PATCH 024/201] fix: Disable db pooling on all commands by default Set envvar DATABASE_POOLING to enable --- frappe/commands/site.py | 2 -- frappe/utils/bench_helper.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 0a97a4107f..80acf647e0 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -68,8 +68,6 @@ def new_site( "Create a new site" from frappe.installer import _new_site - frappe.DISABLE_DATABASE_POOLING = True - frappe.init(site=site, new_site=True) _new_site( diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index a0b011acc1..8de10b73d4 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -1,6 +1,7 @@ import importlib import json import os +import sys import traceback import warnings @@ -106,4 +107,9 @@ if __name__ == "__main__": if not frappe._dev_server: warnings.simplefilter("ignore") + # disable pooling for commands executed via bench unless explicitly stated otherwise + # - except for commands serve & worker + if not {"serve", "worker"} & set(sys.argv) and int(os.environ.get("DATABASE_POOLING", 0)): + frappe.DISABLE_DATABASE_POOLING = True + main() From 4f72eb9eac45ac4a9917e65df8ac22389e247938 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 25 May 2022 11:44:08 +0530 Subject: [PATCH 025/201] refactor: Base Database class * DRY, explicit > implicit usages * Don't re-compute and do multiple calls for errprint, log, mogrify, etc * Use consistent logging methods * Simplify logic / Use newer APIs where applicable --- frappe/database/database.py | 59 ++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index c6cb162fa0..2cd950f785 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,10 +1,11 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE # Database Module # -------------------- import datetime +import json import random import re import string @@ -159,24 +160,16 @@ class Database(object): if debug: time_start = time() - self.log_query(query, values, debug, explain) - if values != (): - - # MySQL-python==1.2.5 hack! if not isinstance(values, (dict, tuple, list)): values = (values,) - self._cursor.execute(query, values) + self.log_query(query, values, debug, explain) - if frappe.flags.in_migrate: - self.log_touched_tables(query, values) + self._cursor.execute(query, values) - else: - self._cursor.execute(query) - - if frappe.flags.in_migrate: - self.log_touched_tables(query) + if frappe.flags.in_migrate: + self.log_touched_tables(query, values) if debug: time_end = time() @@ -184,8 +177,7 @@ class Database(object): except Exception as e: if self.is_syntax_error(e): - frappe.errprint("Syntax error in query:") - frappe.errprint(query) + frappe.errprint(f"Syntax error in query:\n{query}") elif self.is_deadlocked(e): raise frappe.QueryDeadlockError(e) @@ -195,7 +187,7 @@ class Database(object): elif frappe.conf.db_type == "postgres": # TODO: added temporarily - print(e) + frappe.errprint(f"Error in query:\n{e}") raise if ignore_ddl and ( @@ -229,21 +221,23 @@ class Database(object): return self._cursor.fetchall() def log_query(self, query, values, debug, explain): + mogrified_query = None + # for debugging in tests if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): - print(self.mogrify(query, values)) + mogrified_query = mogrified_query or self.mogrify(query, values) + print(mogrified_query) # debug if debug: if explain and query.strip().lower().startswith("select"): self.explain_query(query, values) - frappe.errprint(self.mogrify(query, values)) + mogrified_query = mogrified_query or self.mogrify(query, values) + frappe.errprint(mogrified_query) - # info - if (frappe.conf.get("logging") or False) == 2: - frappe.log("<<<< query") - frappe.log(self.mogrify(query, values)) - frappe.log(">>>>") + if frappe.conf.logging == 2: + mogrified_query = mogrified_query or self.mogrify(query, values) + frappe.log(f"<<<< query\n{mogrified_query}\n>>>>") def mogrify(self, query, values): """build the query string with values""" @@ -252,23 +246,22 @@ class Database(object): else: try: return self._cursor.mogrify(query, values) - except: # noqa: E722 + except BaseException: # noqa: E722 return (query, values) def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" try: frappe.errprint("--- query explain ---") - if values is None: - self._cursor.execute("explain " + query) - else: - self._cursor.execute("explain " + query, values) - import json + + explain_query = f"EXPLAIN {query}" + values = values or () + self._cursor.execute(explain_query, values) frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) frappe.errprint("--- query explain end ---") - except Exception: - frappe.errprint("error in query explain") + except Exception as e: + frappe.errprint(f"error in query explain: {e}") def sql_list(self, query, values=(), debug=False, **kwargs): """Return data as list of single elements (first column). @@ -278,7 +271,7 @@ class Database(object): # doctypes = ["DocType", "DocField", "User", ...] doctypes = frappe.db.sql_list("select name from DocType") """ - return [r[0] for r in self.sql(query, values, **kwargs, debug=debug)] + return self.sql(query, values, **kwargs, debug=debug, pluck=True) def sql_ddl(self, query, values=(), debug=False): """Commit and execute a query. DDL (Data Definition Language) queries that alter schema @@ -1193,7 +1186,7 @@ class Database(object): def log_touched_tables(self, query, values=None): if values: - query = frappe.safe_decode(self._cursor.mogrify(query, values)) + query = frappe.safe_decode(self.mogrify(query, values)) if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" From 958fc2b0b2b5dd269fdb8f2ba373d3876dd3deb9 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 25 May 2022 12:26:24 +0530 Subject: [PATCH 026/201] refactor: Database * Change query notations - QB > raw * Update logic of DB APIs - simplify & perf improvements --- frappe/database/database.py | 76 +++++++++++++---------------- frappe/database/mariadb/database.py | 21 ++++++++ 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 2cd950f785..d8f76e76ff 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -22,7 +22,7 @@ from frappe import _ from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.query_builder.utils import DocType -from frappe.utils import cast, get_datetime, getdate, now, sbool +from frappe.utils import cast, get_datetime, get_table_name, getdate, now, sbool from .query import Query @@ -842,13 +842,8 @@ class Database(object): def touch(self, doctype, docname): """Update the modified timestamp of this document.""" modified = now() - self.sql( - """update `tab{doctype}` set `modified`=%s - where name=%s""".format( - doctype=doctype - ), - (modified, docname), - ) + DocType = frappe.qb.DocType(doctype) + frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run() return modified @staticmethod @@ -960,22 +955,11 @@ class Database(object): return self.table_exists(doctype) def get_tables(self, cached=True): - tables = frappe.cache().get_value("db_tables") - if not tables or not cached: - table_rows = self.sql( - """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema NOT IN ('pg_catalog', 'information_schema') - """ - ) - tables = {d[0] for d in table_rows} - frappe.cache().set_value("db_tables", tables) - return tables + raise NotImplementedError def a_row_exists(self, doctype): """Returns True if atleast one row exists.""" - return self.sql("select name from `tab{doctype}` limit 1".format(doctype=doctype)) + return frappe.get_all(doctype, limit=1, order_by=None, as_list=True) def exists(self, dt, dn=None, cache=False): """Return the document name of a matching document, or None. @@ -1047,28 +1031,27 @@ class Database(object): from frappe.utils import now_datetime - return self.sql( - """select count(name) from `tab{doctype}` - where creation >= %s""".format( - doctype=doctype - ), - now_datetime() - relativedelta(minutes=minutes), - )[0][0] + Table = frappe.qb.DocType(doctype) + + return ( + frappe.qb.from_(Table) + .select(Count(Table.name)) + .where(Table.creation >= now_datetime() - relativedelta(minutes=minutes)) + .run()[0][0] + ) def get_db_table_columns(self, table) -> List[str]: """Returns list of column names from given table.""" columns = frappe.cache().hget("table_columns", table) if columns is None: - columns = [ - r[0] - for r in self.sql( - """ - select column_name - from information_schema.columns - where table_name = %s """, - table, - ) - ] + information_schema = frappe.qb.Schema("information_schema") + + columns = ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_name) + .where(information_schema.columns.table_name == table) + .run(pluck=True) + ) if columns: frappe.cache().hset("table_columns", table, columns) @@ -1087,12 +1070,19 @@ class Database(object): return column in self.get_table_columns(doctype) def get_column_type(self, doctype, column): - return self.sql( - """SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = 'tab{0}' AND column_name = '{1}' """.format( - doctype, column + """Returns column type from database.""" + information_schema = frappe.qb.Schema("information_schema") + table = get_table_name(doctype) + + return ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_type) + .where( + (information_schema.columns.table_name == table) + & (information_schema.columns.column_name == column) ) - )[0][0] + .run(pluck=True)[0] + ) def has_index(self, table_name, index_name): raise NotImplementedError diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index a85df44dac..b022a5c821 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -461,3 +461,24 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): def get_database_list(self): return self.sql("SHOW DATABASES", pluck=True) + + def get_tables(self, cached=True): + """Returns list of tables""" + to_query = not cached + + if cached: + tables = frappe.cache().get_value("db_tables") + to_query = not tables + + if to_query: + information_schema = frappe.qb.Schema("information_schema") + + tables = ( + frappe.qb.from_(information_schema.tables) + .select(information_schema.tables.table_name) + .where(information_schema.tables.table_schema != "information_schema") + .run(pluck=True) + ) + frappe.cache().set_value("db_tables", tables) + + return tables From ff9c89450e4f355dd3cbf5f663d3fef57c41db22 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 25 May 2022 13:01:01 +0530 Subject: [PATCH 027/201] feat(db): _transform_query to convert args passed to db.cursor Transform query & parameters based on client requirements. Eg: MariaDB client doesn't behave similar to PyMySQL or Psycopg2. ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-205 --- frappe/database/database.py | 5 +++++ frappe/database/mariadb/database.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/frappe/database/database.py b/frappe/database/database.py index d8f76e76ff..2af890bdb1 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -89,6 +89,9 @@ class Database(object): def get_database_size(self): raise NotImplementedError + def _transform_query(self, query, values): + return query, values + def sql( self, query, @@ -164,6 +167,8 @@ class Database(object): if not isinstance(values, (dict, tuple, list)): values = (values,) + query, values = self._transform_query(query, values) + self.log_query(query, values, debug, explain) self._cursor.execute(query, values) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index b022a5c821..a66a563ed6 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,4 @@ +import re from collections import defaultdict from typing import TYPE_CHECKING, Dict, List, Tuple, Union @@ -14,6 +15,7 @@ from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name if TYPE_CHECKING: from mariadb import ConnectionPool +_PARAM_COMP = re.compile(r"%\([\w]*\)s") _SITE_POOLS = defaultdict(frappe._dict) _MAX_POOL_SIZE = 64 _POOL_SIZE = 1 @@ -222,6 +224,27 @@ class MariaDBConnectionUtil: conn_settings.update(ssl_params) return conn_settings + def _transform_query(self, query: str, values: Dict) -> str: + """Converts a query with named placeholders to a query with %s and values dict to a tuple. + + This is a workaround since the MariaDB Python client (1.0.11) responds inconsistently + depending on the substitions in the query & type of values passed. + + ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-205 + """ + pos_values = [] + named_tokens = _PARAM_COMP.findall(query) + + if len(named_tokens) == len(values): + return query, values + + for token in named_tokens: + key = token[2:-2] + pos_values.append(values[key]) + query = query.replace(token, "%s", 1) + + return query, pos_values + class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): REGEX_CHARACTER = "regexp" From 8ffdc2d4650c80e85c3835285f1ae10185c370a1 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 25 May 2022 14:01:13 +0530 Subject: [PATCH 028/201] refactor(minor): PostgresDatabase's exceptions Focus on Non-blocking Postgres exception checking by: * Use safe getattr to fetch pgcode instead * Use psycopg errorcodes module to use named variables instead of direct codes...for way superior readability xD Also, moved exceptions out of the main class - just code separation, no namespace change. --- frappe/database/postgres/database.py | 124 +++++++++++++++------------ 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index d60a6d1918..fc2a6b7941 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -3,7 +3,17 @@ from typing import List, Tuple, Union import psycopg2 import psycopg2.extensions -from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION +from psycopg2.errorcodes import ( + CLASS_INTEGRITY_CONSTRAINT_VIOLATION, + DEADLOCK_DETECTED, + DUPLICATE_COLUMN, + INSUFFICIENT_PRIVILEGE, + STRING_DATA_RIGHT_TRUNCATION, + UNDEFINED_COLUMN, + UNDEFINED_TABLE, + UNIQUE_VIOLATION, +) +from psycopg2.errors import SyntaxError from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe @@ -21,7 +31,7 @@ DEC2FLOAT = psycopg2.extensions.new_type( psycopg2.extensions.register_type(DEC2FLOAT) -class PostgresDatabase(Database): +class PostgresExceptionUtil: ProgrammingError = psycopg2.ProgrammingError TableMissingError = psycopg2.ProgrammingError OperationalError = psycopg2.OperationalError @@ -29,6 +39,62 @@ class PostgresDatabase(Database): SQLError = psycopg2.ProgrammingError DataError = psycopg2.DataError InterfaceError = psycopg2.InterfaceError + + @staticmethod + def is_deadlocked(e): + return getattr(e, "pgcode", None) == DEADLOCK_DETECTED + + @staticmethod + def is_timedout(e): + # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError + return isinstance(e, psycopg2.extensions.QueryCanceledError) + + @staticmethod + def is_syntax_error(e): + return isinstance(e, SyntaxError) + + @staticmethod + def is_table_missing(e): + return getattr(e, "pgcode", None) == UNDEFINED_TABLE + + @staticmethod + def is_missing_table(e): + return PostgresDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e): + return getattr(e, "pgcode", None) == UNDEFINED_COLUMN + + @staticmethod + def is_access_denied(e): + return getattr(e, "pgcode", None) == INSUFFICIENT_PRIVILEGE + + @staticmethod + def cant_drop_field_or_key(e): + return getattr(e, "pgcode", None) == CLASS_INTEGRITY_CONSTRAINT_VIOLATION + + @staticmethod + def is_duplicate_entry(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION + + @staticmethod + def is_primary_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_pkey" in cstr(e.args[0]) + + @staticmethod + def is_unique_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_key" in cstr(e.args[0]) + + @staticmethod + def is_duplicate_fieldname(e): + return getattr(e, "pgcode", None) == DUPLICATE_COLUMN + + @staticmethod + def is_data_too_long(e): + return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION + + +class PostgresDatabase(PostgresExceptionUtil, Database): REGEX_CHARACTER = "~" # NOTE; The sequence cache for postgres is per connection. @@ -149,60 +215,6 @@ class PostgresDatabase(Database): def is_type_datetime(code): return code == psycopg2.DATETIME - # exception type - @staticmethod - def is_deadlocked(e): - return e.pgcode == "40P01" - - @staticmethod - def is_timedout(e): - # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError - return isinstance(e, psycopg2.extensions.QueryCanceledError) - - @staticmethod - def is_syntax_error(e): - return isinstance(e, psycopg2.errors.SyntaxError) - - @staticmethod - def is_table_missing(e): - return getattr(e, "pgcode", None) == "42P01" - - @staticmethod - def is_missing_table(e): - return PostgresDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e): - return getattr(e, "pgcode", None) == "42703" - - @staticmethod - def is_access_denied(e): - return e.pgcode == "42501" - - @staticmethod - def cant_drop_field_or_key(e): - return e.pgcode.startswith("23") - - @staticmethod - def is_duplicate_entry(e): - return e.pgcode == "23505" - - @staticmethod - def is_primary_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_pkey" in cstr(e.args[0]) - - @staticmethod - def is_unique_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_key" in cstr(e.args[0]) - - @staticmethod - def is_duplicate_fieldname(e): - return e.pgcode == "42701" - - @staticmethod - def is_data_too_long(e): - return e.pgcode == STRING_DATA_RIGHT_TRUNCATION - def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: old_name = get_table_name(old_name) new_name = get_table_name(new_name) From 5a29177e6b4c6b145d6e5c81422445c91e4dde4b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 17:42:52 +0530 Subject: [PATCH 029/201] fix(db): Log queried tables through generated query --- frappe/database/database.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index cd53c3e1fd..181eda98ab 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -189,9 +189,6 @@ class Database(object): self._cursor.execute(query, values) - if frappe.flags.in_migrate: - self.log_touched_tables(query, values) - if debug: time_end = time() frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) @@ -263,6 +260,9 @@ class Database(object): mogrified_query = mogrified_query or self.mogrify(query, values) frappe.log(f"<<<< query\n{mogrified_query}\n>>>>") + if frappe.flags.in_migrate: + self.log_touched_tables(mogrified_query or query) + def mogrify(self, query, values): """build the query string with values""" if not values: @@ -1217,9 +1217,7 @@ class Database(object): else: return None - def log_touched_tables(self, query, values=None): - if values: - query = frappe.safe_decode(self._cursor.mogrify(query, values)) + def log_touched_tables(self, query): if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" From ded55fd98e6aff5829c049ee3d2852a181eb315e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 17:43:44 +0530 Subject: [PATCH 030/201] fix(db): Skip transformation of query if no named params found --- frappe/database/mariadb/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index a66a563ed6..79378f0d43 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -235,7 +235,7 @@ class MariaDBConnectionUtil: pos_values = [] named_tokens = _PARAM_COMP.findall(query) - if len(named_tokens) == len(values): + if not named_tokens or len(named_tokens) == len(values): return query, values for token in named_tokens: From bfd51aa43a9c2d9a13e319133f29eca1d16c0d6d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jun 2022 17:44:29 +0530 Subject: [PATCH 031/201] fix(qb): Use fallback og table if not found in schema mapper --- frappe/query_builder/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index d2fdeab324..4d3702e228 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -76,7 +76,7 @@ class Postgres(Base, PostgreSQLQuery): if isinstance(table, Table): if table._schema: if table._schema._name == "information_schema": - table = cls.schema_translation[table._table_name] + table = cls.schema_translation.get(table._table_name) or table elif isinstance(table, str): table = cls.DocType(table) From 0adf5e127a91eade38c88da6119365ce554c667f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 12:31:14 +0530 Subject: [PATCH 032/201] fix(db): Track query engine under db._filter_engine --- frappe/database/database.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 181eda98ab..5a84a350a8 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,14 +1,12 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# Database Module -# -------------------- - import datetime import json import random import re import string +import traceback from contextlib import contextmanager from time import time from typing import Dict, List, Optional, Tuple, Union @@ -75,15 +73,16 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} + # self.last_query last sql query executed @property def query(self): - if not hasattr(self, "_query"): + if not hasattr(self, "_filter_engine"): from .query import Query - self._query = Query() + self._filter_engine = Query() del Query - return self._query + return self._filter_engine def setup_type_map(self): pass @@ -205,8 +204,6 @@ class Database(object): elif frappe.conf.db_type == "postgres": # TODO: added temporarily - import traceback - traceback.print_stack() frappe.errprint(f"Error in query:\n{e}") raise From 889ced30357b2f0cae2fb91b3cdaf9f8f92066e1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 12:32:26 +0530 Subject: [PATCH 033/201] refactor: frappe.db.sql * Move everything except _cursor.execute outside try-except block - This caused multiple traceback printing (by recursion of db.sql) * Include values mogrifying & executing via client alone in execution time for query * Reduce indentations lol --- frappe/database/database.py | 44 +++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 5a84a350a8..554612b083 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -166,32 +166,23 @@ class Database(object): # in transaction validations self.check_transaction_status(query) - self.clear_db_table_cache(query) - # autocommit if auto_commit: self.commit() - # execute + if debug: + time_start = time() + + if values: + if not isinstance(values, (tuple, dict, list)): + values = (values,) + query, values = self._transform_query(query, values) + else: + values = None + try: - if debug: - time_start = time() - - if values != (): - if not isinstance(values, (dict, tuple, list)): - values = (values,) - - query, values = self._transform_query(query, values) - - self.log_query(query, values, debug, explain) - self._cursor.execute(query, values) - - if debug: - time_end = time() - frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) - except Exception as e: if self.is_syntax_error(e): frappe.errprint(f"Syntax error in query:\n{query}") @@ -202,19 +193,24 @@ class Database(object): elif self.is_timedout(e): raise frappe.QueryTimeoutError(e) + # TODO: added temporarily elif frappe.conf.db_type == "postgres": - # TODO: added temporarily traceback.print_stack() frappe.errprint(f"Error in query:\n{e}") raise - if ignore_ddl and ( - self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e) + if not ( + ignore_ddl + and (self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e)) ): - pass - else: raise + if debug: + time_end = time() + frappe.errprint(f"Execution time: {time_end - time_start:.2f} sec") + + self.log_query(query, values, debug, explain) + if auto_commit: self.commit() From effa942f4c35f922fb79525955602746aeae5c5d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 13:26:04 +0530 Subject: [PATCH 034/201] refactor: frappe.db.log_query * Mogrify queries and set them as frappe.db.last_query instead of directly interfacing with the clients * This is required for now as the MariaDB client uses binary protocol to talk to the server and doesn't build the queries itself * Add typing hints * Imported Query object as FilterEngine - Query is too ambiguous lol ref: https://jira.mariadb.org/browse/CONPY-208?focusedCommentId=226873&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-226873 --- frappe/database/database.py | 71 +++++++++++++++------------- frappe/database/mariadb/database.py | 4 ++ frappe/database/postgres/database.py | 4 ++ 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 554612b083..7a4953c333 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -9,6 +9,7 @@ import string import traceback from contextlib import contextmanager from time import time +from types import NoneType from typing import Dict, List, Optional, Tuple, Union from pypika.terms import Criterion, NullValue, PseudoColumn @@ -17,6 +18,7 @@ import frappe import frappe.defaults import frappe.model.meta from frappe import _ +from frappe.database.query import Query as FilterEngine from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count @@ -29,6 +31,9 @@ INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') +Query = Union[str, frappe.qb] +QueryValues = Union[Tuple, List, Dict, NoneType] + def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) @@ -78,10 +83,7 @@ class Database(object): @property def query(self): if not hasattr(self, "_filter_engine"): - from .query import Query - - self._filter_engine = Query() - del Query + self._filter_engine = FilterEngine() return self._filter_engine def setup_type_map(self): @@ -105,13 +107,13 @@ class Database(object): def get_database_size(self): raise NotImplementedError - def _transform_query(self, query, values): + def _transform_query(self, query: Query, values: QueryValues): return query, values def sql( self, - query, - values=(), + query: Query, + values: QueryValues = None, as_dict=0, as_list=0, formatted=0, @@ -127,7 +129,7 @@ class Database(object): """Execute a SQL query and fetch all rows. :param query: SQL query. - :param values: List / dict of values to be escaped and substituted in the query. + :param values: Tuple / List / Dict of values to be escaped and substituted in the query. :param as_dict: Return as a dictionary. :param as_list: Always return as a list. :param formatted: Format values like date etc. @@ -234,51 +236,56 @@ class Database(object): else: return self._cursor.fetchall() - def log_query(self, query, values, debug, explain): - mogrified_query = None - - # for debugging in tests + def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: + """Takes the query and logs it to various interfaces according to the settings.""" if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): - mogrified_query = mogrified_query or self.mogrify(query, values) print(mogrified_query) - # debug if debug: - if explain and is_query_type(query, "select"): - self.explain_query(query, values) - mogrified_query = mogrified_query or self.mogrify(query, values) + if explain and is_query_type(mogrified_query, "select"): + self.explain_query(mogrified_query) frappe.errprint(mogrified_query) if frappe.conf.logging == 2: - mogrified_query = mogrified_query or self.mogrify(query, values) frappe.log(f"<<<< query\n{mogrified_query}\n>>>>") if frappe.flags.in_migrate: - self.log_touched_tables(mogrified_query or query) + self.log_touched_tables(mogrified_query) + + def log_query( + self, query: str, values: QueryValues = None, debug: bool = False, explain: bool = False + ) -> str: + # TODO: Use mogrify until MariaDB Connector/C 1.1 is released and we can fetch something + # like cursor._transformed_statement from the cursor object. We can also avoid setting + # mogrified_query if we don't need to log it. + mogrified_query = self.mogrify(query, values) + self._log_query(mogrified_query, debug, explain) + return mogrified_query def mogrify(self, query, values): """build the query string with values""" if not values: return query - else: - try: - return self._cursor.mogrify(query, values) - except BaseException: # noqa: E722 - return (query, values) + + try: + return self._cursor.mogrify(query, values) + except BaseException: # noqa: E722 + if isinstance(values, dict): + return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} + elif isinstance(values, (list, tuple)): + return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) + return (query, values) def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" + frappe.errprint("--- query explain ---") try: - frappe.errprint("--- query explain ---") - - explain_query = f"EXPLAIN {query}" - values = values or () - self._cursor.execute(explain_query, values) - - frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) - frappe.errprint("--- query explain end ---") + self._cursor.execute(f"EXPLAIN {query}", values) except Exception as e: frappe.errprint(f"error in query explain: {e}") + else: + frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) + frappe.errprint("--- query explain end ---") def sql_list(self, query, values=(), debug=False, **kwargs): """Return data as list of single elements (first column). diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 79378f0d43..da33007d5d 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -311,6 +311,10 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): return db_size[0].get("database_size") + def log_query(self, query, values, debug, explain): + self.last_query = super().log_query(query, values, debug, explain) + return self.last_query + @staticmethod def escape(s, percent=True): """Excape quotes and percent in given string.""" diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 0bcc2adcb2..2444d41279 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -147,6 +147,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database): "JSON": ("json", ""), } + @property + def last_query(self): + return self._cursor.query + def get_connection(self): conn = psycopg2.connect( "host='{}' dbname='{}' user='{}' password='{}' port={}".format( From 154d794c6c10327085303fd9a39ae8b28129f5a9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 13:29:35 +0530 Subject: [PATCH 035/201] fix: Recorder to use frappe.db.last_query to pick out last executed stmt --- frappe/recorder.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/frappe/recorder.py b/frappe/recorder.py index 87e001fe31..ff64c082bc 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -12,6 +12,7 @@ import sqlparse import frappe from frappe import _ +from frappe.database.database import is_query_type RECORDER_INTERCEPT_FLAG = "recorder-intercept" RECORDER_REQUEST_SPARSE_HASH = "recorder-requests-sparse" @@ -25,18 +26,13 @@ def sql(*args, **kwargs): end_time = time.time() stack = list(get_current_stack_frames()) - - if frappe.db.db_type == "postgres": - query = frappe.db._cursor.query - else: - query = frappe.db._cursor._executed - - query = sqlparse.format(query.strip(), keyword_case="upper", reindent=True) + last_query = frappe.db.last_query + query = sqlparse.format(last_query.strip(), keyword_case="upper", reindent=True) # Collect EXPLAIN for executed query - if query.lower().strip().split()[0] in ("select", "update", "delete"): + if is_query_type(query, ("select", "update", "delete")): # Only SELECT/UPDATE/DELETE queries can be "EXPLAIN"ed - explain_result = frappe.db._sql("EXPLAIN {}".format(query), as_dict=True) + explain_result = frappe.db._sql(f"EXPLAIN {query}", as_dict=True) else: explain_result = [] From bc3780560eb8db43ad2afafcea9c52ebf0f8c6f2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 14:07:10 +0530 Subject: [PATCH 036/201] perf: Use lazy mogrified query for logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For parameterized queries, there's an improvement of ~30% in query execution via frappe.db.sql - from 58.6 µs ± 2.37 µs to 44.6 µs ± 1.56 µs --- frappe/database/database.py | 38 ++++++++++++++++++--------------- frappe/database/utils.py | 42 +++++++++++++++++++++++++++++++++++++ frappe/recorder.py | 3 +-- 3 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 frappe/database/utils.py diff --git a/frappe/database/database.py b/frappe/database/database.py index 7a4953c333..1bba2c871e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -19,6 +19,7 @@ import frappe.defaults import frappe.model.meta from frappe import _ from frappe.database.query import Query as FilterEngine +from frappe.database.utils import LazyMogrify, Query, QueryValues, is_query_type from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count @@ -31,13 +32,6 @@ INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') -Query = Union[str, frappe.qb] -QueryValues = Union[Tuple, List, Dict, NoneType] - - -def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool: - return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) - class Database(object): """ @@ -78,7 +72,7 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} - # self.last_query last sql query executed + # self.last_query lazy attribute of last sql query executed @property def query(self): @@ -238,19 +232,25 @@ class Database(object): def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: """Takes the query and logs it to various interfaces according to the settings.""" - if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): - print(mogrified_query) + _query = None + + if frappe.conf.allow_tests and frappe.cache().get_value("flag_print_sql"): + _query = _query or str(mogrified_query) + print(_query) if debug: - if explain and is_query_type(mogrified_query, "select"): - self.explain_query(mogrified_query) - frappe.errprint(mogrified_query) + _query = _query or str(mogrified_query) + if explain and is_query_type(_query, "select"): + self.explain_query(_query) + frappe.errprint(_query) if frappe.conf.logging == 2: - frappe.log(f"<<<< query\n{mogrified_query}\n>>>>") + _query = _query or str(mogrified_query) + frappe.log(f"<<<< query\n{_query}\n>>>>") if frappe.flags.in_migrate: - self.log_touched_tables(mogrified_query) + _query = _query or str(mogrified_query) + self.log_touched_tables(_query) def log_query( self, query: str, values: QueryValues = None, debug: bool = False, explain: bool = False @@ -258,11 +258,11 @@ class Database(object): # TODO: Use mogrify until MariaDB Connector/C 1.1 is released and we can fetch something # like cursor._transformed_statement from the cursor object. We can also avoid setting # mogrified_query if we don't need to log it. - mogrified_query = self.mogrify(query, values) + mogrified_query = self.lazy_mogrify(query, values) self._log_query(mogrified_query, debug, explain) return mogrified_query - def mogrify(self, query, values): + def mogrify(self, query: Query, values: QueryValues): """build the query string with values""" if not values: return query @@ -276,6 +276,10 @@ class Database(object): return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) return (query, values) + def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify: + """Wrap the object with str to generate mogrified query.""" + return LazyMogrify(query, values) + def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" frappe.errprint("--- query explain ---") diff --git a/frappe/database/utils.py b/frappe/database/utils.py new file mode 100644 index 0000000000..47a5222ede --- /dev/null +++ b/frappe/database/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + +from functools import cached_property +from types import NoneType +from typing import Dict, List, Tuple, Union + +import frappe +from frappe.query_builder.builder import MariaDB, Postgres + +Query = Union[str, MariaDB, Postgres] +QueryValues = Union[Tuple, List, Dict, NoneType] + + +def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool: + return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) + + +class LazyString: + def _setup(self) -> None: + raise NotImplementedError + + @cached_property + def value(self) -> str: + return self._setup() + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return f"'{self.value}'" + + +class LazyMogrify(LazyString): + __slots__ = () + + def __init__(self, query, values) -> None: + self.query = query + self.values = values + + def _setup(self) -> str: + return frappe.db.mogrify(self.query, self.values) diff --git a/frappe/recorder.py b/frappe/recorder.py index ff64c082bc..3ecf2b2b96 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -26,8 +26,7 @@ def sql(*args, **kwargs): end_time = time.time() stack = list(get_current_stack_frames()) - last_query = frappe.db.last_query - query = sqlparse.format(last_query.strip(), keyword_case="upper", reindent=True) + query = sqlparse.format(str(frappe.db.last_query).strip(), keyword_case="upper", reindent=True) # Collect EXPLAIN for executed query if is_query_type(query, ("select", "update", "delete")): From 25b87a9d497887dd0a07e0e473679b78b619f6ee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 15:04:35 +0530 Subject: [PATCH 037/201] chore: NoneType alias for < PY310 --- frappe/database/database.py | 1 - frappe/database/utils.py | 6 +++++- frappe/model/rename_doc.py | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1bba2c871e..6dd3306b18 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -9,7 +9,6 @@ import string import traceback from contextlib import contextmanager from time import time -from types import NoneType from typing import Dict, List, Optional, Tuple, Union from pypika.terms import Criterion, NullValue, PseudoColumn diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 47a5222ede..0edbc07f1d 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -2,7 +2,11 @@ # License: MIT. See LICENSE from functools import cached_property -from types import NoneType + +try: + from types import NoneType +except ImportError: + NoneType = type(None) from typing import Dict, List, Tuple, Union import frappe diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 25e471d4b0..652703aed5 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -1,5 +1,9 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +try: + from types import NoneType +except ImportError: + NoneType = type(None) from typing import TYPE_CHECKING, Dict, List, Optional import frappe @@ -46,7 +50,7 @@ def update_document_title( # TODO: omit this after runtime type checking (ref: https://github.com/frappe/frappe/pull/14927) for obj in [docname, updated_title, updated_name]: - if not isinstance(obj, (str, type(None))): + if not isinstance(obj, (str, NoneType)): frappe.throw(f"{obj=} must be of type str or None") # handle bad API usages From 1a772e304c194fc91d5efebfd0ba5fd9c367a79c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 16:01:38 +0530 Subject: [PATCH 038/201] fix(db): Store result of last executed query under frappe.db.last_result --- frappe/database/database.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6dd3306b18..414e0c9358 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -212,8 +212,10 @@ class Database(object): if not self._cursor.description: return () + self.last_result = self._cursor.fetchall() + if pluck: - return [r[0] for r in self._cursor.fetchall()] + return [r[0] for r in self.last_result] # scrub output if required if as_dict: @@ -223,11 +225,11 @@ class Database(object): r.update(update) return ret elif as_list: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) + return self.convert_to_lists(self.last_result, formatted, as_utf8) elif as_utf8: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) + return self.convert_to_lists(self.last_result, formatted, as_utf8) else: - return self._cursor.fetchall() + return self.last_result def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: """Takes the query and logs it to various interfaces according to the settings.""" @@ -335,7 +337,7 @@ class Database(object): def fetch_as_dict(self, formatted=0, as_utf8=0): """Internal. Converts results to dict.""" - result = self._cursor.fetchall() + result = self.last_result ret = [] if result: keys = [column[0] for column in self._cursor.description] From 355e997045c78932c425aba6b20f595d40638f24 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 16:21:32 +0530 Subject: [PATCH 039/201] test: Get rid of magical forgiving behaviour of pymysql ("test",) would be interpretted as ("test") which would be just "test". MariaDB client doesn't handle errenous inputs like this that PyMySQL tolerated --- frappe/email/doctype/notification/test_notification.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 4d8b26c559..039b6db2f0 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -84,7 +84,7 @@ class TestNotification(unittest.TestCase): def test_condition(self): """Check notification is triggered based on a condition.""" event = frappe.new_doc("Event") - event.subject = ("test",) + event.subject = "test" event.event_type = "Private" event.starts_on = "2014-06-06 12:00:00" event.insert() @@ -137,7 +137,7 @@ class TestNotification(unittest.TestCase): def test_value_changed(self): event = frappe.new_doc("Event") - event.subject = ("test",) + event.subject = "test" event.event_type = "Private" event.starts_on = "2014-06-06 12:00:00" event.insert() @@ -186,7 +186,7 @@ class TestNotification(unittest.TestCase): frappe.db.commit() event = frappe.new_doc("Event") - event.subject = ("test-2",) + event.subject = "test-2" event.event_type = "Private" event.starts_on = "2014-06-06 12:00:00" event.insert() @@ -200,9 +200,8 @@ class TestNotification(unittest.TestCase): event.delete() def test_date_changed(self): - event = frappe.new_doc("Event") - event.subject = ("test",) + event.subject = "test" event.event_type = "Private" event.starts_on = "2014-01-01 12:00:00" event.insert() From 71ed8417d3f3683077f633992999e027b53dde98 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jun 2022 16:23:27 +0530 Subject: [PATCH 040/201] test: frappe.db.describe returns List[Tuple] through mariadb --- frappe/tests/test_db.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 73b5446404..d6cdbdd655 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -517,10 +517,9 @@ class TestDDLCommandsMaria(unittest.TestCase): test_table_name = "TestNotes" def setUp(self) -> None: - frappe.db.commit() - frappe.db.sql( + frappe.db.sql_ddl( f""" - CREATE TABLE `tab{self.test_table_name}` (`id` INT NULL, content TEXT, PRIMARY KEY (`id`)); + CREATE TABLE IF NOT EXISTS `tab{self.test_table_name}` (`id` INT NULL, content TEXT, PRIMARY KEY (`id`)); """ ) @@ -545,10 +544,10 @@ class TestDDLCommandsMaria(unittest.TestCase): def test_describe(self) -> None: self.assertEqual( - ( + [ ("id", "int(11)", "NO", "PRI", None, ""), ("content", "text", "YES", "", None, ""), - ), + ], frappe.db.describe(self.test_table_name), ) From a03bf6b0bba75b5936627aec9e85b8fadc48f855 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jun 2022 11:31:55 +0530 Subject: [PATCH 041/201] fix(db): Transform values only if not None --- frappe/database/database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 414e0c9358..68ab33d4f6 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -169,12 +169,10 @@ class Database(object): if debug: time_start = time() - if values: + if values != None: if not isinstance(values, (tuple, dict, list)): values = (values,) query, values = self._transform_query(query, values) - else: - values = None try: self._cursor.execute(query, values) From ca0016a996b7ed4dfb83b6f7f324037780fe3bee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jun 2022 11:32:40 +0530 Subject: [PATCH 042/201] test(sequence): Use mariadb client's exception handling --- frappe/tests/test_sequence.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/tests/test_sequence.py b/frappe/tests/test_sequence.py index a60e4b1ac9..82bb8ab257 100644 --- a/frappe/tests/test_sequence.py +++ b/frappe/tests/test_sequence.py @@ -1,5 +1,4 @@ import psycopg2 -import pymysql import frappe from frappe.tests.utils import FrappeTestCase @@ -35,10 +34,10 @@ class TestSequence(FrappeTestCase): try: frappe.db.get_next_sequence_val(seq_name) - except pymysql.err.OperationalError as e: - self.assertEqual(e.args[0], 4084) except psycopg2.errors.SequenceGeneratorLimitExceeded: pass + except frappe.db.ProgrammingError as e: + self.assertEqual(getattr(e, "errno", None), 4084) else: self.fail("NEXTVAL didn't raise any error upon sequence's end") From 90c716bce0de7710f7f63d6ea42ff01df10f4061 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jun 2022 12:10:43 +0530 Subject: [PATCH 043/201] fix(sequence): Setup & use SequenceGeneratorLimitExceeded error --- frappe/database/mariadb/database.py | 4 ++++ frappe/database/postgres/database.py | 3 ++- frappe/database/sequence.py | 17 +++++++++++------ frappe/tests/test_sequence.py | 6 +----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index da33007d5d..327c995d48 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -46,6 +46,10 @@ class MariaDBExceptionUtil: SQLError = mariadb.ProgrammingError DataError = mariadb.DataError + # match ER_SEQUENCE_RUN_OUT - https://mariadb.com/kb/en/mariadb-error-codes/ + SequenceGeneratorLimitExceeded = mariadb.ProgrammingError + SequenceGeneratorLimitExceeded.errno = 4084 + @staticmethod def is_deadlocked(e: mariadb.Error) -> bool: return getattr(e, "errno", None) == ER.LOCK_DEADLOCK diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 2444d41279..f5ecabd6c4 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -13,7 +13,7 @@ from psycopg2.errorcodes import ( UNDEFINED_TABLE, UNIQUE_VIOLATION, ) -from psycopg2.errors import SyntaxError +from psycopg2.errors import SequenceGeneratorLimitExceeded, SyntaxError from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe @@ -44,6 +44,7 @@ class PostgresExceptionUtil: SQLError = psycopg2.ProgrammingError DataError = psycopg2.DataError InterfaceError = psycopg2.InterfaceError + SequenceGeneratorLimitExceeded = SequenceGeneratorLimitExceeded @staticmethod def is_deadlocked(e): diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py index 6a352d20d1..54362a5895 100644 --- a/frappe/database/sequence.py +++ b/frappe/database/sequence.py @@ -57,12 +57,17 @@ def create_sequence( def get_next_val(doctype_name: str, slug: str = "_id_seq") -> int: - return db.multisql( - { - "postgres": f"select nextval('\"{scrub(doctype_name + slug)}\"')", - "mariadb": f"select nextval(`{scrub(doctype_name + slug)}`)", - } - )[0][0] + sequence_name = scrub(f"{doctype_name}{slug}") + + if db.db_type == "postgres": + sequence_name = f"'\"{sequence_name}\"'" + elif db.db_type == "mariadb": + sequence_name = f"`{sequence_name}`" + + try: + return db.sql(f"SELECT nextval({sequence_name})")[0][0] + except IndexError: + raise db.SequenceGeneratorLimitExceeded def set_next_val( diff --git a/frappe/tests/test_sequence.py b/frappe/tests/test_sequence.py index 82bb8ab257..c6ea0bc8c0 100644 --- a/frappe/tests/test_sequence.py +++ b/frappe/tests/test_sequence.py @@ -1,5 +1,3 @@ -import psycopg2 - import frappe from frappe.tests.utils import FrappeTestCase @@ -34,10 +32,8 @@ class TestSequence(FrappeTestCase): try: frappe.db.get_next_sequence_val(seq_name) - except psycopg2.errors.SequenceGeneratorLimitExceeded: + except frappe.db.SequenceGeneratorLimitExceeded: pass - except frappe.db.ProgrammingError as e: - self.assertEqual(getattr(e, "errno", None), 4084) else: self.fail("NEXTVAL didn't raise any error upon sequence's end") From 3af8d5caea0b667d7028e5cb23b6fd993778eb5a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jun 2022 12:47:44 +0530 Subject: [PATCH 044/201] fix: Add fallbacks for values Psycopg seems to like None over () and MariaDB - PyMySQL can't seem to agree on anything - so this seems to keep everyone happy...a very delicate balance :crie: --- frappe/database/database.py | 5 +++-- frappe/database/postgres/database.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 68ab33d4f6..b6fc6f0fa8 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -71,7 +71,8 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} - # self.last_query lazy attribute of last sql query executed + # self.db_type: str + # self.last_query (lazy) attribute of last sql query executed @property def query(self): @@ -101,7 +102,7 @@ class Database(object): raise NotImplementedError def _transform_query(self, query: Query, values: QueryValues): - return query, values + return query, values or None def sql( self, diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index f5ecabd6c4..0d4a177741 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -193,6 +193,9 @@ class PostgresDatabase(PostgresExceptionUtil, Database): modify_query(query), modify_values(values), *args, **kwargs ) + def lazy_mogrify(self, *args, **kwargs) -> str: + return self.last_query + def get_tables(self, cached=True): return [ d[0] From 9b6a048bcd9b17d9ceec9cf0fa3cbf7aae2e1abb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jun 2022 12:49:12 +0530 Subject: [PATCH 045/201] refactor(minor): Use db.db_type instead of conf.db_type --- frappe/database/database.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index b6fc6f0fa8..8e5e19bbf0 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -188,7 +188,7 @@ class Database(object): raise frappe.QueryTimeoutError(e) # TODO: added temporarily - elif frappe.conf.db_type == "postgres": + elif self.db_type == "postgres": traceback.print_stack() frappe.errprint(f"Error in query:\n{e}") raise @@ -949,7 +949,7 @@ class Database(object): frappe.call(method[0], *(method[1] or []), **(method[2] or {})) self.sql("commit") - if frappe.conf.db_type == "postgres": + if self.db_type == "postgres": # Postgres requires explicitly starting new transaction self.begin() @@ -1187,7 +1187,7 @@ class Database(object): return self.is_missing_column(e) or self.is_table_missing(e) def multisql(self, sql_dict, values=(), **kwargs): - current_dialect = frappe.db.db_type or "mariadb" + current_dialect = self.db_type or "mariadb" query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) @@ -1260,9 +1260,9 @@ class Database(object): query = frappe.qb.into(table) if ignore_duplicates: # Pypika does not have same api for ignoring duplicates - if frappe.conf.db_type == "mariadb": + if self.db_type == "mariadb": query = query.ignore() - elif frappe.conf.db_type == "postgres": + elif self.db_type == "postgres": query = query.on_conflict().do_nothing() values_to_insert = values[start_index : start_index + chunk_size] From 1f1d91a0565459c67de6f4116a0d356452fb0415 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 18 Jun 2022 14:04:38 +0530 Subject: [PATCH 046/201] fix: Make postgres' last_query lazy decodable --- frappe/database/postgres/database.py | 3 ++- frappe/database/utils.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 0d4a177741..414cac1a3f 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -19,6 +19,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable +from frappe.database.utils import LazyDecode from frappe.utils import cstr, get_table_name # cast decimals as floats @@ -150,7 +151,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): @property def last_query(self): - return self._cursor.query + return LazyDecode(self._cursor.query) def get_connection(self): conn = psycopg2.connect( diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 0edbc07f1d..6da4aade4a 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -35,6 +35,16 @@ class LazyString: return f"'{self.value}'" +class LazyDecode(LazyString): + __slots__ = () + + def __init__(self, value: str) -> None: + self._value = value + + def _setup(self) -> None: + return self._value.decode() + + class LazyMogrify(LazyString): __slots__ = () From 48243346e3832a664689a131f657b534bddb72ab Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 20 Jun 2022 13:00:24 +0530 Subject: [PATCH 047/201] fix(get_contact_list): Don't pass conditions as query value * Conditions passed are not valid prepared statement values. They can be passed as string substitution since they're generated by DBQuery. * Added typing hints & other improvements * Removed seemingly pointless try-except block --- frappe/desk/reportview.py | 3 +-- frappe/email/__init__.py | 37 +++++++++++++++++-------------------- frappe/model/db_query.py | 4 ++-- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index d6dce68399..c7295730de 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -684,8 +684,7 @@ def build_match_conditions(doctype, user=None, as_condition=True): ) if as_condition: return match_conditions.replace("%", "%%") - else: - return match_conditions + return match_conditions def get_filters_cond( diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index fae60baebf..51917cc7af 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from typing import Dict, List + import frappe from frappe.desk.reportview import build_match_conditions @@ -10,31 +12,26 @@ def sendmail_to_system_managers(subject, content): @frappe.whitelist() -def get_contact_list(txt, page_length=20): +def get_contact_list(txt, page_length=20) -> List[Dict]: """Returns contacts (from autosuggest)""" - cached_contacts = get_cached_contacts(txt) - if cached_contacts: + if cached_contacts := get_cached_contacts(txt): return cached_contacts[:page_length] - try: - match_conditions = build_match_conditions("Contact") - match_conditions = "and {0}".format(match_conditions) if match_conditions else "" + reportview_conditions = build_match_conditions("Contact") + match_conditions = f"and {reportview_conditions}" if reportview_conditions else "" - out = frappe.db.sql( - """select email_id as value, - concat(first_name, ifnull(concat(' ',last_name), '' )) as description - from tabContact - where name like %(txt)s or email_id like %(txt)s - %(condition)s - limit %(page_length)s""", - {"txt": "%" + txt + "%", "condition": match_conditions, "page_length": page_length}, - as_dict=True, - ) - out = filter(None, out) - - except: - raise + out = frappe.db.sql( + f"""select email_id as value, + concat(first_name, ifnull(concat(' ',last_name), '' )) as description + from tabContact + where name like %(txt)s or email_id like %(txt)s + {match_conditions} + limit %(page_length)s""", + {"txt": f"%{txt}%", "page_length": page_length}, + as_dict=True, + ) + out = list(filter(None, out)) update_contact_cache(out) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 82913db98d..dfe5304a87 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -6,7 +6,7 @@ import copy import json import re from datetime import datetime -from typing import List +from typing import List, Union import frappe import frappe.defaults @@ -720,7 +720,7 @@ class DatabaseQuery(object): return condition - def build_match_conditions(self, as_condition=True): + def build_match_conditions(self, as_condition=True) -> Union[str, List]: """add match conditions if applicable""" self.match_filters = [] self.match_conditions = [] From a58a5bb848ff22959a6253e68a57411206bd22fd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 20 Jun 2022 13:06:05 +0530 Subject: [PATCH 048/201] fix: Improve _transform_query unique key identification --- frappe/database/database.py | 4 ++-- frappe/database/mariadb/database.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 8e5e19bbf0..73460614cb 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -170,7 +170,7 @@ class Database(object): if debug: time_start = time() - if values != None: + if values is not None: if not isinstance(values, (tuple, dict, list)): values = (values,) query, values = self._transform_query(query, values) @@ -179,7 +179,7 @@ class Database(object): self._cursor.execute(query, values) except Exception as e: if self.is_syntax_error(e): - frappe.errprint(f"Syntax error in query:\n{query}") + frappe.errprint(f"Syntax error in query:\n{query} {values}") elif self.is_deadlocked(e): raise frappe.QueryDeadlockError(e) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 327c995d48..1ca8e821fb 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -239,7 +239,7 @@ class MariaDBConnectionUtil: pos_values = [] named_tokens = _PARAM_COMP.findall(query) - if not named_tokens or len(named_tokens) == len(values): + if not named_tokens or len(set(named_tokens)) == len(values): return query, values for token in named_tokens: From 14003e5ac957dbfd31c3d122ea8b25bd0856e309 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 20 Jun 2022 15:31:15 +0530 Subject: [PATCH 049/201] refactor: DISABLE_DATABASE_CONNECTION_POOLING conf + var name --- frappe/__init__.py | 2 +- frappe/database/mariadb/database.py | 6 +++--- frappe/utils/bench_helper.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 542c783319..45c5ca500e 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -43,7 +43,7 @@ __title__ = "Frappe Framework" controllers = {} local = Local() STANDARD_USERS = ("Guest", "Administrator") -DISABLE_DATABASE_POOLING = None +DISABLE_DATABASE_CONNECTION_POOLING = None _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) _qb_patched = {} diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1ca8e821fb..c444e492a2 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -33,9 +33,9 @@ def is_connection_pooling_enabled() -> bool: process. This will override config key `disable_database_connection_pooling`. Set key `disable_database_connection_pooling` in site config for persistent settings across workers.""" - if frappe.DISABLE_DATABASE_POOLING is not None: - return not frappe.DISABLE_DATABASE_POOLING - return frappe.local.conf.disable_database_connection_pooling + if frappe.DISABLE_DATABASE_CONNECTION_POOLING is not None: + return not frappe.DISABLE_DATABASE_CONNECTION_POOLING + return not frappe.local.conf.disable_database_connection_pooling class MariaDBExceptionUtil: diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index 8de10b73d4..8d109d9737 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -110,6 +110,6 @@ if __name__ == "__main__": # disable pooling for commands executed via bench unless explicitly stated otherwise # - except for commands serve & worker if not {"serve", "worker"} & set(sys.argv) and int(os.environ.get("DATABASE_POOLING", 0)): - frappe.DISABLE_DATABASE_POOLING = True + frappe.DISABLE_DATABASE_CONNECTION_POOLING = True main() From f08f29b8a347a103e92832785827d9fc389c7209 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jun 2022 11:44:59 +0530 Subject: [PATCH 050/201] Revert "fix: Disable db pooling on all commands by default" This reverts commit 0a8941c58317956e0661f90fa2357e39674f63c3 since it may not be required as we start pooling with single connections only. --- frappe/commands/site.py | 2 ++ frappe/utils/bench_helper.py | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 13d702d6f3..4db143b076 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -69,6 +69,8 @@ def new_site( "Create a new site" from frappe.installer import _new_site + frappe.DISABLE_DATABASE_CONNECTION_POOLING = True + frappe.init(site=site, new_site=True) _new_site( diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index 8d109d9737..a0b011acc1 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -1,7 +1,6 @@ import importlib import json import os -import sys import traceback import warnings @@ -107,9 +106,4 @@ if __name__ == "__main__": if not frappe._dev_server: warnings.simplefilter("ignore") - # disable pooling for commands executed via bench unless explicitly stated otherwise - # - except for commands serve & worker - if not {"serve", "worker"} & set(sys.argv) and int(os.environ.get("DATABASE_POOLING", 0)): - frappe.DISABLE_DATABASE_CONNECTION_POOLING = True - main() From ef078a4ab56379e106433848bd425da9e4e27c21 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jun 2022 19:40:17 +0530 Subject: [PATCH 051/201] refactor(db-read_only): Track conn type in Database instance --- frappe/__init__.py | 4 +++- frappe/database/__init__.py | 10 +++++++--- frappe/database/database.py | 12 +++++++++++- frappe/database/mariadb/database.py | 20 +++++++++----------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index e61b1829fa..09248fa99f 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -279,7 +279,9 @@ def connect_replica(): user = local.conf.replica_db_name password = local.conf.replica_db_password - local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port) + local.replica_db = get_db( + host=local.conf.replica_host, user=user, password=password, port=port, read_only=True + ) # swap db connections local.primary_db = local.db diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 7de3fabf01..423442d344 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -39,17 +39,21 @@ def drop_user_and_database(db_name, root_login=None, root_password=None): ) -def get_db(host=None, user=None, password=None, port=None): +def get_db(host=None, user=None, password=None, port=None, read_only=False): import frappe if frappe.conf.db_type == "postgres": import frappe.database.postgres.database - return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port) + return frappe.database.postgres.database.PostgresDatabase( + host, user, password, port=port, read_only=read_only + ) else: import frappe.database.mariadb.database - return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port) + return frappe.database.mariadb.database.MariaDBDatabase( + host, user, password, port=port, read_only=read_only + ) def setup_help_database(help_db_name): diff --git a/frappe/database/database.py b/frappe/database/database.py index 73460614cb..f3f539b792 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -52,12 +52,22 @@ class Database(object): class InvalidColumnName(frappe.ValidationError): pass - def __init__(self, host=None, user=None, password=None, ac_name=None, use_default=0, port=None): + def __init__( + self, + host=None, + user=None, + password=None, + ac_name=None, + use_default=0, + port=None, + read_only=False, + ): self.setup_type_map() self.host = host or frappe.conf.db_host or "127.0.0.1" self.port = port or frappe.conf.db_port or "" self.user = user or frappe.conf.db_name self.db_name = frappe.conf.db_name + self.read_only = read_only # Uses READ ONLY connection if set self._conn = None if ac_name: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index c444e492a2..c255d40d28 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -133,12 +133,10 @@ class MariaDBConnectionUtil: self.close_connection_pools() return self.create_connection() - read_only = frappe.conf.read_from_replica and frappe.conf.replica_host - if frappe.local.site not in _SITE_POOLS: - site_pool = self.create_connection_pool(read_only=read_only) + site_pool = self.create_connection_pool() else: - site_pool = self.get_connection_pool(read_only=read_only) + site_pool = self.get_connection_pool() try: conn = site_pool.get_connection() @@ -165,26 +163,26 @@ class MariaDBConnectionUtil: pass _SITE_POOLS.pop(frappe.local.site, None) - def get_pool_name(self, read_only=False) -> str: - pool_type = "read-only" if read_only else "default" + def get_pool_name(self) -> str: + pool_type = "read-only" if self.read_only else "default" return f"{frappe.local.site}-{pool_type}" - def get_connection_pool(self, read_only=False) -> "ConnectionPool": + def get_connection_pool(self) -> "ConnectionPool": """Return MariaDB connection pool object. If `read_only` is True, return a read only pool. """ - return _SITE_POOLS[frappe.local.site]["read_only" if read_only else "default"] + return _SITE_POOLS[frappe.local.site]["read_only" if self.read_only else "default"] - def create_connection_pool(self, read_only=False): + def create_connection_pool(self): pool = mariadb.ConnectionPool( - pool_name=self.get_pool_name(read_only=read_only), + pool_name=self.get_pool_name(), pool_size=_MAX_POOL_SIZE, pool_reset_connection=False, ) pool.set_config(**self.get_connection_settings()) - if read_only: + if self.read_only: _SITE_POOLS[frappe.local.site].read_only = pool else: _SITE_POOLS[frappe.local.site].default = pool From b8d2c195a6e6c438846913b45037c564a8a7af90 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jun 2022 19:42:12 +0530 Subject: [PATCH 052/201] fix: Disable connection pooling via bench commands unless specified --- frappe/database/database.py | 1 - frappe/utils/bench_helper.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index f3f539b792..89852c3e1b 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1168,7 +1168,6 @@ class Database(object): def close(self): """Close database connection.""" if self._conn: - # self._cursor.close() self._conn.close() self._cursor = None self._conn = None diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index a0b011acc1..10ace1b1b6 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -106,4 +106,6 @@ if __name__ == "__main__": if not frappe._dev_server: warnings.simplefilter("ignore") + frappe.DISABLE_DATABASE_CONNECTION_POOLING = not int(os.environ.get("DATABASE_POOLING", "0")) + main() From b867dedf1536b9fa6afdfd06d063ef4b48f85186 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jun 2022 23:40:20 +0530 Subject: [PATCH 053/201] refactor(run-ui-tests): Maintain list of cypress plugins --- frappe/commands/utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 41a4b27bcf..a6bf48fac9 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -874,11 +874,19 @@ def run_ui_tests( and os.path.exists(coverage_plugin_path) and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6 ): - # install cypress + # install cypress & dependent plugins click.secho("Installing Cypress...", fg="yellow") - frappe.commands.popen( - "yarn add cypress@^6 cypress-file-upload@^5 @4tw/cypress-drag-drop@^2 cypress-real-events @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile" + packages = " ".join( + [ + "cypress@^6", + "cypress-file-upload@^5", + "@4tw/cypress-drag-drop@^2", + "cypress-real-events", + "@testing-library/cypress@^8", + "@cypress/code-coverage@^3", + ] ) + frappe.commands.popen(f"yarn add {packages} --no-lockfile") # run for headless mode run_or_open = "run --browser chrome --record" if headless else "open" From b96bd8a45ba648b7302c06561a64812b7baa9322 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 26 Jun 2022 16:21:26 +0530 Subject: [PATCH 054/201] feat: encrypt 2FA secrets --- frappe/patches.txt | 1 + frappe/patches/v12_0/encrypt_2fa_secrets.py | 70 +++++++++++++++++++++ frappe/twofactor.py | 48 +++++++++----- 3 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 frappe/patches/v12_0/encrypt_2fa_secrets.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 66422c7db0..d3274105fe 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -121,6 +121,7 @@ execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files +frappe.patches.v12_0.encrypt_2fa_secrets execute:frappe.reload_doc('core', 'doctype', 'doctype') execute:frappe.reload_doc('custom', 'doctype', 'property_setter') frappe.patches.v13_0.remove_invalid_options_for_data_fields diff --git a/frappe/patches/v12_0/encrypt_2fa_secrets.py b/frappe/patches/v12_0/encrypt_2fa_secrets.py new file mode 100644 index 0000000000..195a282908 --- /dev/null +++ b/frappe/patches/v12_0/encrypt_2fa_secrets.py @@ -0,0 +1,70 @@ +import frappe +import frappe.defaults +from frappe.model.naming import make_autoname +from frappe.twofactor import PARENT_FOR_DEFAULTS +from frappe.utils import now_datetime +from frappe.utils.password import encrypt + +DOCTYPE = "DefaultValue" +OLD_PARENT = "__default" + + +def execute(): + table = frappe.qb.DocType(DOCTYPE) + + # set parent for `*_otplogin` + ( + frappe.qb.update(table) + .set(table.parent, PARENT_FOR_DEFAULTS) + .where(table.parent == OLD_PARENT) + .where(table.defkey.like("%_otplogin")) + ).run() + + # create new encrypted records for `*_otpsecret` + secrets = { + key: value + for key, value in frappe.defaults.get_defaults_for(parent=OLD_PARENT).items() + if key.endswith("_otpsecret") + } + + fields = ( + "name", + "creation", + "modified", + "modified_by", + "owner", + "parent", + "parenttype", + "parentfield", + "defkey", + "defvalue", + ) + + user = frappe.session.user + now = str(now_datetime()) + + values = [ + ( + make_autoname("hash", DOCTYPE), + now, + now, + user, + user, + PARENT_FOR_DEFAULTS, + "__default", + "system_defaults", + key, + encrypt(value), + ) + for key, value in secrets.items() + ] + + frappe.db.bulk_insert(DOCTYPE, fields, values) + + frappe.db.delete( + DOCTYPE, + { + "defkey": ("in", list(secrets)), + "parent": OLD_PARENT, + }, + ) diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 6d01331d7d..db165371fa 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -8,9 +8,25 @@ import pyotp from pyqrcode import create as qrcreate import frappe +import frappe.defaults from frappe import _ from frappe.utils import cint, get_datetime, get_url, time_diff_in_seconds from frappe.utils.background_jobs import enqueue +from frappe.utils.password import decrypt, encrypt + +PARENT_FOR_DEFAULTS = "__2fa" + + +def get_default(key): + return frappe.db.get_default(key, parent=PARENT_FOR_DEFAULTS) + + +def set_default(key, value): + frappe.db.set_default(key, value, parent=PARENT_FOR_DEFAULTS) + + +def clear_default(key): + frappe.defaults.clear_default(key, parent=PARENT_FOR_DEFAULTS) class ExpiredLoginException(Exception): @@ -118,11 +134,13 @@ def two_factor_is_enabled_for_(user): def get_otpsecret_for_(user): """Set OTP Secret for user even if not set.""" - otp_secret = frappe.db.get_default(user + "_otpsecret") - if not otp_secret: - otp_secret = b32encode(os.urandom(10)).decode("utf-8") - frappe.db.set_default(user + "_otpsecret", otp_secret) - frappe.db.commit() + if otp_secret := get_default(user + "_otpsecret"): + return decrypt(otp_secret) + + otp_secret = b32encode(os.urandom(10)).decode("utf-8") + set_default(user + "_otpsecret", encrypt(otp_secret)) + frappe.db.commit() + return otp_secret @@ -162,8 +180,8 @@ def confirm_otp_token(login_manager, otp=None, tmp_id=None): totp = pyotp.TOTP(otp_secret) if totp.verify(otp): # show qr code only once - if not frappe.db.get_default(login_manager.user + "_otplogin"): - frappe.db.set_default(login_manager.user + "_otplogin", 1) + if not get_default(login_manager.user + "_otplogin"): + set_default(login_manager.user + "_otplogin", 1) delete_qrimage(login_manager.user) tracker.add_success_attempt() return True @@ -180,7 +198,7 @@ def get_verification_obj(user, token, otp_secret): verification_obj = process_2fa_for_sms(user, token, otp_secret) elif verification_method == "OTP App": # check if this if the first time that the user is trying to login. If so, send an email - if not frappe.db.get_default(user + "_otplogin"): + if not get_default(user + "_otplogin"): verification_obj = process_2fa_for_email(user, token, otp_secret, otp_issuer, method="OTP App") else: verification_obj = process_2fa_for_otp_app(user, otp_secret, otp_issuer) @@ -207,7 +225,7 @@ def process_2fa_for_sms(user, token, otp_secret): def process_2fa_for_otp_app(user, otp_secret, otp_issuer): """Process OTP App method for 2fa.""" totp_uri = pyotp.TOTP(otp_secret).provisioning_uri(user, issuer_name=otp_issuer) - if frappe.db.get_default(user + "_otplogin"): + if get_default(user + "_otplogin"): otp_setup_completed = True else: otp_setup_completed = False @@ -222,7 +240,7 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"): message = None status = True prompt = "" - if method == "OTP App" and not frappe.db.get_default(user + "_otplogin"): + if method == "OTP App" and not get_default(user + "_otplogin"): """Sending one-time email for OTP App""" totp_uri = pyotp.TOTP(otp_secret).provisioning_uri(user, issuer_name=otp_issuer) qrcode_link = get_link_for_qrcode(user, totp_uri) @@ -328,7 +346,7 @@ def send_token_via_sms(otpsecret, token=None, phone_no=None): is_async=True, job_name=None, now=False, - **sms_args + **sms_args, ) return True @@ -364,7 +382,7 @@ def send_token_via_email(user, token, otp_secret, otp_issuer, subject=None, mess is_async=True, job_name=None, now=False, - **email_args + **email_args, ) return True @@ -464,8 +482,8 @@ def reset_otp_secret(user): otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") user_email = frappe.db.get_value("User", user, "email") if frappe.session.user in ["Administrator", user]: - frappe.defaults.clear_default(user + "_otplogin") - frappe.defaults.clear_default(user + "_otpsecret") + clear_default(user + "_otplogin") + clear_default(user + "_otpsecret") email_args = { "recipients": user_email, "sender": None, @@ -484,7 +502,7 @@ def reset_otp_secret(user): is_async=True, job_name=None, now=False, - **email_args + **email_args, ) return frappe.msgprint( _("OTP Secret has been reset. Re-registration will be required on next login.") From 0e4044eebaf27791a499ffb2f4c383391c1baad6 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Sun, 26 Jun 2022 16:47:42 +0530 Subject: [PATCH 055/201] fix: return if no secrets found --- frappe/patches/v12_0/encrypt_2fa_secrets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/patches/v12_0/encrypt_2fa_secrets.py b/frappe/patches/v12_0/encrypt_2fa_secrets.py index 195a282908..002dbe0f03 100644 --- a/frappe/patches/v12_0/encrypt_2fa_secrets.py +++ b/frappe/patches/v12_0/encrypt_2fa_secrets.py @@ -27,6 +27,9 @@ def execute(): if key.endswith("_otpsecret") } + if not secrets: + return + fields = ( "name", "creation", From aa8396531328fba1dacbf7e3f5889549bf7d5aa4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jun 2022 12:19:54 +0530 Subject: [PATCH 056/201] ci(ui-tests): Print 'bench start' log on failure Co-authored-by: Ankush Menat --- .github/helper/install.sh | 2 +- .github/workflows/ui-tests.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 3ef7db34f6..41fdead675 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -59,7 +59,7 @@ cd ./apps/frappe || exit yarn add node-sass@4.13.1 cd ../.. -bench start & +bench start &> bench_start.log & bench --site test_site reinstall --yes if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 09b2a3caf8..ecc77f491d 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -175,3 +175,7 @@ jobs: files: /home/runner/frappe-bench/sites/coverage.xml verbose: true flags: server + + - name: Show bench console if tests failed + if: ${{ failure() }} + run: cat ~/frappe-bench/bench_start.log \ No newline at end of file From 786d52f6f97d0db04fd3b4e4692fa1b51ca7b14f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 27 Jun 2022 13:56:08 +0530 Subject: [PATCH 057/201] chore: disable recorder UI test --- cypress/integration/recorder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js index 7d4c83abf5..57d3c01356 100644 --- a/cypress/integration/recorder.js +++ b/cypress/integration/recorder.js @@ -1,4 +1,4 @@ -context('Recorder', () => { +context.skip('Recorder', () => { before(() => { cy.login(); }); From 58645998c0dee47f644e2bc78be2091ed109dd59 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 29 Jun 2022 15:03:37 +0530 Subject: [PATCH 058/201] fix: fixed fields with operators and added abs --- frappe/database/query.py | 10 +++++++++- frappe/query_builder/functions.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 60eeaee8c2..10cb62b4a2 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -389,7 +389,14 @@ class Engine: for arg in args: field = literal_eval_(arg.strip()) if to_cast: - field = Field(field) + try: + operator_fields = arg.split() + field = OPERATOR_MAP[operator_fields[1]]( + Field(operator_fields[0]), + Field(operator_fields[2]), + ) + except IndexError: + field = Field(field) _args.append(field) return getattr(functions, func)(*_args) @@ -410,6 +417,7 @@ class Engine: if not issubclass(type(field), Criterion): if any([func in field and f"{func}(" in field for func in SQL_FUNCTIONS]): functions.append(field) + return [self.get_function_object(function) for function in functions] def remove_string_functions(self, fields, function_objects): diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index d2debd6da1..6481c24442 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -95,6 +95,7 @@ class SqlFunctions(Enum): Avg = "avg" Max = "max" Min = "min" + Abs = "abs" def _max(dt, fieldname, filters=None, **kwargs): From daf3f05fea8eb1cb6e95f5cb9484b9d28303f448 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 29 Jun 2022 16:32:00 +0530 Subject: [PATCH 059/201] fix(db): Hanlde sequences in db.sql values --- frappe/database/mariadb/database.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index c255d40d28..4febc21f8a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union import mariadb from mariadb.constants import FIELD_TYPE from pymysql.constants import ER -from pymysql.converters import escape_string +from pymysql.converters import escape_sequence, escape_string import frappe from frappe.database.database import Database @@ -15,6 +15,7 @@ from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name if TYPE_CHECKING: from mariadb import ConnectionPool +_FIND_ITER_PATTERN = re.compile("%s") _PARAM_COMP = re.compile(r"%\([\w]*\)s") _SITE_POOLS = defaultdict(frappe._dict) _MAX_POOL_SIZE = 64 @@ -235,6 +236,24 @@ class MariaDBConnectionUtil: ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-205 """ pos_values = [] + + # Handle sequences in values - PyMySQL & Psycopg allowed them but MariaDB client doesn't + # This leads to a DataError. MariaDB connector expects a flat tuple. Build queries with + # the ['%s'] * len(values) pattern to avoid this block. + if isinstance(values, (tuple, list)) and any(isinstance(v, (tuple, list)) for v in values): + values = list(values) + find_iter = _FIND_ITER_PATTERN.finditer(query) + + for i, val in enumerate(values): + pos = next(find_iter) + if isinstance(val, list): + query = ( + query[: pos.start()] + + escape_sequence(val, charset=self._conn.character_set) + + query[pos.end() :] + ) + del values[i] + named_tokens = _PARAM_COMP.findall(query) if not named_tokens or len(set(named_tokens)) == len(values): From 4da5fdcd02d0aa5e7bc761b329d04fd61900a25d Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 29 Jun 2022 15:45:38 +0530 Subject: [PATCH 060/201] fix: fixed spaces in args fix: lint --- frappe/database/database.py | 4 +++- frappe/database/query.py | 22 +++++++++++++--------- frappe/tests/test_db_query.py | 4 +++- frappe/tests/test_query.py | 11 ++++++++++- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 3d48f1ebe9..4c5d54f857 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1050,7 +1050,9 @@ class Database(object): cache_count = frappe.cache().get_value("doctype:count:{}".format(dt)) if cache_count is not None: return cache_count - query = frappe.qb.engine.get_query(table=dt, filters=filters, fields=Count("*"), distinct=distinct) + query = frappe.qb.engine.get_query( + table=dt, filters=filters, fields=Count("*"), distinct=distinct + ) count = self.sql(query, debug=debug)[0][0] if not filters and cache: frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) diff --git a/frappe/database/query.py b/frappe/database/query.py index 10cb62b4a2..b69ab7958f 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -387,16 +387,20 @@ class Engine: _args = [] for arg in args: - field = literal_eval_(arg.strip()) + has_operator = False + initial_fields = literal_eval_(arg.strip()) if to_cast: - try: - operator_fields = arg.split() - field = OPERATOR_MAP[operator_fields[1]]( - Field(operator_fields[0]), - Field(operator_fields[2]), - ) - except IndexError: - field = Field(field) + for _operator in OPERATOR_MAP.keys(): + if _operator in initial_fields: + has_operator = True + field = OPERATOR_MAP[_operator]( + *map(lambda field: Field(field.strip()), arg.split(_operator)) + ) + + field = Field(initial_fields) if not has_operator else field + else: + field = initial_fields + _args.append(field) return getattr(functions, func)(*_args) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 8727951f4a..ad9f59b3cd 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -143,7 +143,9 @@ class TestReportview(unittest.TestCase): ) def test_none_filter(self): - query = frappe.qb.engine.get_query("DocType", fields="name", filters={"restrict_to_domain": None}) + query = frappe.qb.engine.get_query( + "DocType", fields="name", filters={"restrict_to_domain": None} + ) sql = str(query).replace("`", "").replace('"', "") condition = "restrict_to_domain IS NULL" self.assertIn(condition, sql) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index e7682a0d0c..88a631ca67 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -42,7 +42,7 @@ class TestQuery(unittest.TestCase): ) def test_functions_fields(self): - from frappe.query_builder.functions import Count, Max + from frappe.query_builder.functions import Abs, Count, Max self.assertEqual( frappe.qb.engine.get_query("User", fields="Count(name)", filters={}).get_sql(), @@ -54,6 +54,15 @@ class TestQuery(unittest.TestCase): frappe.qb.from_("User").select(Count(Field("name")), Max(Field("name"))).get_sql(), ) + self.assertEqual( + frappe.qb.engine.get_query( + "User", fields=["abs(name-email)", "Count(name)"], filters={} + ).get_sql(), + frappe.qb.from_("User") + .select(Abs(Field("name") - Field("email")), Count(Field("name"))) + .get_sql(), + ) + self.assertEqual( frappe.qb.engine.get_query("User", fields=[Count("*")], filters={}).get_sql(), frappe.qb.from_("User").select(Count("*")).get_sql(), From 25bb945de724a898bd4c22923e0ce004a102359a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 29 Jun 2022 17:21:49 +0530 Subject: [PATCH 061/201] feat: Added truediv & mul operators --- frappe/database/query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/database/query.py b/frappe/database/query.py index b69ab7958f..337c9d892a 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -169,6 +169,8 @@ OPERATOR_MAP: Dict[str, Callable] = { "=<": operator.le, ">=": operator.ge, "=>": operator.ge, + "/": operator.truediv, + "*": operator.mul, "in": func_in, "not in": func_not_in, "like": like, From 701bf2ede6ed04c1cf0a899fdcd2a8008d9e01ab Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 29 Jun 2022 22:33:51 +0530 Subject: [PATCH 062/201] fix: fixed false operator placements in query --- frappe/database/query.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 337c9d892a..4a7d0e8a67 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -2,6 +2,7 @@ import operator import re from ast import literal_eval from functools import cached_property +from types import BuiltinFunctionType from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Union import frappe @@ -389,17 +390,20 @@ class Engine: _args = [] for arg in args: - has_operator = False initial_fields = literal_eval_(arg.strip()) if to_cast: + has_primitive_operator = False for _operator in OPERATOR_MAP.keys(): if _operator in initial_fields: - has_operator = True - field = OPERATOR_MAP[_operator]( - *map(lambda field: Field(field.strip()), arg.split(_operator)) - ) + operator_mapping = OPERATOR_MAP[_operator] + # Only perform this if operator is of primitive type. + if isinstance(operator_mapping, BuiltinFunctionType): + has_primitive_operator = True + field = operator_mapping( + *map(lambda field: Field(field.strip()), arg.split(_operator)), + ) - field = Field(initial_fields) if not has_operator else field + field = Field(initial_fields) if not has_primitive_operator else field else: field = initial_fields From 7edc20dd1788b562f56d89ccb8bfcb3f4c676dd8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 30 Jun 2022 15:33:00 +0530 Subject: [PATCH 063/201] Revert "Revert "feat: Adding support to Query engine"" This reverts commit 813dcc1848b6409ac4a734359d54413665c02db8. --- frappe/__init__.py | 9 +- frappe/database/database.py | 32 +--- frappe/database/query.py | 156 +++++++++++++++++- .../desk/doctype/number_card/number_card.py | 2 +- frappe/desk/listview.py | 2 +- frappe/query_builder/__init__.py | 1 + frappe/query_builder/builder.py | 9 + frappe/query_builder/functions.py | 22 ++- frappe/query_builder/utils.py | 6 + frappe/tests/test_db_query.py | 2 +- frappe/tests/test_query.py | 58 ++++++- frappe/utils/goal.py | 2 +- 12 files changed, 257 insertions(+), 44 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 063a62c84b..0f55854535 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -22,7 +22,12 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import click from werkzeug.local import Local, release_local -from frappe.query_builder import get_query_builder, patch_query_aggregation, patch_query_execute +from frappe.query_builder import ( + get_qb_engine, + get_query_builder, + patch_query_aggregation, + patch_query_execute, +) from frappe.utils.caching import request_cache from frappe.utils.data import cstr, sbool @@ -240,7 +245,7 @@ def init(site, sites_path=None, new_site=False): local.session = _dict() local.dev_server = _dev_server local.qb = get_query_builder(local.conf.db_type or "mariadb") - + local.qb.engine = get_qb_engine() setup_module_map() if not _qb_patched.get(local.conf.db_type): diff --git a/frappe/database/database.py b/frappe/database/database.py index a52264ed6d..3d48f1ebe9 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -12,7 +12,7 @@ from contextlib import contextmanager from time import time from typing import Dict, List, Optional, Tuple, Union -from pypika.terms import Criterion, NullValue, PseudoColumn +from pypika.terms import Criterion, NullValue import frappe import frappe.defaults @@ -75,15 +75,6 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} - @property - def query(self): - if not hasattr(self, "_query"): - from .query import Query - - self._query = Query() - del Query - return self._query - def setup_type_map(self): pass @@ -600,7 +591,7 @@ class Database(object): return [map(values.get, fields)] else: - r = self.query.get_sql( + r = frappe.qb.engine.get_query( "Singles", filters={"field": ("in", tuple(fields)), "doctype": doctype}, fields=["field", "value"], @@ -633,7 +624,7 @@ class Database(object): # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - queried_result = self.query.get_sql( + queried_result = frappe.qb.engine.get_query( "Singles", filters={"doctype": doctype}, fields=["field", "value"], @@ -706,7 +697,7 @@ class Database(object): if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] - val = self.query.get_sql( + val = frappe.qb.engine.get_query( table="Singles", filters={"doctype": doctype, "field": fieldname}, fields="value", @@ -748,14 +739,7 @@ class Database(object): ): field_objects = [] - if not isinstance(fields, Criterion): - for field in fields: - if "(" in str(field) or " as " in str(field): - field_objects.append(PseudoColumn(field)) - else: - field_objects.append(field) - - query = self.query.get_sql( + query = frappe.qb.engine.get_query( table=doctype, filters=filters, orderby=order_by, @@ -865,7 +849,7 @@ class Database(object): frappe.clear_document_cache(dt, docname) else: - query = self.query.build_conditions(table=dt, filters=dn, update=True) + query = frappe.qb.engine.build_conditions(table=dt, filters=dn, update=True) # TODO: Fix this; doesn't work rn - gavin@frappe.io # frappe.cache().hdel_keys(dt, "document_cache") # Workaround: clear all document caches @@ -1066,7 +1050,7 @@ class Database(object): cache_count = frappe.cache().get_value("doctype:count:{}".format(dt)) if cache_count is not None: return cache_count - query = self.query.get_sql(table=dt, filters=filters, fields=Count("*"), distinct=distinct) + query = frappe.qb.engine.get_query(table=dt, filters=filters, fields=Count("*"), distinct=distinct) count = self.sql(query, debug=debug)[0][0] if not filters and cache: frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) @@ -1206,7 +1190,7 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ filters = filters or kwargs.get("conditions") - query = self.query.build_conditions(table=doctype, filters=filters).delete() + query = frappe.qb.engine.build_conditions(table=doctype, filters=filters).delete() if "debug" not in kwargs: kwargs["debug"] = debug return query.run(**kwargs) diff --git a/frappe/database/query.py b/frappe/database/query.py index f7cc143cf7..60eeaee8c2 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,16 +1,23 @@ import operator import re +from ast import literal_eval from functools import cached_property -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Union import frappe from frappe import _ from frappe.boot import get_additional_filters_from_hooks from frappe.model.db_query import get_timespan_date_range -from frappe.query_builder import Criterion, Field, Order, Table +from frappe.query_builder import Criterion, Field, Order, Table, functions +from frappe.query_builder.functions import SqlFunctions TAB_PATTERN = re.compile("^tab") WORDS_PATTERN = re.compile(r"\w+") +BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") +SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] + +if TYPE_CHECKING: + from pypika.functions import Function def like(key: Field, value: str) -> frappe.qb: @@ -93,7 +100,7 @@ def func_between(key: Field, value: Union[List, Tuple]) -> frappe.qb: def func_is(key, value): "Wrapper for IS" - return Field(key).isnotnull() if value.lower() == "set" else Field(key).isnull() + return key.isnotnull() if value.lower() == "set" else key.isnull() def func_timespan(key: Field, value: str) -> frappe.qb: @@ -143,6 +150,13 @@ def change_orderby(order: str): return order[0], Order.desc +def literal_eval_(literal): + try: + return literal_eval(literal) + except (ValueError, SyntaxError): + return literal + + # default operators OPERATOR_MAP: Dict[str, Callable] = { "+": operator.add, @@ -168,7 +182,7 @@ OPERATOR_MAP: Dict[str, Callable] = { } -class Query: +class Engine: tables: dict = {} @cached_property @@ -238,7 +252,7 @@ class Query: Returns: conditions (frappe.qb): frappe.qb object """ - if kwargs.get("orderby"): + if kwargs.get("orderby") and kwargs.get("orderby") != "KEEP_DEFAULT_ORDERING": orderby = kwargs.get("orderby") if isinstance(orderby, str) and len(orderby.split()) > 1: for ordby in orderby.split(","): @@ -250,6 +264,7 @@ class Query: if kwargs.get("limit"): conditions = conditions.limit(kwargs.get("limit")) + conditions = conditions.offset(kwargs.get("offset", 0)) if kwargs.get("distinct"): conditions = conditions.distinct() @@ -257,6 +272,9 @@ class Query: if kwargs.get("for_update"): conditions = conditions.for_update() + if kwargs.get("groupby"): + conditions = conditions.groupby(kwargs.get("groupby")) + return conditions def misc_query(self, table: str, filters: Union[List, Tuple] = None, **kwargs): @@ -308,6 +326,10 @@ class Query: conditions = self.add_conditions(conditions, **kwargs) return conditions + for key, value in filters.items(): + if isinstance(value, bool): + filters.update({key: str(int(value))}) + for key in filters: value = filters.get(key) _operator = self.OPERATOR_MAP["="] @@ -317,7 +339,8 @@ class Query: continue if isinstance(value, (list, tuple)): _operator = self.OPERATOR_MAP[value[0].casefold()] - conditions = conditions.where(_operator(Field(key), value[1])) + _value = value[1] if value[1] else ("",) + conditions = conditions.where(_operator(Field(key), _value)) else: if value is not None: conditions = conditions.where(_operator(Field(key), value)) @@ -354,7 +377,117 @@ class Query: return criterion - def get_sql( + def get_function_object(self, field: str) -> "Function": + """Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object""" + func = field.split("(", maxsplit=1)[0].capitalize() + args_start, args_end = len(func) + 1, field.index(")") + args = field[args_start:args_end].split(",") + + to_cast = "*" not in args + _args = [] + + for arg in args: + field = literal_eval_(arg.strip()) + if to_cast: + field = Field(field) + _args.append(field) + + return getattr(functions, func)(*_args) + + def function_objects_from_string(self, fields): + functions = "" + for func in SQL_FUNCTIONS: + if f"{func}(" in fields: + functions = str(func) + str(BRACKETS_PATTERN.search(fields).group()) + return [self.get_function_object(functions)] + if not functions: + return [] + + def function_objects_from_list(self, fields): + functions = [] + for field in fields: + field = field.casefold() if isinstance(field, str) else field + if not issubclass(type(field), Criterion): + if any([func in field and f"{func}(" in field for func in SQL_FUNCTIONS]): + functions.append(field) + return [self.get_function_object(function) for function in functions] + + def remove_string_functions(self, fields, function_objects): + """Remove string functions from fields which have already been converted to function objects""" + for function in function_objects: + if isinstance(fields, str): + fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) + else: + updated_fields = [] + for field in fields: + if isinstance(field, str): + updated_fields.append( + BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "") + ) + else: + updated_fields.append(field) + + fields = [field for field in updated_fields if field] + + return fields + + def set_fields(self, fields, **kwargs): + fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" + if isinstance(fields, list) and None in fields and Field not in fields: + return None + + function_objects = [] + + is_list = isinstance(fields, (list, tuple, set)) + if is_list and len(fields) == 1: + fields = fields[0] + is_list = False + + if is_list: + function_objects += self.function_objects_from_list(fields=fields) + + is_str = isinstance(fields, str) + if is_str: + fields = fields.casefold() + function_objects += self.function_objects_from_string(fields=fields) + + fields = self.remove_string_functions(fields, function_objects) + + if is_str and "," in fields: + fields = [field.replace(" ", "") if "as" not in field else field for field in fields.split(",")] + is_list, is_str = True, False + + if is_str: + if fields == "*": + return fields + if " as " in fields: + fields, reference = fields.split(" as ") + fields = Field(fields).as_(reference) + + if not is_str and fields: + if issubclass(type(fields), Criterion): + return fields + updated_fields = [] + if "*" in fields: + return fields + for field in fields: + if not isinstance(field, Criterion) and field: + if " as " in field: + field, reference = field.split(" as ") + updated_fields.append(Field(field.strip()).as_(reference)) + else: + updated_fields.append(Field(field)) + + fields = updated_fields + + # Need to check instance again since fields modified. + if not isinstance(fields, (list, tuple, set)): + fields = [fields] if fields else [] + + fields.extend(function_objects) + return fields + + def get_query( self, table: str, fields: Union[List, Tuple], @@ -364,15 +497,20 @@ class Query: # Clean up state before each query self.tables = {} criterion = self.build_conditions(table, filters, **kwargs) + fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs) + + join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" if len(self.tables) > 1: primary_table = self.tables[table] del self.tables[table] for table_object in self.tables.values(): - criterion = criterion.left_join(table_object).on(table_object.parent == primary_table.name) + criterion = getattr(criterion, join)(table_object).on( + table_object.parent == primary_table.name + ) if isinstance(fields, (list, tuple)): - query = criterion.select(*kwargs.get("field_objects", fields)) + query = criterion.select(*fields) elif isinstance(fields, Criterion): query = criterion.select(fields) diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 74c8e9eb99..8d031aac01 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -204,7 +204,7 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): if txt: search_conditions = [numberCard[field].like("%{txt}%".format(txt=txt)) for field in searchfields] - condition_query = frappe.db.query.build_conditions(doctype, filters) + condition_query = frappe.qb.engine.build_conditions(doctype, filters) return ( condition_query.select(numberCard.name, numberCard.label, numberCard.document_type) diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py index 5149f8bf86..11c985d1ff 100644 --- a/frappe/desk/listview.py +++ b/frappe/desk/listview.py @@ -37,7 +37,7 @@ def get_group_by_count(doctype: str, current_filters: str, field: str) -> List[D ToDo = DocType("ToDo") User = DocType("User") count = Count("*").as_("count") - filtered_records = frappe.db.query.build_conditions(doctype, current_filters).select("name") + filtered_records = frappe.qb.engine.build_conditions(doctype, current_filters).select("name") return ( frappe.qb.from_(ToDo) diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 1bf9ec97d9..eb1d9df08f 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -7,6 +7,7 @@ from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValue from frappe.query_builder.utils import ( Column, DocType, + get_qb_engine, get_query_builder, patch_query_aggregation, patch_query_execute, diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index d2fdeab324..c23d76974c 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,3 +1,5 @@ +import typing + from pypika import MySQLQuery, Order, PostgreSQLQuery, terms from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder from pypika.queries import QueryBuilder, Schema, Table @@ -13,6 +15,13 @@ class Base: Schema = Schema Table = Table + # Added dynamic type hints for engine attribute + # which is to be assigned later. + if typing.TYPE_CHECKING: + from frappe.database.query import Engine + + engine: Engine + @staticmethod def functions(name: str, *args, **kwargs) -> Function: return Function(name, *args, **kwargs) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index f03c139f57..d2debd6da1 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -1,8 +1,9 @@ +from enum import Enum + from pypika.functions import * from pypika.terms import Arithmetic, ArithmeticExpression, CustomFunction, Function import frappe -from frappe.database.query import Query from frappe.query_builder.custom import GROUP_CONCAT, MATCH, STRING_AGG, TO_TSVECTOR from frappe.query_builder.utils import ImportMapper, db_type_is @@ -14,6 +15,11 @@ class Concat_ws(Function): super(Concat_ws, self).__init__("CONCAT_WS", *terms, **kwargs) +class Locate(Function): + def __init__(self, *terms, **kwargs): + super(Locate, self).__init__("LOCATE", *terms, **kwargs) + + GroupConcat = ImportMapper({db_type_is.MARIADB: GROUP_CONCAT, db_type_is.POSTGRES: STRING_AGG}) Match = ImportMapper({db_type_is.MARIADB: MATCH, db_type_is.POSTGRES: TO_TSVECTOR}) @@ -73,14 +79,24 @@ class Cast_(Function): def _aggregate(function, dt, fieldname, filters, **kwargs): return ( - Query() - .build_conditions(dt, filters) + frappe.qb.engine.build_conditions(dt, filters) .select(function(PseudoColumn(fieldname))) .run(**kwargs)[0][0] or 0 ) +class SqlFunctions(Enum): + DayOfYear = "dayofyear" + Extract = "extract" + Locate = "locate" + Count = "count" + Sum = "sum" + Avg = "avg" + Max = "max" + Min = "min" + + def _max(dt, fieldname, filters=None, **kwargs): return _aggregate(Max, dt, fieldname, filters, **kwargs) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 10bab38a63..b601665dee 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -45,6 +45,12 @@ def get_query_builder(type_of_db: str) -> Union[Postgres, MariaDB]: return picks[db] +def get_qb_engine(): + from frappe.database.query import Engine + + return Engine() + + def get_attr(method_string): modulename = ".".join(method_string.split(".")[:-1]) methodname = method_string.split(".")[-1] diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index c1b2e05266..8727951f4a 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -143,7 +143,7 @@ class TestReportview(unittest.TestCase): ) def test_none_filter(self): - query = frappe.db.query.get_sql("DocType", fields="name", filters={"restrict_to_domain": None}) + query = frappe.qb.engine.get_query("DocType", fields="name", filters={"restrict_to_domain": None}) sql = str(query).replace("`", "").replace('"', "") condition = "restrict_to_domain IS NULL" self.assertIn(condition, sql) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 949c3e9433..e7682a0d0c 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1,14 +1,15 @@ import unittest import frappe +from frappe.query_builder import Field from frappe.tests.test_query_builder import db_type_is, run_only_if -@run_only_if(db_type_is.MARIADB) class TestQuery(unittest.TestCase): + @run_only_if(db_type_is.MARIADB) def test_multiple_tables_in_filters(self): self.assertEqual( - frappe.db.query.get_sql( + frappe.qb.engine.get_query( "DocType", ["*"], [ @@ -18,3 +19,56 @@ class TestQuery(unittest.TestCase): ).get_sql(), "SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'", ) + + def test_string_fields(self): + self.assertEqual( + frappe.qb.engine.get_query( + "User", fields="name, email", filters={"name": "Administrator"} + ).get_sql(), + frappe.qb.from_("User") + .select(Field("name"), Field("email")) + .where(Field("name") == "Administrator") + .get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query( + "User", fields=["name, email"], filters={"name": "Administrator"} + ).get_sql(), + frappe.qb.from_("User") + .select(Field("name"), Field("email")) + .where(Field("name") == "Administrator") + .get_sql(), + ) + + def test_functions_fields(self): + from frappe.query_builder.functions import Count, Max + + self.assertEqual( + frappe.qb.engine.get_query("User", fields="Count(name)", filters={}).get_sql(), + frappe.qb.from_("User").select(Count(Field("name"))).get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query("User", fields=["Count(name)", "Max(name)"], filters={}).get_sql(), + frappe.qb.from_("User").select(Count(Field("name")), Max(Field("name"))).get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query("User", fields=[Count("*")], filters={}).get_sql(), + frappe.qb.from_("User").select(Count("*")).get_sql(), + ) + + def test_qb_fields(self): + user_doctype = frappe.qb.DocType("User") + self.assertEqual( + frappe.qb.engine.get_query( + user_doctype, fields=[user_doctype.name, user_doctype.email], filters={} + ).get_sql(), + frappe.qb.from_(user_doctype).select(user_doctype.name, user_doctype.email).get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query(user_doctype, fields=user_doctype.email, filters={}).get_sql(), + frappe.qb.from_(user_doctype).select(user_doctype.email).get_sql(), + ) diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index fb348496da..9273b83cbf 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -25,7 +25,7 @@ def get_monthly_results( date_format = "%m-%Y" if frappe.db.db_type != "postgres" else "MM-YYYY" return dict( - frappe.db.query.build_conditions(table=goal_doctype, filters=filters) + frappe.qb.engine.build_conditions(table=goal_doctype, filters=filters) .select( DateFormat(Table[date_col], date_format).as_("month_year"), Function(aggregation, goal_field), From 39d30ffeaa1afcce9e63b7ddde5e8e22e34d3127 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jun 2022 19:12:10 +0530 Subject: [PATCH 064/201] fix: Database._transform_query * Handle dict substitutions in transformations too * Allow list / tuple values * Check for values inconsistencies before flattening step --- frappe/database/mariadb/database.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 4febc21f8a..b620c235a3 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -236,6 +236,18 @@ class MariaDBConnectionUtil: ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-205 """ pos_values = [] + named_tokens = _PARAM_COMP.findall(query) + + for token in named_tokens: + key = token[2:-2] + val = values[key] + if isinstance(val, (tuple, list)): + val = escape_sequence(val) + pos_values.append(val) + query = query.replace(token, "%s", 1) + + if pos_values: + values = pos_values # Handle sequences in values - PyMySQL & Psycopg allowed them but MariaDB client doesn't # This leads to a DataError. MariaDB connector expects a flat tuple. Build queries with @@ -246,7 +258,7 @@ class MariaDBConnectionUtil: for i, val in enumerate(values): pos = next(find_iter) - if isinstance(val, list): + if isinstance(val, (list, tuple)): query = ( query[: pos.start()] + escape_sequence(val, charset=self._conn.character_set) @@ -254,17 +266,7 @@ class MariaDBConnectionUtil: ) del values[i] - named_tokens = _PARAM_COMP.findall(query) - - if not named_tokens or len(set(named_tokens)) == len(values): - return query, values - - for token in named_tokens: - key = token[2:-2] - pos_values.append(values[key]) - query = query.replace(token, "%s", 1) - - return query, pos_values + return query, values class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): From cc33a7eae5c73760f24792e27aaddc09b1a8e1a8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jun 2022 19:41:47 +0530 Subject: [PATCH 065/201] chore(deps): Bump MariaDB client from 1.0.11 to 1.1.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cd132b4dec..a103df83d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "html5lib~=1.1", "ipython~=8.4.0", "ldap3~=2.9", - "mariadb~=1.0.11", + "mariadb~=1.1.2", "markdown2~=2.4.0", "maxminddb-geolite2==2018.703", "num2words~=0.5.10", From 88faad640969a7878e6267a609507ea8c2a298d5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 1 Jul 2022 13:30:18 +0530 Subject: [PATCH 066/201] fix: pyupgrade --- frappe/query_builder/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index ec36baae19..c1c7657b86 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -17,7 +17,7 @@ class Concat_ws(Function): class Locate(Function): def __init__(self, *terms, **kwargs): - super(Locate, self).__init__("LOCATE", *terms, **kwargs) + super().__init__("LOCATE", *terms, **kwargs) GroupConcat = ImportMapper({db_type_is.MARIADB: GROUP_CONCAT, db_type_is.POSTGRES: STRING_AGG}) From 448f9573f51a9b1e6a497c6c4809395f35fa2211 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 4 Jul 2022 11:56:55 +0530 Subject: [PATCH 067/201] ci: Separate PY / conflict validation step, merge install --- .github/helper/install_dependencies.sh | 7 ------- .github/workflows/patch-mariadb-tests.yml | 21 ++++++++++++--------- .github/workflows/server-mariadb-tests.yml | 18 +++++++++++------- .github/workflows/server-postgres-tests.yml | 18 +++++++++++------- .github/workflows/ui-tests.yml | 18 +++++++++++------- 5 files changed, 45 insertions(+), 37 deletions(-) diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index 6a837f268a..397203f3cc 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -2,13 +2,6 @@ set -e -# Check for merge conflicts before proceeding -python -m compileall -f "${GITHUB_WORKSPACE}" -if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" - then echo "Found merge conflicts" - exit 1 -fi - # install wkhtmltopdf wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz tar -xf /tmp/wkhtmltox.tar.xz -C /tmp diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 0c9fe2bb8a..5194444d49 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -30,6 +30,14 @@ jobs: - name: Clone uses: actions/checkout@v3 + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Setup Python uses: "gabrielfalcao/pyenv-action@v9" with: @@ -92,22 +100,17 @@ jobs: ${{ runner.os }}-yarn- - name: Install Dependencies - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: server - - - name: Install if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh pip install frappe-bench pyenv global $(pyenv versions | grep '3.10') bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: - DB: mariadb + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server + DB: mariadb - name: Run Patch Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 719972f535..028ef30746 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -44,6 +44,14 @@ jobs: with: python-version: '3.10' + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Check if build should be run id: check-build run: | @@ -104,18 +112,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: server - name: Run Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 8f015f43e6..af7541dd6e 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -47,6 +47,14 @@ jobs: with: python-version: '3.10' + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Check if build should be run id: check-build run: | @@ -107,18 +115,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: postgres - TYPE: server - name: Run Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 5dc5cb9c4c..65debe989c 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -43,6 +43,14 @@ jobs: with: python-version: '3.10' + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Check if build should be run id: check-build run: | @@ -113,18 +121,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: ui - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: ui - name: Instrument Source Code if: ${{ steps.check-build.outputs.build == 'strawberry' }} From 1723a6dc9dbe15502f9bec5c901d563318413d6e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 5 Jul 2022 14:59:11 +0530 Subject: [PATCH 068/201] ci: Add mariadb system dependency --- .github/helper/install.sh | 8 ++------ .github/helper/install_dependencies.sh | 17 +++++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index d038be1b23..88ca012c9b 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -1,9 +1,9 @@ #!/bin/bash - set -e - cd ~ || exit +echo "Setting Up Bench..." + pip install frappe-bench bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}" @@ -17,10 +17,6 @@ if [ "$TYPE" == "server" ]; then fi if [ "$DB" == "mariadb" ];then - curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup - sudo bash mariadb_repo_setup --mariadb-server-version=10.6 - sudo apt install mariadb-client - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index 397203f3cc..f16bd61a53 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -1,12 +1,13 @@ #!/bin/bash - set -e - # install wkhtmltopdf -wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz -tar -xf /tmp/wkhtmltox.tar.xz -C /tmp -sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf -sudo chmod o+x /usr/local/bin/wkhtmltopdf +echo "Setting Up System Dependencies..." -# install cups -sudo apt update && sudo apt install libcups2-dev libmariadb-dev redis-server +wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb +sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb + +curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup +sudo bash mariadb_repo_setup --mariadb-server-version=10.6 + +sudo apt update +sudo apt install libcups2-dev redis-server libmariadb3 libmariadb-dev mariadb-client From dbb37acedf7918a4a12a8444343b76439ed6d818 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 5 Jul 2022 17:47:14 +0530 Subject: [PATCH 069/201] fix: Transform queries with all types of `values` --- frappe/database/database.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6808f51d31..07ef86072e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -113,7 +113,7 @@ class Database: raise NotImplementedError def _transform_query(self, query: Query, values: QueryValues): - return query, values or None + return query, values or () def sql( self, @@ -181,10 +181,9 @@ class Database: if debug: time_start = time() - if values is not None: - if not isinstance(values, (tuple, dict, list)): - values = (values,) - query, values = self._transform_query(query, values) + if not isinstance(values, (tuple, dict, list)): + values = (values,) + query, values = self._transform_query(query, values) try: self._cursor.execute(query, values) From 4dc2ecefba95d4a9e886477312c776ed0c529072 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 5 Jul 2022 17:47:54 +0530 Subject: [PATCH 070/201] fix: Use mariadb's constants instead of pymysql --- frappe/database/mariadb/database.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 671b28d168..8b69eba7ae 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -3,8 +3,7 @@ from collections import defaultdict from typing import TYPE_CHECKING import mariadb -from mariadb.constants import FIELD_TYPE -from pymysql.constants import ER +from mariadb.constants import ERR, FIELD_TYPE from pymysql.converters import escape_sequence, escape_string import frappe @@ -53,15 +52,15 @@ class MariaDBExceptionUtil: @staticmethod def is_deadlocked(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.LOCK_DEADLOCK + return getattr(e, "errno", None) == ERR.ER_LOCK_DEADLOCK @staticmethod def is_timedout(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.LOCK_WAIT_TIMEOUT + return getattr(e, "errno", None) == ERR.ER_LOCK_WAIT_TIMEOUT @staticmethod def is_table_missing(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.NO_SUCH_TABLE + return getattr(e, "errno", None) == ERR.ER_NO_SUCH_TABLE @staticmethod def is_missing_table(e: mariadb.Error) -> bool: @@ -69,31 +68,31 @@ class MariaDBExceptionUtil: @staticmethod def is_missing_column(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.BAD_FIELD_ERROR + return getattr(e, "errno", None) == ERR.ER_BAD_FIELD_ERROR @staticmethod def is_duplicate_fieldname(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.DUP_FIELDNAME + return getattr(e, "errno", None) == ERR.ER_DUP_FIELDNAME @staticmethod def is_duplicate_entry(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.DUP_ENTRY + return getattr(e, "errno", None) == ERR.ER_DUP_ENTRY @staticmethod def is_access_denied(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.ACCESS_DENIED_ERROR + return getattr(e, "errno", None) == ERR.ER_ACCESS_DENIED_ERROR @staticmethod def cant_drop_field_or_key(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.CANT_DROP_FIELD_OR_KEY + return getattr(e, "errno", None) == ERR.ER_CANT_DROP_FIELD_OR_KEY @staticmethod def is_syntax_error(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.PARSE_ERROR + return getattr(e, "errno", None) == ERR.ER_PARSE_ERROR @staticmethod def is_data_too_long(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ER.DATA_TOO_LONG + return getattr(e, "errno", None) == ERR.ER_DATA_TOO_LONG @staticmethod def is_primary_key_violation(e: mariadb.Error) -> bool: @@ -266,7 +265,7 @@ class MariaDBConnectionUtil: ) del values[i] - return query, values + return query, values or () class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): From 1faa2fd1e2eab6458e197b0d46fe22d28bf803e0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 6 Jul 2022 10:52:48 +0530 Subject: [PATCH 071/201] fix: Format string instead of % to avoid TypeError --- frappe/database/database.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 07ef86072e..54a54d01a7 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -279,11 +279,13 @@ class Database: try: return self._cursor.mogrify(query, values) - except BaseException: + except AttributeError: if isinstance(values, dict): - return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} + return query.format( + **{k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} + ) elif isinstance(values, (list, tuple)): - return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) + return query.format(*(frappe.db.escape(v) if isinstance(v, str) else v for v in values)) return query, values def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify: From 6a76f8ad5f2d3b6c8052c5db6c3b6b9025230acf Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 6 Jul 2022 11:47:06 +0530 Subject: [PATCH 072/201] fix(pg): Transform Falsy values as None ref: https://github.com/frappe/frappe/runs/7208763442 --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 54a54d01a7..7ee522d99f 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -113,7 +113,7 @@ class Database: raise NotImplementedError def _transform_query(self, query: Query, values: QueryValues): - return query, values or () + return query, values or None def sql( self, From 41b93392b16ade56d829daa66b9805e85aea8533 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 6 Jul 2022 15:58:45 +0530 Subject: [PATCH 073/201] fix: Database._transform_result Transform data not already casted by mariadb client --- frappe/database/database.py | 7 ++++-- frappe/database/mariadb/database.py | 34 ++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7ee522d99f..1003618a6e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -112,9 +112,12 @@ class Database: def get_database_size(self): raise NotImplementedError - def _transform_query(self, query: Query, values: QueryValues): + def _transform_query(self, query: Query, values: QueryValues) -> tuple: return query, values or None + def _transform_result(self, result: list[tuple]) -> list[tuple]: + return result + def sql( self, query: Query, @@ -221,7 +224,7 @@ class Database: if not self._cursor.description: return () - self.last_result = self._cursor.fetchall() + self.last_result = self._transform_result(self._cursor.fetchall()) if pluck: return [r[0] for r in self.last_result] diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 8b69eba7ae..c112fcf62b 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,5 +1,6 @@ import re from collections import defaultdict +from decimal import Decimal from typing import TYPE_CHECKING import mariadb @@ -200,11 +201,7 @@ class MariaDBConnectionUtil: "host": self.host, "user": self.user, "password": self.password, - "converter": { - FIELD_TYPE.NEWDECIMAL: float, - FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: escape_string, - }, + "converter": self.CONVERSION_MAP, } if self.user != "root": @@ -226,6 +223,10 @@ class MariaDBConnectionUtil: conn_settings.update(ssl_params) return conn_settings + +class MariaDBCursorPatchUtil: + """Patch mariadb.cursor.Cursor to handle things not supported by pinned version of MariaDB client.""" + def _transform_query(self, query: str, values: dict) -> str: """Converts a query with named placeholders to a query with %s and values dict to a tuple. @@ -267,8 +268,24 @@ class MariaDBConnectionUtil: return query, values or () + def _transform_result(self, result: list[tuple], description=tuple[tuple]) -> list[tuple]: + # ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-213 + _result = [] + for row in result: + _row = [] + for el in row: + if isinstance(el, Decimal): + el = float(el) + elif isinstance(el, UnicodeWithAttrs): + el = escape_string(el) + _row.append(el) + _result.append(tuple(_row)) + return _result -class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): + +class MariaDBDatabase( + MariaDBCursorPatchUtil, MariaDBConnectionUtil, MariaDBExceptionUtil, Database +): REGEX_CHARACTER = "regexp" # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, @@ -279,6 +296,11 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): # using the system after a restore. # issue link: https://jira.mariadb.org/browse/MDEV-21786 SEQUENCE_CACHE = 50 + CONVERSION_MAP = { + FIELD_TYPE.NEWDECIMAL: float, + FIELD_TYPE.DATETIME: get_datetime, + UnicodeWithAttrs: escape_string, + } def setup_type_map(self): self.db_type = "mariadb" From 576fa32af4d16c11cbb753ce7947d2274332c15f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 7 Jul 2022 12:09:36 +0530 Subject: [PATCH 074/201] fix(mariadb): SequenceGeneratorLimitExceeded is an OperationalError now :D --- frappe/database/mariadb/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index c112fcf62b..bed863b3cc 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -48,7 +48,7 @@ class MariaDBExceptionUtil: DataError = mariadb.DataError # match ER_SEQUENCE_RUN_OUT - https://mariadb.com/kb/en/mariadb-error-codes/ - SequenceGeneratorLimitExceeded = mariadb.ProgrammingError + SequenceGeneratorLimitExceeded = mariadb.OperationalError SequenceGeneratorLimitExceeded.errno = 4084 @staticmethod From 8ccc0d039be2c8b9f22e08f646a84e7918055300 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 7 Jul 2022 15:57:57 +0530 Subject: [PATCH 075/201] fix: Database._transform_query * Don't try to convert None to sequence * Skip transforming values that mariadb~=1.1 can handle - only worry about sequences for now --- frappe/database/database.py | 14 ++++--- frappe/database/mariadb/database.py | 62 +++++++++++------------------ 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1003618a6e..9b5f7bbfe5 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -9,6 +9,7 @@ import string import traceback from contextlib import contextmanager from time import time +from types import NoneType from pypika.terms import Criterion, NullValue, PseudoColumn @@ -184,7 +185,7 @@ class Database: if debug: time_start = time() - if not isinstance(values, (tuple, dict, list)): + if not isinstance(values, (NoneType, tuple, dict, list)): values = (values,) query, values = self._transform_query(query, values) @@ -206,6 +207,11 @@ class Database: frappe.errprint(f"Error in query:\n{e}") raise + elif isinstance(e, self.ProgrammingError): + traceback.print_stack() + frappe.errprint(f"Error in query:\n{query, values}") + raise + if not ( ignore_ddl and (self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e)) @@ -284,11 +290,9 @@ class Database: return self._cursor.mogrify(query, values) except AttributeError: if isinstance(values, dict): - return query.format( - **{k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} - ) + return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} elif isinstance(values, (list, tuple)): - return query.format(*(frappe.db.escape(v) if isinstance(v, str) else v for v in values)) + return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) return query, values def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index bed863b3cc..71317b5884 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -8,7 +8,7 @@ from mariadb.constants import ERR, FIELD_TYPE from pymysql.converters import escape_sequence, escape_string import frappe -from frappe.database.database import Database +from frappe.database.database import Database, QueryValues from frappe.database.mariadb.schema import MariaDBTable from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name @@ -227,48 +227,34 @@ class MariaDBConnectionUtil: class MariaDBCursorPatchUtil: """Patch mariadb.cursor.Cursor to handle things not supported by pinned version of MariaDB client.""" - def _transform_query(self, query: str, values: dict) -> str: - """Converts a query with named placeholders to a query with %s and values dict to a tuple. + def _transform_query(self, query: str, values: QueryValues) -> tuple: + """Transform the query to handle things not supported by pinned version of MariaDB client. - This is a workaround since the MariaDB Python client (1.0.11) responds inconsistently - depending on the substitions in the query & type of values passed. - - ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-205 + Transformations: + - Escape sequences in values """ - pos_values = [] - named_tokens = _PARAM_COMP.findall(query) + _values = [] - for token in named_tokens: - key = token[2:-2] - val = values[key] - if isinstance(val, (tuple, list)): - val = escape_sequence(val) - pos_values.append(val) - query = query.replace(token, "%s", 1) + if isinstance(values, (tuple, list)): + for val in values: + if isinstance(val, (tuple, list)): + _values.append(escape_sequence(val, charset=self._conn.character_set)) + else: + _values.append(val) + values = _values + else: + for token in _PARAM_COMP.findall(query): + key = token[2:-2] + try: + val = values[key] + except KeyError: + raise self.ProgrammingError(f"Missing value for key '{key}'") + if isinstance(val, (tuple, list)): + values[key] = escape_sequence(val, charset=self._conn.character_set) - if pos_values: - values = pos_values + return query, values or [] - # Handle sequences in values - PyMySQL & Psycopg allowed them but MariaDB client doesn't - # This leads to a DataError. MariaDB connector expects a flat tuple. Build queries with - # the ['%s'] * len(values) pattern to avoid this block. - if isinstance(values, (tuple, list)) and any(isinstance(v, (tuple, list)) for v in values): - values = list(values) - find_iter = _FIND_ITER_PATTERN.finditer(query) - - for i, val in enumerate(values): - pos = next(find_iter) - if isinstance(val, (list, tuple)): - query = ( - query[: pos.start()] - + escape_sequence(val, charset=self._conn.character_set) - + query[pos.end() :] - ) - del values[i] - - return query, values or () - - def _transform_result(self, result: list[tuple], description=tuple[tuple]) -> list[tuple]: + def _transform_result(self, result: list[tuple]) -> list[tuple]: # ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-213 _result = [] for row in result: From e7023fa74ddf927a6947b2b0b68b1e8b0d00af1b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 7 Jul 2022 17:07:40 +0530 Subject: [PATCH 076/201] feat(ldap): Allow setting "User Type" for new users Prior to this, every user found in LDAP would mean a System User is created - now pick the type and role you want to give newly created users. For the given user types, the role may be picked from: System User | ldap_settings.default_user_role (Fetched from LDAP settings) Website User | N/A {{ Custom Type }} | user_type.role (Fetched from User Type record) --- .../ldap_group_mapping.json | 7 ++-- .../doctype/ldap_settings/ldap_settings.json | 34 +++++++++++++++---- .../doctype/ldap_settings/ldap_settings.py | 27 ++++++++++----- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json index 92db68e962..9bfe1eac56 100644 --- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json +++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json @@ -1,4 +1,5 @@ { + "actions": [], "creation": "2019-05-29 01:24:29.585060", "doctype": "DocType", "editable_grid": 1, @@ -19,13 +20,14 @@ "fieldname": "erpnext_role", "fieldtype": "Link", "in_list_view": 1, - "label": "ERPNext Role", + "label": "User Role", "options": "Role", "reqd": 1 } ], "istable": 1, - "modified": "2019-07-15 06:46:38.050408", + "links": [], + "modified": "2022-07-07 16:28:44.828514", "modified_by": "Administrator", "module": "Integrations", "name": "LDAP Group Mapping", @@ -34,5 +36,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.json b/frappe/integrations/doctype/ldap_settings/ldap_settings.json index fd45a71538..f5472a5097 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.json +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.json @@ -42,7 +42,10 @@ "column_break_33", "ldap_group_member_attribute", "ldap_group_mappings_section", + "default_user_type", + "column_break_38", "default_role", + "section_break_40", "ldap_groups", "ldap_group_field" ], @@ -79,9 +82,11 @@ "reqd": 1 }, { + "depends_on": "eval: doc.default_user_type == \"System User\"", "fieldname": "default_role", "fieldtype": "Link", - "label": "Default Role on Creation", + "label": "Default User Role", + "mandatory_depends_on": "eval: doc.default_user_type == \"System User\"", "options": "Role", "reqd": 1 }, @@ -249,10 +254,10 @@ "label": "Group Object Class" }, { - "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com", - "fieldname": "ldap_custom_group_search", - "fieldtype": "Data", - "label": "Custom Group Search" + "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com", + "fieldname": "ldap_custom_group_search", + "fieldtype": "Data", + "label": "Custom Group Search" }, { "description": "Requires any valid fdn path. i.e. ou=users,dc=example,dc=com", @@ -268,12 +273,28 @@ "fieldtype": "Data", "label": "LDAP search path for Groups", "reqd": 1 + }, + { + "fieldname": "default_user_type", + "fieldtype": "Link", + "label": "Default User Type", + "options": "User Type", + "reqd": 1 + }, + { + "fieldname": "column_break_38", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_40", + "fieldtype": "Section Break", + "hide_border": 1 } ], "in_create": 1, "issingle": 1, "links": [], - "modified": "2021-07-27 11:51:43.328271", + "modified": "2022-07-07 16:51:46.230793", "modified_by": "Administrator", "module": "Integrations", "name": "LDAP Settings", @@ -294,5 +315,6 @@ "read_only": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index ef6493717f..249d4e31f3 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -1,11 +1,16 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import TYPE_CHECKING + import frappe from frappe import _, safe_encode from frappe.model.document import Document from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, should_run_2fa +if TYPE_CHECKING: + from frappe.core.doctype.user.user import User + class LDAPSettings(Document): def validate(self): @@ -134,7 +139,7 @@ class LDAPSettings(Document): setattr(user, key, value) user.save(ignore_permissions=True) - def sync_roles(self, user, additional_groups=None): + def sync_roles(self, user: "User", additional_groups=None): current_roles = {d.role for d in user.get("roles")} @@ -158,7 +163,9 @@ class LDAPSettings(Document): user.remove_roles(*roles_to_remove) def create_or_update_user(self, user_data, groups=None): - user = None + user: "User" = None + role: str = None + if frappe.db.exists("User", user_data["email"]): user = frappe.get_doc("User", user_data["email"]) LDAPSettings.update_user_fields(user=user, user_data=user_data) @@ -169,16 +176,20 @@ class LDAPSettings(Document): "doctype": "User", "send_welcome_email": 0, "language": "", - "user_type": "System User", - # "roles": [{ - # "role": self.default_role - # }] + "user_type": self.default_user_type, } ) user = frappe.get_doc(doc) user.insert(ignore_permissions=True) - # always add default role. - user.add_roles(self.default_role) + + if self.default_user_type == "System User": + role = self.default_role + else: + role = frappe.db.get_value("User Type", user.user_type, "role") + + if role: + user.add_roles(role) + self.sync_roles(user, groups) return user From b20f77b9b965ba5fb03f3f7b88883413ef3e2251 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 8 Jul 2022 11:39:37 +0530 Subject: [PATCH 077/201] fix(ldap): Set default user type to System User --- frappe/integrations/doctype/ldap_settings/ldap_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index 249d4e31f3..23f4965438 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -14,6 +14,8 @@ if TYPE_CHECKING: class LDAPSettings(Document): def validate(self): + self.default_user_type = self.default_user_type or "System User" + if not self.enabled: return From ee97038c71e5a72dde107105d6868062c57e02fb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 7 Jul 2022 17:34:08 +0530 Subject: [PATCH 078/201] chore: Add typing + reduce import paths --- .../doctype/ldap_settings/ldap_settings.py | 101 +++++++----------- 1 file changed, 37 insertions(+), 64 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index 23f4965438..f9d083a12f 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -1,8 +1,20 @@ -# Copyright (c) 2015, Frappe Technologies and contributors +# Copyright (c) 2022, Frappe Technologies and contributors # License: MIT. See LICENSE +import ssl from typing import TYPE_CHECKING +import ldap3 +from ldap3 import AUTO_BIND_TLS_BEFORE_BIND, HASHED_SALTED_SHA, MODIFY_REPLACE +from ldap3.abstract.entry import Entry +from ldap3.core.exceptions import ( + LDAPAttributeError, + LDAPInvalidCredentialsResult, + LDAPInvalidFilterError, + LDAPNoSuchObjectResult, +) +from ldap3.utils.hashed import hashed + import frappe from frappe import _, safe_encode from frappe.model.document import Document @@ -20,7 +32,6 @@ class LDAPSettings(Document): return if not self.flags.ignore_mandatory: - if ( self.ldap_search_string.count("(") == self.ldap_search_string.count(")") and self.ldap_search_string.startswith("(") @@ -35,8 +46,6 @@ class LDAPSettings(Document): try: if conn.result["type"] == "bindResponse" and self.base_dn: - import ldap3 - conn.search( search_base=self.ldap_search_path_user, search_filter="(objectClass=*)", @@ -47,13 +56,13 @@ class LDAPSettings(Document): search_base=self.ldap_search_path_group, search_filter="(objectClass=*)", attributes=["cn"] ) - except ldap3.core.exceptions.LDAPAttributeError as ex: + except LDAPAttributeError as ex: frappe.throw( _("LDAP settings incorrect. validation response was: {0}").format(ex), title=_("Misconfigured"), ) - except ldap3.core.exceptions.LDAPNoSuchObjectResult: + except LDAPNoSuchObjectResult: frappe.throw( _("Ensure the user and group search paths are correct."), title=_("Misconfigured") ) @@ -82,12 +91,8 @@ class LDAPSettings(Document): ) ) - def connect_to_ldap(self, base_dn, password, read_only=True): + def connect_to_ldap(self, base_dn, password, read_only=True) -> ldap3.Connection: try: - import ssl - - import ldap3 - if self.require_trusted_certificate == "Yes": tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT) else: @@ -101,9 +106,9 @@ class LDAPSettings(Document): tls_configuration.ca_certs_file = self.local_ca_certs_file server = ldap3.Server(host=self.ldap_server_url, tls=tls_configuration) - bind_type = ldap3.AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True + bind_type = AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True - conn = ldap3.Connection( + return ldap3.Connection( server=server, user=base_dn, password=password, @@ -112,18 +117,16 @@ class LDAPSettings(Document): raise_exceptions=True, ) - return conn - except ImportError: msg = _("Please Install the ldap3 library via pip to use ldap functionality.") frappe.throw(msg, title=_("LDAP Not Installed")) - except ldap3.core.exceptions.LDAPInvalidCredentialsResult: + except LDAPInvalidCredentialsResult: frappe.throw(_("Invalid username or password")) except Exception as ex: frappe.throw(_(str(ex))) @staticmethod - def get_ldap_client_settings(): + def get_ldap_client_settings() -> dict: # return the settings to be used on the client side. result = {"enabled": False} ldap = frappe.get_cached_doc("LDAP Settings") @@ -133,21 +136,16 @@ class LDAPSettings(Document): return result @classmethod - def update_user_fields(cls, user, user_data): - + def update_user_fields(cls, user: "User", user_data: dict): updatable_data = {key: value for key, value in user_data.items() if key != "email"} for key, value in updatable_data.items(): setattr(user, key, value) user.save(ignore_permissions=True) - def sync_roles(self, user: "User", additional_groups=None): - + def sync_roles(self, user: "User", additional_groups: list = None): current_roles = {d.role for d in user.get("roles")} - - needed_roles = set() - needed_roles.add(self.default_role) - + needed_roles = {self.default_role} lower_groups = [g.lower() for g in additional_groups or []] all_mapped_roles = {r.erpnext_role for r in self.ldap_groups} @@ -164,7 +162,7 @@ class LDAPSettings(Document): user.remove_roles(*roles_to_remove) - def create_or_update_user(self, user_data, groups=None): + def create_or_update_user(self, user_data: dict, groups: list = None): user: "User" = None role: str = None @@ -216,38 +214,28 @@ class LDAPSettings(Document): return ldap_attributes - def fetch_ldap_groups(self, user, conn): - import ldap3 + def fetch_ldap_groups(self, user: Entry, conn: ldap3.Connection) -> list: + if not isinstance(user, Entry): + raise TypeError("Invalid type, attribute 'user' must be of type 'ldap3.abstract.entry.Entry'") - if type(user) is not ldap3.abstract.entry.Entry: - raise TypeError( - "Invalid type, attribute {} must be of type '{}'".format("user", "ldap3.abstract.entry.Entry") - ) - - if type(conn) is not ldap3.core.connection.Connection: - raise TypeError( - "Invalid type, attribute {} must be of type '{}'".format("conn", "ldap3.Connection") - ) + if not isinstance(conn, ldap3.Connection): + raise TypeError("Invalid type, attribute 'conn' must be of type 'ldap3.Connection'") fetch_ldap_groups = None - ldap_object_class = None ldap_group_members_attribute = None if self.ldap_directory_server.lower() == "active directory": - ldap_object_class = "Group" ldap_group_members_attribute = "member" user_search_str = user.entry_dn elif self.ldap_directory_server.lower() == "openldap": - ldap_object_class = "posixgroup" ldap_group_members_attribute = "memberuid" user_search_str = getattr(user, self.ldap_username_field).value elif self.ldap_directory_server.lower() == "custom": - ldap_object_class = self.ldap_group_objectclass ldap_group_members_attribute = self.ldap_group_member_attribute ldap_custom_group_search = self.ldap_custom_group_search or "{0}" @@ -258,39 +246,31 @@ class LDAPSettings(Document): # this path will be hit for everyone with preconfigured ldap settings. this must be taken into account so as not to break ldap for those users. if self.ldap_group_field: - fetch_ldap_groups = getattr(user, self.ldap_group_field).values if ldap_object_class is not None: conn.search( search_base=self.ldap_search_path_group, - search_filter="(&(objectClass={})({}={}))".format( - ldap_object_class, ldap_group_members_attribute, user_search_str - ), + search_filter=f"(&(objectClass={ldap_object_class})({ldap_group_members_attribute}={user_search_str}))", attributes=["cn"], ) # Build search query if len(conn.entries) >= 1: - fetch_ldap_groups = [] for group in conn.entries: fetch_ldap_groups.append(group["cn"].value) return fetch_ldap_groups - def authenticate(self, username, password): - + def authenticate(self, username: str, password: str): if not self.enabled: frappe.throw(_("LDAP is not enabled.")) user_filter = self.ldap_search_string.format(username) ldap_attributes = self.get_ldap_attributes() - conn = self.connect_to_ldap(self.base_dn, self.get_password(raise_exception=False)) try: - import ldap3 - conn.search( search_base=self.ldap_search_path_user, search_filter=f"{user_filter}", @@ -299,26 +279,21 @@ class LDAPSettings(Document): if len(conn.entries) == 1 and conn.entries[0]: user = conn.entries[0] - groups = self.fetch_ldap_groups(user, conn) # only try and connect as the user, once we have their fqdn entry. if user.entry_dn and password and conn.rebind(user=user.entry_dn, password=password): - return self.create_or_update_user(self.convert_ldap_entry_to_dict(user), groups=groups) - raise ldap3.core.exceptions.LDAPInvalidCredentialsResult # even though nothing foundor failed authentication raise invalid credentials + raise LDAPInvalidCredentialsResult # even though nothing foundor failed authentication raise invalid credentials - except ldap3.core.exceptions.LDAPInvalidFilterError: + except LDAPInvalidFilterError: frappe.throw(_("Please use a valid LDAP search filter"), title=_("Misconfigured")) - except ldap3.core.exceptions.LDAPInvalidCredentialsResult: + except LDAPInvalidCredentialsResult: frappe.throw(_("Invalid username or password")) def reset_password(self, user, password, logout_sessions=False): - from ldap3 import HASHED_SALTED_SHA, MODIFY_REPLACE - from ldap3.utils.hashed import hashed - search_filter = f"({self.ldap_email_field}={user})" conn = self.connect_to_ldap( @@ -347,8 +322,7 @@ class LDAPSettings(Document): else: frappe.throw(_("No LDAP User found for email: {0}").format(user)) - def convert_ldap_entry_to_dict(self, user_entry): - + def convert_ldap_entry_to_dict(self, user_entry: Entry): # support multiple email values email = user_entry[self.ldap_email_field] @@ -359,7 +333,6 @@ class LDAPSettings(Document): } # optional fields - if self.ldap_middle_name_field: data["middle_name"] = user_entry[self.ldap_middle_name_field].value @@ -379,7 +352,7 @@ class LDAPSettings(Document): def login(): # LDAP LOGIN LOGIC args = frappe.form_dict - ldap = frappe.get_doc("LDAP Settings") + ldap: LDAPSettings = frappe.get_doc("LDAP Settings") user = ldap.authenticate(frappe.as_unicode(args.usr), frappe.as_unicode(args.pwd)) @@ -396,7 +369,7 @@ def login(): @frappe.whitelist() def reset_password(user, password, logout): - ldap = frappe.get_doc("LDAP Settings") + ldap: LDAPSettings = frappe.get_doc("LDAP Settings") if not ldap.enabled: frappe.throw(_("LDAP is not enabled.")) ldap.reset_password(user, password, logout_sessions=int(logout)) From c97562b736e19ac442fcf37d593277e3dc371364 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 8 Jul 2022 21:05:42 +0530 Subject: [PATCH 079/201] feat: Added Timestamp support --- frappe/query_builder/functions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index c1c7657b86..824de7fbf5 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -20,6 +20,14 @@ class Locate(Function): super().__init__("LOCATE", *terms, **kwargs) +class Timestamp(Function): + def __init__(self, term: str, time=None, alias=None): + if time: + super().__init__("TIMESTAMP", term, time, alias=alias) + else: + super().__init__("TIMESTAMP", term, alias=alias) + + GroupConcat = ImportMapper({db_type_is.MARIADB: GROUP_CONCAT, db_type_is.POSTGRES: STRING_AGG}) Match = ImportMapper({db_type_is.MARIADB: MATCH, db_type_is.POSTGRES: TO_TSVECTOR}) @@ -96,6 +104,7 @@ class SqlFunctions(Enum): Max = "max" Min = "min" Abs = "abs" + Timestamp = "timestamp" def _max(dt, fieldname, filters=None, **kwargs): From 2fac4f807433cacb4380e56dd530214c89d41202 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 8 Jul 2022 21:33:58 +0530 Subject: [PATCH 080/201] test: Added tests for timestamp --- frappe/tests/test_query.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 88a631ca67..5c523ac555 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -42,7 +42,7 @@ class TestQuery(unittest.TestCase): ) def test_functions_fields(self): - from frappe.query_builder.functions import Abs, Count, Max + from frappe.query_builder.functions import Abs, Count, Max, Timestamp self.assertEqual( frappe.qb.engine.get_query("User", fields="Count(name)", filters={}).get_sql(), @@ -68,6 +68,13 @@ class TestQuery(unittest.TestCase): frappe.qb.from_("User").select(Count("*")).get_sql(), ) + self.assertEqual( + frappe.qb.engine.get_query( + "User", fields="timestamp(creation, modified)", filters={} + ).get_sql(), + frappe.qb.from_("User").select(Timestamp(Field("creation"), Field("modified"))).get_sql(), + ) + def test_qb_fields(self): user_doctype = frappe.qb.DocType("User") self.assertEqual( From 613065fa2e65cfab8ef04c03afd66e072965bc25 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 9 Jul 2022 22:27:29 +0530 Subject: [PATCH 081/201] feat: Added support for aliasing in function objects --- frappe/database/query.py | 21 +++++++++++++++++---- frappe/tests/test_query.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 553160afc6..3169bc52a5 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -186,7 +186,7 @@ OPERATOR_MAP: dict[str, Callable] = { class Engine: - tables: dict = {} + tables: dict[str, str] = {} @cached_property def OPERATOR_MAP(self): @@ -384,6 +384,7 @@ class Engine: args_start, args_end = len(func) + 1, field.index(")") args = field[args_start:args_end].split(",") + _, alias = field.split(" as ") if " as " in field else (None, None) to_cast = "*" not in args _args = [] @@ -406,14 +407,15 @@ class Engine: field = initial_fields _args.append(field) - - return getattr(functions, func)(*_args) + return getattr(functions, func)(*_args, alias=alias or None) def function_objects_from_string(self, fields): functions = "" for func in SQL_FUNCTIONS: if f"{func}(" in fields: + _, alias = fields.split(" as ") if " as " in fields else ("", "") functions = str(func) + str(BRACKETS_PATTERN.search(fields).group()) + functions += " as " + alias return [self.get_function_object(functions)] if not functions: return [] @@ -432,14 +434,25 @@ class Engine: """Remove string functions from fields which have already been converted to function objects""" for function in function_objects: if isinstance(fields, str): + has_alias = False + if function.alias: + has_alias = True fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) + if has_alias: + fields = fields.replace(" as " + function.alias.casefold(), "") else: updated_fields = [] for field in fields: + has_alias = False + if function.alias: + has_alias = True if isinstance(field, str): - updated_fields.append( + _field = ( BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "") ) + if has_alias: + _field = _field.replace(" as " + function.alias.casefold(), "") + updated_fields.append(_field) else: updated_fields.append(field) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 5c523ac555..10208fec70 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -2,6 +2,7 @@ import unittest import frappe from frappe.query_builder import Field +from frappe.query_builder.functions import Abs, Count, Max, Timestamp from frappe.tests.test_query_builder import db_type_is, run_only_if @@ -42,8 +43,6 @@ class TestQuery(unittest.TestCase): ) def test_functions_fields(self): - from frappe.query_builder.functions import Abs, Count, Max, Timestamp - self.assertEqual( frappe.qb.engine.get_query("User", fields="Count(name)", filters={}).get_sql(), frappe.qb.from_("User").select(Count(Field("name"))).get_sql(), @@ -88,3 +87,32 @@ class TestQuery(unittest.TestCase): frappe.qb.engine.get_query(user_doctype, fields=user_doctype.email, filters={}).get_sql(), frappe.qb.from_(user_doctype).select(user_doctype.email).get_sql(), ) + + def test_aliasing(self): + user_doctype = frappe.qb.DocType("User") + self.assertEqual( + frappe.qb.engine.get_query( + user_doctype, fields=["name as owner", "email as id"], filters={} + ).get_sql(), + frappe.qb.from_(user_doctype) + .select(user_doctype.name.as_("owner"), user_doctype.email.as_("id")) + .get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query( + user_doctype, fields="name as owner, email as id", filters={} + ).get_sql(), + frappe.qb.from_(user_doctype) + .select(user_doctype.name.as_("owner"), user_doctype.email.as_("id")) + .get_sql(), + ) + + self.assertEqual( + frappe.qb.engine.get_query( + user_doctype, fields=["Count(name) as c", "email as id"], filters={} + ).get_sql(), + frappe.qb.from_(user_doctype) + .select(user_doctype.email.as_("id"), Count(Field("name")).as_("c")) + .get_sql(), + ) From 17b3f171ff5ce465c34f7481cc17ba371e777dc7 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 11 Jul 2022 13:18:19 +0530 Subject: [PATCH 082/201] fix: update patch to use case when...then --- frappe/patches.txt | 2 +- frappe/patches/v12_0/encrypt_2fa_secrets.py | 73 --------------------- frappe/patches/v13_0/encrypt_2fa_secrets.py | 45 +++++++++++++ 3 files changed, 46 insertions(+), 74 deletions(-) delete mode 100644 frappe/patches/v12_0/encrypt_2fa_secrets.py create mode 100644 frappe/patches/v13_0/encrypt_2fa_secrets.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 98a2da6970..b432ec7786 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -121,7 +121,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') frappe.patches.v12_0.remove_example_email_thread_notify execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders() frappe.patches.v12_0.set_correct_url_in_files -frappe.patches.v12_0.encrypt_2fa_secrets execute:frappe.reload_doc('core', 'doctype', 'doctype') #2022-06-21 execute:frappe.reload_doc('custom', 'doctype', 'property_setter') frappe.patches.v13_0.remove_invalid_options_for_data_fields @@ -184,6 +183,7 @@ frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v13_0.set_first_day_of_the_week +frappe.patches.v13_0.encrypt_2fa_secrets execute:frappe.reload_doc('custom', 'doctype', 'custom_field') frappe.patches.v14_0.update_workspace2 # 20.09.2021 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 diff --git a/frappe/patches/v12_0/encrypt_2fa_secrets.py b/frappe/patches/v12_0/encrypt_2fa_secrets.py deleted file mode 100644 index 002dbe0f03..0000000000 --- a/frappe/patches/v12_0/encrypt_2fa_secrets.py +++ /dev/null @@ -1,73 +0,0 @@ -import frappe -import frappe.defaults -from frappe.model.naming import make_autoname -from frappe.twofactor import PARENT_FOR_DEFAULTS -from frappe.utils import now_datetime -from frappe.utils.password import encrypt - -DOCTYPE = "DefaultValue" -OLD_PARENT = "__default" - - -def execute(): - table = frappe.qb.DocType(DOCTYPE) - - # set parent for `*_otplogin` - ( - frappe.qb.update(table) - .set(table.parent, PARENT_FOR_DEFAULTS) - .where(table.parent == OLD_PARENT) - .where(table.defkey.like("%_otplogin")) - ).run() - - # create new encrypted records for `*_otpsecret` - secrets = { - key: value - for key, value in frappe.defaults.get_defaults_for(parent=OLD_PARENT).items() - if key.endswith("_otpsecret") - } - - if not secrets: - return - - fields = ( - "name", - "creation", - "modified", - "modified_by", - "owner", - "parent", - "parenttype", - "parentfield", - "defkey", - "defvalue", - ) - - user = frappe.session.user - now = str(now_datetime()) - - values = [ - ( - make_autoname("hash", DOCTYPE), - now, - now, - user, - user, - PARENT_FOR_DEFAULTS, - "__default", - "system_defaults", - key, - encrypt(value), - ) - for key, value in secrets.items() - ] - - frappe.db.bulk_insert(DOCTYPE, fields, values) - - frappe.db.delete( - DOCTYPE, - { - "defkey": ("in", list(secrets)), - "parent": OLD_PARENT, - }, - ) diff --git a/frappe/patches/v13_0/encrypt_2fa_secrets.py b/frappe/patches/v13_0/encrypt_2fa_secrets.py new file mode 100644 index 0000000000..1814ff50c5 --- /dev/null +++ b/frappe/patches/v13_0/encrypt_2fa_secrets.py @@ -0,0 +1,45 @@ +import frappe +import frappe.defaults +from frappe.cache_manager import clear_defaults_cache +from frappe.twofactor import PARENT_FOR_DEFAULTS +from frappe.utils.password import encrypt + +DOCTYPE = "DefaultValue" +OLD_PARENT = "__default" + + +def execute(): + table = frappe.qb.DocType(DOCTYPE) + + # set parent for `*_otplogin` + ( + frappe.qb.update(table) + .set(table.parent, PARENT_FOR_DEFAULTS) + .where(table.parent == OLD_PARENT) + .where(table.defkey.like("%_otplogin")) + ).run() + + # update records for `*_otpsecret` + secrets = { + key: value + for key, value in frappe.defaults.get_defaults_for(parent=OLD_PARENT).items() + if key.endswith("_otpsecret") + } + + if not secrets: + return + + defvalue_cases = frappe.qb.terms.Case() + + for key, value in secrets.items(): + defvalue_cases.when(table.defkey == key, encrypt(value)) + + ( + frappe.qb.update(table) + .set(table.parent, PARENT_FOR_DEFAULTS) + .set(table.defvalue, defvalue_cases) + .where(table.parent == OLD_PARENT) + .where(table.defkey.like("%_otpsecret")) + ).run() + + clear_defaults_cache() From ea3bfc914c1e3e448ca976bbbb96c01ebbaedf99 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 11 Jul 2022 13:43:38 +0530 Subject: [PATCH 083/201] test: use newly created `get_default` helper --- frappe/tests/test_twofactor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_twofactor.py b/frappe/tests/test_twofactor.py index d9bf982cd2..b04cb6a91d 100644 --- a/frappe/tests/test_twofactor.py +++ b/frappe/tests/test_twofactor.py @@ -12,6 +12,7 @@ from frappe.twofactor import ( authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, + get_default, get_otpsecret_for_, get_verification_obj, should_run_2fa, @@ -111,7 +112,7 @@ class TestTwoFactor(unittest.TestCase): def test_get_otpsecret_for_user(self): """OTP secret should be set for user.""" self.assertTrue(get_otpsecret_for_(self.user)) - self.assertTrue(frappe.db.get_default(self.user + "_otpsecret")) + self.assertTrue(get_default(self.user + "_otpsecret")) def test_confirm_otp_token(self): """Ensure otp is confirmed""" From dbbd9e90953476de6d88d492fa4a9e31605b8b53 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 12 Jul 2022 17:14:42 +0530 Subject: [PATCH 084/201] fix: removing functions from strings when alias is same as function name --- frappe/database/query.py | 19 +++++-------------- frappe/tests/test_query.py | 4 ++-- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 3169bc52a5..c742df9c62 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -385,6 +385,7 @@ class Engine: args = field[args_start:args_end].split(",") _, alias = field.split(" as ") if " as " in field else (None, None) + to_cast = "*" not in args _args = [] @@ -413,9 +414,7 @@ class Engine: functions = "" for func in SQL_FUNCTIONS: if f"{func}(" in fields: - _, alias = fields.split(" as ") if " as " in fields else ("", "") functions = str(func) + str(BRACKETS_PATTERN.search(fields).group()) - functions += " as " + alias return [self.get_function_object(functions)] if not functions: return [] @@ -434,26 +433,18 @@ class Engine: """Remove string functions from fields which have already been converted to function objects""" for function in function_objects: if isinstance(fields, str): - has_alias = False if function.alias: - has_alias = True - fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) - if has_alias: fields = fields.replace(" as " + function.alias.casefold(), "") + fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) else: updated_fields = [] for field in fields: - has_alias = False - if function.alias: - has_alias = True if isinstance(field, str): - _field = ( + if function.alias: + field = field.replace(" as " + function.alias.casefold(), "") + field = ( BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "") ) - if has_alias: - _field = _field.replace(" as " + function.alias.casefold(), "") - updated_fields.append(_field) - else: updated_fields.append(field) fields = [field for field in updated_fields if field] diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 10208fec70..cdaba2ce77 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -110,9 +110,9 @@ class TestQuery(unittest.TestCase): self.assertEqual( frappe.qb.engine.get_query( - user_doctype, fields=["Count(name) as c", "email as id"], filters={} + user_doctype, fields=["Count(name) as count", "email as id"], filters={} ).get_sql(), frappe.qb.from_(user_doctype) - .select(user_doctype.email.as_("id"), Count(Field("name")).as_("c")) + .select(user_doctype.email.as_("id"), Count(Field("name")).as_("count")) .get_sql(), ) From 9530c90c09ae8d70160f13eab03dfd0890727b78 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 18:19:39 +0530 Subject: [PATCH 085/201] fix(quill): don't go beyond container for bubble (#17475) Container uses bubble type quill editor, options for which seem to spill out and hide behind the commend container. This fixes the UI glitch. ref: https://quilljs.com/docs/configuration/#bounds --- frappe/public/js/frappe/form/controls/comment.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/frappe/form/controls/comment.js b/frappe/public/js/frappe/form/controls/comment.js index b9b2d6a987..4e1c7f97a4 100644 --- a/frappe/public/js/frappe/form/controls/comment.js +++ b/frappe/public/js/frappe/form/controls/comment.js @@ -71,6 +71,7 @@ frappe.ui.form.ControlComment = class ControlComment extends frappe.ui.form.Cont const options = super.get_quill_options(); return Object.assign(options, { theme: 'bubble', + bounds: this.quill_container[0], modules: Object.assign(options.modules, { mention: this.get_mention_options() }) From ca39cfb11d886c92287677e7e2135aec47523485 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 18:38:28 +0530 Subject: [PATCH 086/201] feat: support strikethrough in text-editor/comments (#17478) IMO strikethrough is useful way more often (than many other options that we already support in editor) so adding it in default config. closes #17470 --- frappe/public/js/frappe/form/controls/comment.js | 2 +- frappe/public/js/frappe/form/controls/text_editor.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/comment.js b/frappe/public/js/frappe/form/controls/comment.js index 4e1c7f97a4..4550d7045f 100644 --- a/frappe/public/js/frappe/form/controls/comment.js +++ b/frappe/public/js/frappe/form/controls/comment.js @@ -103,7 +103,7 @@ frappe.ui.form.ControlComment = class ControlComment extends frappe.ui.form.Cont get_toolbar_options() { return [ - ['bold', 'italic', 'underline'], + ['bold', 'italic', 'underline', 'strike'], ['blockquote', 'code-block'], [{ 'direction': "rtl" }], ['link', 'image'], diff --git a/frappe/public/js/frappe/form/controls/text_editor.js b/frappe/public/js/frappe/form/controls/text_editor.js index d190e11cea..e4e1fff18a 100644 --- a/frappe/public/js/frappe/form/controls/text_editor.js +++ b/frappe/public/js/frappe/form/controls/text_editor.js @@ -184,7 +184,7 @@ frappe.ui.form.ControlTextEditor = class ControlTextEditor extends frappe.ui.for return [ [{ header: [1, 2, 3, false] }], [{ size: font_sizes }], - ['bold', 'italic', 'underline', 'clean'], + ['bold', 'italic', 'underline', 'strike', 'clean'], [{ 'color': [] }, { 'background': [] }], ['blockquote', 'code-block'], // Adding Direction tool to give the user the ability to change text direction. From fbb89bdfe9f91e7628759b048cc159461eb3ae72 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Tue, 12 Jul 2022 18:58:07 +0530 Subject: [PATCH 087/201] fix: delete user mention cache when a user is disabled or enabled (#17451) User Mention cache is deleted on 3 occasions: * when a new user is inserted * when allowed_in_mention or user_type value has changed * when a user is deleted But we didn't delete it when a user was enabled or disabled as we maintain the mention cache for enabled users --- frappe/core/doctype/user/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index bc5c20eb92..d6fe883fae 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -134,11 +134,11 @@ class User(Document): if self.time_zone: frappe.defaults.set_default("time_zone", self.time_zone, self.name) - if self.has_value_changed("allow_in_mentions") or self.has_value_changed("user_type"): - frappe.cache().delete_key("users_for_mentions") - if self.has_value_changed("enabled"): + frappe.cache().delete_key("users_for_mentions") frappe.cache().delete_key("enabled_users") + elif self.has_value_changed("allow_in_mentions") or self.has_value_changed("user_type"): + frappe.cache().delete_key("users_for_mentions") def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" From 7a59fc7ecf20f97ba97097f167fb0eee5b3787a3 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 12 Jul 2022 15:15:49 +0000 Subject: [PATCH 088/201] fix: simplify `defaults._clear_cache` (#17485) --- frappe/defaults.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/frappe/defaults.py b/frappe/defaults.py index c2f4a3fe56..d0a49ef692 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -241,8 +241,4 @@ def get_defaults_for(parent="__default"): def _clear_cache(parent): - if parent in common_default_keys: - frappe.clear_cache() - else: - clear_notifications(user=parent) - frappe.clear_cache(user=parent) + frappe.clear_cache(user=parent if parent not in common_default_keys else None) From ff734532aad97ca5abbb7bbbe75f02fdc3a3979e Mon Sep 17 00:00:00 2001 From: vorasmit Date: Tue, 12 Jul 2022 21:06:52 +0530 Subject: [PATCH 089/201] fix: set primary action after clearing previous action (#17454) --- frappe/public/js/frappe/ui/dialog.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index 1618db9939..3e2d22ffa2 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -145,7 +145,8 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { return this.get_primary_btn() .removeClass("hide") .html(label) - .click(function() { + .off('click') + .on('click', function() { me.primary_action_fulfilled = true; // get values and send it // as first parameter to click callback From 64463791a18e4fc9c76698d72a69e841952b4e7a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 12 Jul 2022 22:07:11 +0530 Subject: [PATCH 090/201] feat: Added support for multiple functions in string fields & fixed aliasing --- frappe/database/query.py | 13 ++++++------- frappe/tests/test_query.py | 9 +++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index c742df9c62..10824f4c91 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -16,6 +16,7 @@ TAB_PATTERN = re.compile("^tab") WORDS_PATTERN = re.compile(r"\w+") BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] +COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") if TYPE_CHECKING: from pypika.functions import Function @@ -411,13 +412,8 @@ class Engine: return getattr(functions, func)(*_args, alias=alias or None) def function_objects_from_string(self, fields): - functions = "" - for func in SQL_FUNCTIONS: - if f"{func}(" in fields: - functions = str(func) + str(BRACKETS_PATTERN.search(fields).group()) - return [self.get_function_object(functions)] - if not functions: - return [] + fields = list(map(lambda str: str.strip(), COMMA_PATTERN.split(fields))) + return self.function_objects_from_list(fields=fields) def function_objects_from_list(self, fields): functions = [] @@ -436,6 +432,9 @@ class Engine: if function.alias: fields = fields.replace(" as " + function.alias.casefold(), "") fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) + # Check if only comma is left in fields after stripping functions. + if "," in fields and (len(fields.strip()) == 1): + fields = "" else: updated_fields = [] for field in fields: diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index cdaba2ce77..8afcaf07d0 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -74,6 +74,15 @@ class TestQuery(unittest.TestCase): frappe.qb.from_("User").select(Timestamp(Field("creation"), Field("modified"))).get_sql(), ) + self.assertEqual( + frappe.qb.engine.get_query( + "User", fields="Count(name) as count, Max(email) as max_email", filters={} + ).get_sql(), + frappe.qb.from_("User") + .select(Count(Field("name")).as_("count"), Max(Field("email")).as_("max_email")) + .get_sql(), + ) + def test_qb_fields(self): user_doctype = frappe.qb.DocType("User") self.assertEqual( From 7658c60f574377b7d33144c3c544fe3551facf4b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 13 Jul 2022 03:19:58 +0530 Subject: [PATCH 091/201] feat: Added fall back for custom functions --- frappe/database/query.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 10824f4c91..0f0b7362f7 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -10,7 +10,7 @@ from frappe import _ from frappe.boot import get_additional_filters_from_hooks from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table, functions -from frappe.query_builder.functions import SqlFunctions +from frappe.query_builder.functions import Function, SqlFunctions TAB_PATTERN = re.compile("^tab") WORDS_PATTERN = re.compile(r"\w+") @@ -18,9 +18,6 @@ BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") -if TYPE_CHECKING: - from pypika.functions import Function - def like(key: Field, value: str) -> frappe.qb: """Wrapper method for `LIKE` @@ -409,7 +406,11 @@ class Engine: field = initial_fields _args.append(field) - return getattr(functions, func)(*_args, alias=alias or None) + try: + return getattr(functions, func)(*_args, alias=alias or None) + except AttributeError: + # Fall back for functions not present in `SqlFunctions`` + return Function(func, *_args, alias=alias or None) def function_objects_from_string(self, fields): fields = list(map(lambda str: str.strip(), COMMA_PATTERN.split(fields))) @@ -420,7 +421,7 @@ class Engine: for field in fields: field = field.casefold() if isinstance(field, str) else field if not issubclass(type(field), Criterion): - if any([func in field and f"{func}(" in field for func in SQL_FUNCTIONS]): + if any([f"{func}(" in field for func in SQL_FUNCTIONS]) or "(" in field: functions.append(field) return [self.get_function_object(function) for function in functions] From 572d9cc6967647aabc6ac8894964f504c689866d Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 13 Jul 2022 08:54:52 +0530 Subject: [PATCH 092/201] fix: Return from function after clearing interval --- frappe/email/doctype/newsletter/newsletter.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index d2e5a3c047..3c52e61cbb 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -146,6 +146,7 @@ frappe.ui.form.on('Newsletter', { if (stats.sent + stats.error >= frm.doc.total_recipients || (!stats.total && !stats.emails_queued)) { frm.sending_status && clearInterval(frm.sending_status); frm.sending_status = null; + return; } } From 3fdd89a737f178391b2cb6262b71c1a4c1d83f32 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Jul 2022 11:02:56 +0530 Subject: [PATCH 093/201] refactor!: remove old weekly cleanup for route history (#17493) This is now configurable with log settings. This also fixes circular import issue that occurs when restoring backup --- frappe/database/query.py | 3 +- .../doctype/route_history/route_history.py | 33 ++----------------- frappe/hooks.py | 1 - 3 files changed, 5 insertions(+), 32 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index c87117466b..38da501645 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -5,7 +5,6 @@ from typing import Any, Callable import frappe from frappe import _ -from frappe.boot import get_additional_filters_from_hooks from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table @@ -173,6 +172,8 @@ class Query: @cached_property def OPERATOR_MAP(self): + from frappe.boot import get_additional_filters_from_hooks + # default operators all_operators = OPERATOR_MAP.copy() diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index c62311ae02..a576ac73f5 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -4,45 +4,18 @@ import frappe from frappe.deferred_insert import deferred_insert as _deferred_insert from frappe.model.document import Document -from frappe.query_builder import DocType, Interval -from frappe.query_builder.functions import Count, Now class RouteHistory(Document): @staticmethod def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + table = frappe.qb.DocType("Route History") frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) -def flush_old_route_records(): - """Deletes all route records except last 500 records per user""" - records_to_keep_limit = 500 - RouteHistory = DocType("Route History") - - users = ( - frappe.qb.from_(RouteHistory) - .select(RouteHistory.user) - .groupby(RouteHistory.user) - .having(Count(RouteHistory.name) > records_to_keep_limit) - ).run(pluck=True) - - for user in users: - last_record_to_keep = frappe.get_all( - "Route History", - filters={"user": user}, - limit_start=500, - fields=["modified"], - order_by="modified desc", - limit=1, - ) - - frappe.db.delete( - "Route History", - {"modified": ("<=", last_record_to_keep[0].modified), "user": user}, - ) - - @frappe.whitelist() def deferred_insert(routes): routes = [ diff --git a/frappe/hooks.py b/frappe/hooks.py index 54d068fa9d..a337d8e0d3 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -242,7 +242,6 @@ scheduler_events = { "weekly_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly", "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly", - "frappe.desk.doctype.route_history.route_history.flush_old_route_records", "frappe.desk.form.document_follow.send_weekly_updates", "frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary", "frappe.integrations.doctype.google_drive.google_drive.weekly_backup", From 26dd606831519d080c41f87afe43408fedba53e3 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 19 May 2022 18:46:30 +0530 Subject: [PATCH 094/201] refactor: GoogleOAuth * refactor: single callback method for google oauth --- .../doctype/email_account/email_account.json | 3 +- .../google_calendar/google_calendar.py | 8 +- .../google_contacts/google_contacts.js | 7 +- .../google_contacts/google_contacts.py | 121 +++----------- .../doctype/google_drive/google_drive.js | 7 +- .../doctype/google_drive/google_drive.py | 115 +++---------- .../google_settings/google_settings.py | 4 - frappe/integrations/google_oauth.py | 158 ++++++++++++++++++ .../website_settings/google_indexing.py | 95 +++-------- .../website_settings/website_settings.js | 8 +- .../website_settings/website_settings.py | 33 +--- 11 files changed, 247 insertions(+), 312 deletions(-) create mode 100644 frappe/integrations/google_oauth.py diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 65053bab3d..1e3e995669 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -603,5 +603,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index d8dc7fab1d..09ed012454 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -13,7 +13,7 @@ from googleapiclient.errors import HttpError import frappe from frappe import _ -from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document from frappe.utils import ( add_days, @@ -90,7 +90,7 @@ class GoogleCalendar(Document): } try: - r = requests.post(get_auth_url(), data=data).json() + r = requests.post(GoogleOAuth.OAUTH_URL, data=data).json() except requests.exceptions.HTTPError: button_label = frappe.bold(_("Allow Google Calendar Access")) frappe.throw( @@ -130,7 +130,7 @@ def authorize_access(g_calendar, reauthorize=None): "redirect_uri": redirect_uri, "grant_type": "authorization_code", } - r = requests.post(get_auth_url(), data=data).json() + r = requests.post(GoogleOAuth.OAUTH_URL, data=data).json() if "refresh_token" in r: frappe.db.set_value( @@ -191,7 +191,7 @@ def get_google_calendar_object(g_calendar): credentials_dict = { "token": account.get_access_token(), "refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False), - "token_uri": get_auth_url(), + "token_uri": GoogleOAuth.OAUTH_URL, "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), "scopes": "https://www.googleapis.com/auth/calendar/v3", diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.js b/frappe/integrations/doctype/google_contacts/google_contacts.js index 7cbef46699..6e8035f38d 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.js +++ b/frappe/integrations/doctype/google_contacts/google_contacts.js @@ -37,16 +37,11 @@ frappe.ui.form.on('Google Contacts', { } }, authorize_google_contacts_access: function(frm) { - let reauthorize = 0; - if(frm.doc.authorization_code) { - reauthorize = 1; - } - frappe.call({ method: "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", args: { "g_contact": frm.doc.name, - "reauthorize": reauthorize + "reauthorize": frm.doc.authorization_code ? 1 : 0 }, callback: function(r) { if(!r.exc) { diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 5e4869be43..51ebcdd730 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -2,19 +2,14 @@ # License: MIT. See LICENSE -import google.oauth2.credentials -import requests -from googleapiclient.discovery import build from googleapiclient.errors import HttpError import frappe from frappe import _ -from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document from frappe.utils import get_request_site_address -SCOPES = "https://www.googleapis.com/auth/contacts" - class GoogleContacts(Document): def validate(self): @@ -22,120 +17,56 @@ class GoogleContacts(Document): frappe.throw(_("Enable Google API in Google Settings.")) def get_access_token(self): - google_settings = frappe.get_doc("Google Settings") - - if not google_settings.enable: - frappe.throw(_("Google Contacts Integration is disabled.")) - if not self.refresh_token: button_label = frappe.bold(_("Allow Google Contacts Access")) raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label)) - data = { - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False), - "grant_type": "refresh_token", - "scope": SCOPES, - } - - try: - r = requests.post(get_auth_url(), data=data).json() - except requests.exceptions.HTTPError: - button_label = frappe.bold(_("Allow Google Contacts Access")) - frappe.throw( - _( - "Something went wrong during the token generation. Click on {0} to generate a new one." - ).format(button_label) - ) + oauth_obj = GoogleOAuth("contacts") + r = oauth_obj.refresh_access_token( + self.get_password(fieldname="refresh_token", raise_exception=False) + ) return r.get("access_token") -@frappe.whitelist() -def authorize_access(g_contact, reauthorize=None): +@frappe.whitelist(methods=["POST", "GET"]) +def authorize_access(g_contact, reauthorize=False, code=None): """ If no Authorization code get it from Google and then request for Refresh Token. Google Contact Name is set to flags to set_value after Authorization Code is obtained. """ - google_settings = frappe.get_doc("Google Settings") - google_contact = frappe.get_doc("Google Contacts", g_contact) - - redirect_uri = ( - get_request_site_address(True) - + "?cmd=frappe.integrations.doctype.google_contacts.google_contacts.google_callback" + oauth_code = ( + frappe.db.get_value("Google Contacts", g_contact, "authorization_code") if not code else code ) + oauth_obj = GoogleOAuth("contacts") - if not google_contact.authorization_code or reauthorize: - frappe.cache().hset("google_contacts", "google_contact", google_contact.name) - return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri) - else: - try: - data = { - "code": google_contact.authorization_code, - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password( - fieldname="client_secret", raise_exception=False - ), - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - } - r = requests.post(get_auth_url(), data=data).json() - - if "refresh_token" in r: - frappe.db.set_value( - "Google Contacts", google_contact.name, "refresh_token", r.get("refresh_token") - ) - frappe.db.commit() - - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = f"/app/Form/Google%20Contacts/{google_contact.name}" - - frappe.msgprint(_("Google Contacts has been configured.")) - except Exception as e: - frappe.throw(e) - - -def get_authentication_url(client_id=None, redirect_uri=None): - return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( - client_id, SCOPES, redirect_uri + if not oauth_code or reauthorize: + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", + "g_contact": g_contact, + "redirect": "/app/Form/Google%20Contacts/{}".format(g_contact), + }, ) - } - -@frappe.whitelist() -def google_callback(code=None): - """ - Authorization code is sent to callback as per the API configuration - """ - google_contact = frappe.cache().hget("google_contacts", "google_contact") - frappe.db.set_value("Google Contacts", google_contact, "authorization_code", code) - frappe.db.commit() - - authorize_access(google_contact) + frappe.db.set_value("Google Contacts", g_contact, "authorization_code", oauth_code) + r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + if r: + frappe.db.set_value("Google Contacts", g_contact, "refresh_token", r.get("refresh_token")) def get_google_contacts_object(g_contact): """ Returns an object of Google Calendar along with Google Calendar doc. """ - google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Contacts", g_contact) + oauth_obj = GoogleOAuth("contacts") - credentials_dict = { - "token": account.get_access_token(), - "refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False), - "token_uri": get_auth_url(), - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/contacts", - } - - credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_contacts = build( - serviceName="people", version="v1", credentials=credentials, static_discovery=False + google_contacts = oauth_obj.get_google_service_object( + account.get_access_token(), + account.get_password(fieldname="indexing_refresh_token", raise_exception=False), ) return google_contacts, account diff --git a/frappe/integrations/doctype/google_drive/google_drive.js b/frappe/integrations/doctype/google_drive/google_drive.js index c314d02e7e..b38c0fb8e6 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.js +++ b/frappe/integrations/doctype/google_drive/google_drive.js @@ -41,15 +41,10 @@ frappe.ui.form.on('Google Drive', { } }, authorize_google_drive_access: function(frm) { - let reauthorize = 0; - if (frm.doc.authorization_code) { - reauthorize = 1; - } - frappe.call({ method: "frappe.integrations.doctype.google_drive.google_drive.authorize_access", args: { - "reauthorize": reauthorize + "reauthorize": frm.doc.authorization_code ? 1 : 0 }, callback: function(r) { if (!r.exc) { diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index c472cbc741..b9416e6669 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -4,15 +4,12 @@ import os from urllib.parse import quote -import google.oauth2.credentials -import requests from apiclient.http import MediaFileUpload -from googleapiclient.discovery import build from googleapiclient.errors import HttpError import frappe from frappe import _ -from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.google_oauth import GoogleOAuth from frappe.integrations.offsite_backup_utils import ( get_latest_backup_file, send_email, @@ -23,8 +20,6 @@ from frappe.utils import get_backups_path, get_bench_path, get_request_site_addr from frappe.utils.background_jobs import enqueue from frappe.utils.backups import new_backup -SCOPES = "https://www.googleapis.com/auth/drive" - class GoogleDrive(Document): def validate(self): @@ -33,118 +28,56 @@ class GoogleDrive(Document): self.backup_folder_id = "" def get_access_token(self): - google_settings = frappe.get_doc("Google Settings") - - if not google_settings.enable: - frappe.throw(_("Google Integration is disabled.")) - if not self.refresh_token: button_label = frappe.bold(_("Allow Google Drive Access")) raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label)) - data = { - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False), - "grant_type": "refresh_token", - "scope": SCOPES, - } - - try: - r = requests.post(get_auth_url(), data=data).json() - except requests.exceptions.HTTPError: - button_label = frappe.bold(_("Allow Google Drive Access")) - frappe.throw( - _( - "Something went wrong during the token generation. Click on {0} to generate a new one." - ).format(button_label) - ) + oauth_obj = GoogleOAuth("drive") + r = oauth_obj.refresh_access_token( + self.get_password(fieldname="refresh_token", raise_exception=False) + ) return r.get("access_token") -@frappe.whitelist() -def authorize_access(reauthorize=None): +@frappe.whitelist(methods=["POST", "GET"]) +def authorize_access(reauthorize=False, code=None): """ If no Authorization code get it from Google and then request for Refresh Token. Google Contact Name is set to flags to set_value after Authorization Code is obtained. """ - google_settings = frappe.get_doc("Google Settings") - google_drive = frappe.get_doc("Google Drive") - - redirect_uri = ( - get_request_site_address(True) - + "?cmd=frappe.integrations.doctype.google_drive.google_drive.google_callback" + oauth_code = ( + frappe.db.get_value("Google Drive", "Google Drive", "authorization_code") if not code else code ) + oauth_obj = GoogleOAuth("drive") - if not google_drive.authorization_code or reauthorize: + if not oauth_code or reauthorize: if reauthorize: frappe.db.set_value("Google Drive", None, "backup_folder_id", "") - return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri) - else: - try: - data = { - "code": google_drive.authorization_code, - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password( - fieldname="client_secret", raise_exception=False - ), - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - } - r = requests.post(get_auth_url(), data=data).json() - - if "refresh_token" in r: - frappe.db.set_value("Google Drive", google_drive.name, "refresh_token", r.get("refresh_token")) - frappe.db.commit() - - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = "/app/Form/{}".format(quote("Google Drive")) - - frappe.msgprint(_("Google Drive has been configured.")) - except Exception as e: - frappe.throw(e) - - -def get_authentication_url(client_id, redirect_uri): - return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( - client_id, SCOPES, redirect_uri + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.integrations.doctype.google_drive.google_drive.authorize_access", + "redirect": "/app/Form/{0}".format(quote("Google Drive")), + }, ) - } - -@frappe.whitelist() -def google_callback(code=None): - """ - Authorization code is sent to callback as per the API configuration - """ - frappe.db.set_value("Google Drive", None, "authorization_code", code) - frappe.db.commit() - - authorize_access() + frappe.db.set_value("Google Drive", None, "authorization_code", oauth_code) + r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + frappe.db.set_value("Google Drive", "Google Drive", "refresh_token", r.get("refresh_token")) def get_google_drive_object(): """ Returns an object of Google Drive. """ - google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Drive") + oauth_obj = GoogleOAuth("drive") - credentials_dict = { - "token": account.get_access_token(), - "refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False), - "token_uri": get_auth_url(), - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/drive/v3", - } - - credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_drive = build( - serviceName="drive", version="v3", credentials=credentials, static_discovery=False + google_drive = oauth_obj.get_google_service_object( + account.get_access_token(), + account.get_password(fieldname="indexing_refresh_token", raise_exception=False), ) return google_drive, account diff --git a/frappe/integrations/doctype/google_settings/google_settings.py b/frappe/integrations/doctype/google_settings/google_settings.py index c70a4b531f..e464e0d090 100644 --- a/frappe/integrations/doctype/google_settings/google_settings.py +++ b/frappe/integrations/doctype/google_settings/google_settings.py @@ -9,10 +9,6 @@ class GoogleSettings(Document): pass -def get_auth_url(): - return "https://www.googleapis.com/oauth2/v4/token" - - @frappe.whitelist() def get_file_picker_settings(): """Return all the data FileUploader needs to start the Google Drive Picker.""" diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py new file mode 100644 index 0000000000..f8731a1e6a --- /dev/null +++ b/frappe/integrations/google_oauth.py @@ -0,0 +1,158 @@ +import json +from typing import Dict, Union + +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from requests import post, get + +import frappe + +CALLBACK_METHOD = "/api/method/frappe.integrations.google_oauth.callback" +_SCOPES = { + "mail": ("https://mail.google.com/"), + "contacts": ("https://www.googleapis.com/auth/contacts"), + "drive": ("https://www.googleapis.com/auth/drive"), + "indexing": ("https://www.googleapis.com/auth/indexing") +} +_SERVICES = { + "contacts": ("people", "v1"), + "drive": ("drive", "v3"), + "indexing": ("indexing", "v3") +} + + +class GoogleOAuth: + OAUTH_URL = "https://oauth2.googleapis.com/token" + + def __init__(self, domain: str, validate: bool=True): + self.google_settings = frappe.get_single("Google Settings") + self.domain = domain.lower() + self.scopes = " ".join(_SCOPES[self.domain]) if isinstance(_SCOPES[self.domain], (list, tuple)) else _SCOPES[self.domain] + + if validate: + self.validate_google_settings() + + def validate_google_settings(self): + if not self.google_settings.enable: + frappe.throw(frappe._("Please enable Google Settings before continuing.")) + + if not (self.google_settings.client_id and self.google_settings.client_secret): + frappe.throw(frappe._("Please update Google Settings before continuing.")) + + def authorize(self, oauth_code: str, site_address: str) -> Dict[str, Union[str, int]]: + data = { + "code": oauth_code, + "client_id": self.google_settings.client_id, + "client_secret": self.google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), + "grant_type": "authorization_code", + "scope": self.scopes, + "redirect_uri": site_address + CALLBACK_METHOD, + } + + return handle_response( + post(self.OAUTH_URL, data=data).json(), + "Google Oauth Authorization Error", + "Something went wrong during the authorization.", + ) + + def refresh_access_token(self, refresh_token: str) -> Dict[str, Union[str, int]]: + data = { + "client_id": self.google_settings.client_id, + "client_secret": self.google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), + "refresh_token": refresh_token, + "grant_type": "refresh_token", + "scope": self.scopes, + } + + return handle_response( + post(self.OAUTH_URL, data=data).json(), + "Google Oauth Access Token Refresh Error", + "Something went wrong during the access token generation.", + raise_err=True, + ) + + def get_authentication_url( + self, site_address: str, state: Dict[str, str] = None + ) -> Dict[str, str]: + """Return authentication url with the client id and redirect uri.""" + + state = json.dumps(state) + callback_url = site_address + CALLBACK_METHOD + + return { + "url": "https://accounts.google.com/o/oauth2/v2/auth?" + + "access_type=offline&response_type=code&prompt=consent&include_granted_scopes=true&" + + "client_id={0}&scope={1}&redirect_uri={2}&state={3}".format( + self.google_settings.client_id, self.scopes, callback_url, state + ) + } + + def get_google_service_object(self, access_token: str, refresh_token: str): + credentials_dict = { + "token": access_token, + "refresh_token": refresh_token, + "token_uri": self.OAUTH_URL, + "client_id": self.google_settings.client_id, + "client_secret": self.google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), + "scopes": self.scopes, + } + + return build( + serviceName=_SERVICES[self.domain][0], + version=_SERVICES[self.domain][1], + credentials=Credentials(**credentials_dict), + static_discovery=False, + ) + + +def handle_response( + response: Dict[str, Union[str, int]], + error_title: str, + error_message: str, + raise_err: bool = False, +): + if "error" in response: + frappe.log_error( + frappe._(error_title), frappe._(response.get("error_description", error_message)) + ) + + if raise_err: + frappe.throw(frappe._(error_title), frappe._(error_message)) + + return {} + + return response + + +def is_valid_access_token(access_token: str) -> bool: + response = get( + "https://oauth2.googleapis.com/tokeninfo", + params={'access_token': access_token} + ).json() + + if "error" in response: + return False + + return True + + +@frappe.whitelist(methods=["GET"]) +def callback(state: str, code: str = None, error: str = None) -> None: + state = json.loads(state) + redirect = state.pop("redirect", "/app") + + if not error: + state.update({"code": code}) + frappe.get_attr(state.pop("method"))(**state) + + # GET request, hence using commit to persist changes + frappe.db.commit() + + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = redirect diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index d0657a1928..b89918dff2 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -4,99 +4,52 @@ from urllib.parse import quote -import google.oauth2.credentials -import requests -from googleapiclient.discovery import build from googleapiclient.errors import HttpError import frappe from frappe import _ -from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.google_oauth import GoogleOAuth from frappe.utils import get_request_site_address -SCOPES = "https://www.googleapis.com/auth/indexing" - -@frappe.whitelist() -def authorize_access(reauthorize=None): +@frappe.whitelist(methods=["POST", "GET"]) +def authorize_access(reauthorize=False, code=None): """If no Authorization code get it from Google and then request for Refresh Token.""" - google_settings = frappe.get_doc("Google Settings") - website_settings = frappe.get_doc("Website Settings") - - redirect_uri = ( - get_request_site_address(True) - + "?cmd=frappe.website.doctype.website_settings.google_indexing.google_callback" + oauth_code = ( + frappe.db.get_value("Website Settings", "Website Settings", "indexing_authorization_code") + if not code + else code ) - if not website_settings.indexing_authorization_code or reauthorize: - return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri) - else: - try: - data = { - "code": website_settings.indexing_authorization_code, - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password( - fieldname="client_secret", raise_exception=False - ), - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - } - res = requests.post(get_auth_url(), data=data).json() + oauth_obj = GoogleOAuth("indexing") - if "refresh_token" in res: - frappe.db.set_value( - "Website Settings", website_settings.name, "indexing_refresh_token", res.get("refresh_token") - ) - frappe.db.commit() - - frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = "/app/Form/{}".format(quote("Website Settings")) - - frappe.msgprint(_("Google Indexing has been configured.")) - except Exception as e: - frappe.throw(e) - - -def get_authentication_url(client_id, redirect_uri): - """Return authentication url with the client id and redirect uri.""" - return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( - client_id, SCOPES, redirect_uri + if not oauth_code or reauthorize: + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.website.doctype.website_settings.google_indexing.authorize_access", + "redirect": "/app/Form/{0}".format(quote("Website Settings")), + }, ) - } - -@frappe.whitelist() -def google_callback(code=None): - """Authorization code is sent to callback as per the API configuration.""" - frappe.db.set_value("Website Settings", None, "indexing_authorization_code", code) - frappe.db.commit() - - authorize_access() + frappe.db.set_value("Website Settings", None, "indexing_authorization_code", oauth_code) + res = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + frappe.db.set_value( + "Website Settings", "Website Settings", "indexing_refresh_token", res.get("refresh_token") + ) def get_google_indexing_object(): """Returns an object of Google Indexing object.""" - google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Website Settings") + oauth_obj = GoogleOAuth("indexing") - credentials_dict = { - "token": account.get_access_token(), - "refresh_token": account.get_password(fieldname="indexing_refresh_token", raise_exception=False), - "token_uri": get_auth_url(), - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/indexing", - } - - credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_indexing = build( - serviceName="indexing", version="v3", credentials=credentials, static_discovery=False + return oauth_obj.get_google_service_object( + account.get_access_token(), + account.get_password(fieldname="indexing_refresh_token", raise_exception=False), ) - return google_indexing - def publish_site(url, operation_type="URL_UPDATED"): """Send an update/remove url request.""" diff --git a/frappe/website/doctype/website_settings/website_settings.js b/frappe/website/doctype/website_settings/website_settings.js index 2f15b4c00e..17ff4bed6e 100644 --- a/frappe/website/doctype/website_settings/website_settings.js +++ b/frappe/website/doctype/website_settings/website_settings.js @@ -42,16 +42,10 @@ frappe.ui.form.on('Website Settings', { }, authorize_api_indexing_access: function(frm) { - let reauthorize = 0; - if (frm.doc.authorization_code) { - reauthorize = 1; - } - frappe.call({ method: "frappe.website.doctype.website_settings.google_indexing.authorize_access", args: { - "g_indexing": frm.doc.name, - "reauthorize": reauthorize + "reauthorize": frm.doc.indexing_authorization_code ? 1 : 0 }, callback: function(r) { if (!r.exc) { diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index 4088be88c2..31e452c03f 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -4,12 +4,10 @@ from urllib.parse import quote import frappe from frappe import _ -from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document from frappe.utils import encode, get_request_site_address -INDEXING_SCOPES = "https://www.googleapis.com/auth/indexing" - class WebsiteSettings(Document): def validate(self): @@ -89,34 +87,15 @@ class WebsiteSettings(Document): frappe.clear_cache() def get_access_token(self): - import requests - - google_settings = frappe.get_doc("Google Settings") - - if not google_settings.enable: - frappe.throw(_("Google Integration is disabled.")) - if not self.indexing_refresh_token: button_label = frappe.bold(_("Allow API Indexing Access")) raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label)) - data = { - "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "refresh_token": self.get_password(fieldname="indexing_refresh_token", raise_exception=False), - "grant_type": "refresh_token", - "scope": INDEXING_SCOPES, - } - - try: - res = requests.post(get_auth_url(), data=data).json() - except requests.exceptions.HTTPError: - button_label = frappe.bold(_("Allow Google Indexing Access")) - frappe.throw( - _( - "Something went wrong during the token generation. Click on {0} to generate a new one." - ).format(button_label) - ) + oauth_obj = GoogleOAuth("indexing") + res = oauth_obj.refresh_access_token( + self.get_password(fieldname="indexing_refresh_token", raise_exception=False) + ) + print(res) return res.get("access_token") From 07a577af867fe7c4c2486106a7e559b5e9c9717a Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 27 May 2022 11:56:34 +0530 Subject: [PATCH 095/201] feat: google oauth for google emails * used unique constraint on email_id in Email Account Doctype --- frappe/core/doctype/communication/mixins.py | 4 +- frappe/core/doctype/user/user.py | 10 +-- .../doctype/email_account/email_account.js | 17 +++++ .../doctype/email_account/email_account.json | 36 ++++++++- .../doctype/email_account/email_account.py | 74 ++++++++++++++----- frappe/email/receive.py | 27 ++++++- frappe/email/smtp.py | 31 +++++++- frappe/email/utils.py | 58 +++++++++++++++ frappe/integrations/google_oauth.py | 21 ++++-- .../website_settings/website_settings.py | 1 - 10 files changed, 233 insertions(+), 46 deletions(-) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 695b8bebae..ad74b47026 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -92,7 +92,7 @@ class CommunicationEmailMixin: cc_list = self.mail_cc( is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender ) - return [self.get_email_with_displayname(email) for email in cc_list] + return [self.get_email_with_displayname(email) for email in cc_list if email] def mail_bcc(self, is_inbound_mail_communcation=False): """ @@ -120,7 +120,7 @@ class CommunicationEmailMixin: def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) - return [self.get_email_with_displayname(email) for email in bcc_list] + return [self.get_email_with_displayname(email) for email in bcc_list if email] def mail_sender(self): email_account = self.get_outgoing_email_account() diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index d6fe883fae..eb3c11a4db 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -763,19 +763,11 @@ def has_email_account(email): @frappe.whitelist(allow_guest=False) def get_email_awaiting(user): - waiting = frappe.get_all( + return frappe.get_all( "User Email", fields=["email_account", "email_id"], filters={"awaiting_password": 1, "parent": user}, ) - if waiting: - return waiting - else: - user_email_table = DocType("User Email") - frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where( - user_email_table.parent == user - ).run() - return False def ask_pass_update(): diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index a357126a48..af6947c07c 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -1,6 +1,7 @@ frappe.email_defaults = { "GMail": { "email_server": "imap.gmail.com", + "incoming_port": 993, "use_ssl": 1, "enable_outgoing": 1, "smtp_server": "smtp.gmail.com", @@ -142,6 +143,22 @@ frappe.ui.form.on("Email Account", { } }, + authorize_google_api_access: function(frm) { + frappe.call({ + method: "frappe.email.doctype.email_account.email_account.authorize_google_access", + args: { + "email_account": frm.doc.name, + "reauthorize": frm.doc.refresh_token ? 1 : 0 + }, + callback: function(r) { + if (!r.exc) { + frm.save(); + window.open(r.message.url); + } + } + }); + }, + email_id:function(frm) { //pull domain and if no matching domain go create one frm.events.update_domain(frm); diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 1e3e995669..34572c600d 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -18,6 +18,10 @@ "awaiting_password", "ascii_encode_password", "column_break_10", + "use_google_oauth", + "authorize_google_api_access", + "google_refresh_token", + "google_access_token", "login_id_is_different", "login_id", "mailbox_settings", @@ -79,7 +83,8 @@ "in_list_view": 1, "label": "Email Address", "options": "Email", - "reqd": 1 + "reqd": 1, + "unique": 1 }, { "default": "0", @@ -576,12 +581,39 @@ "fieldname": "section_break_25", "fieldtype": "Section Break", "label": "IMAP Details" + }, + { + "default": "0", + "depends_on": "eval: doc.service === \"GMail\"", + "fieldname": "use_google_oauth", + "fieldtype": "Check", + "label": "Use Google OAuth" + }, + { + "depends_on": "eval: doc.service === \"GMail\" && doc.use_google_oauth", + "fieldname": "authorize_google_api_access", + "fieldtype": "Button", + "label": "Authorize API Access" + }, + { + "fieldname": "google_refresh_token", + "fieldtype": "Data", + "hidden": 1, + "label": "Google Refresh Token", + "read_only": 1 + }, + { + "fieldname": "google_access_token", + "fieldtype": "Small Text", + "hidden": 1, + "label": "Google Access Token", + "read_only": 1 } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-11-30 09:03:25.728637", + "modified": "2022-05-29 18:11:06.463553", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 02afe4f4b5..f73aa02183 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -8,6 +8,7 @@ import socket import time from datetime import datetime, timedelta from poplib import error_proto +from urllib.parse import quote import frappe from frappe import _, are_emails_muted, safe_encode @@ -15,8 +16,16 @@ from frappe.desk.form import assign_to from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from frappe.email.smtp import SMTPServer from frappe.email.utils import get_port +from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document -from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address +from frappe.utils import ( + cint, + comma_or, + cstr, + get_request_site_address, + parse_addr, + validate_email_address, +) from frappe.utils.background_jobs import enqueue, get_jobs from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template @@ -63,6 +72,7 @@ class EmailAccount(Document): def validate(self): """Validate Email Address and check POP3/IMAP and SMTP connections is enabled.""" + if self.email_id: validate_email_address(self.email_id, True) @@ -76,25 +86,11 @@ class EmailAccount(Document): if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0: frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id))) - duplicate_email_account = frappe.get_all( - "Email Account", filters={"email_id": self.email_id, "name": ("!=", self.name)} - ) - if duplicate_email_account: - frappe.throw( - _("Email ID must be unique, Email Account already exists for {0}").format( - frappe.bold(self.email_id) - ) - ) - if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - if ( - not self.awaiting_password - and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch - ): - if self.password or self.smtp_server in ("127.0.0.1", "localhost"): + if not frappe.local.flags.in_install and (self.use_google_oauth or not self.awaiting_password): + if self.google_refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -103,7 +99,10 @@ class EmailAccount(Document): self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - frappe.throw(_("Password is required or select Awaiting Password")) + if self.use_google_oauth: + frappe.throw(_("Please Authorize Google by using `Authorize API access` button")) + else: + frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: if not self.send_notification_to: @@ -208,6 +207,9 @@ class EmailAccount(Document): "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, + "use_google_oauth": self.use_google_oauth or 0, + "google_refresh_token": getattr(self, "google_refresh_token", None), + "google_access_token": getattr(self, "google_access_token", None), } ) @@ -274,7 +276,9 @@ class EmailAccount(Document): @property def _password(self): - raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test) + raise_exception = not ( + self.use_google_oauth or self.no_smtp_authentication or frappe.flags.in_test + ) return self.get_password(raise_exception=raise_exception) @property @@ -405,12 +409,16 @@ class EmailAccount(Document): def sendmail_config(self): return { + "email_account": self.name, "server": self.smtp_server, "port": cint(self.smtp_port), "login": getattr(self, "login_id", None) or self.email_id, "password": self._password, "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), + "use_google_oauth": self.use_google_oauth or 0, + "google_refresh_token": getattr(self, "google_refresh_token", None), + "google_access_token": getattr(self, "google_access_token", None), } def get_smtp_server(self): @@ -491,6 +499,7 @@ class EmailAccount(Document): def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages["uid_list"][index] if messages.get("uid_list") else None + uid = uid.decode() if isinstance(uid, bytes) else uid seen_status = messages.get("seen_status", {}).get(uid) if self.email_sync_option != "UNSEEN" or seen_status != "SEEN": # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' @@ -771,14 +780,18 @@ def notify_unreplied(): def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" + if frappe.cache().get_value("workers:no-internet") == True: if test_internet(): frappe.cache().set_value("workers:no-internet", False) else: return + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] for email_account in frappe.get_list( - "Email Account", filters={"enable_incoming": 1, "awaiting_password": 0} + "Email Account", + filters={"enable_incoming": 1}, + or_filters={"awaiting_password": 0, "use_google_oauth": 1}, ): if now: pull_from_email_account(email_account.name) @@ -906,3 +919,24 @@ def set_email_password(email_account, user, password): return False return True + + +@frappe.whitelist(methods=["POST"]) +def authorize_google_access(email_account, reauthorize=False, code=None): + doctype = "Email Account" + refresh_token = frappe.db.get_value(doctype, email_account, "google_refresh_token") + oauth_obj = GoogleOAuth("mail") + + if not (refresh_token or code) or reauthorize: + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.email.doctype.email_account.email_account.authorize_google_access", + "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), + "email_account": email_account, + }, + ) + + res = oauth_obj.authorize(code, get_request_site_address(True)) + frappe.db.set_value(doctype, email_account, "google_refresh_token", res.get("refresh_token")) + frappe.db.set_value(doctype, email_account, "google_access_token", res.get("access_token")) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 93e1a68285..81ef7927c1 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -18,6 +18,7 @@ from email_reply_parser import EmailReplyParser import frappe from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file import MaxFileSizeReachedError, get_random_filename +from frappe.email.utils import connect_google_oauth from frappe.utils import ( add_days, cint, @@ -98,7 +99,18 @@ class EmailServer: self.imap = Timed_IMAP4( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - self.imap.login(self.settings.username, self.settings.password) + + if self.settings.use_google_oauth and self.settings.google_refresh_token: + connect_google_oauth( + self.imap, + self.settings.email_account, + self.settings.username, + self.settings.google_access_token, + self.settings.google_refresh_token, + ) + else: + self.imap.login(self.settings.username, self.settings.password) + # connection established! return True @@ -119,8 +131,17 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - self.pop.user(self.settings.username) - self.pop.pass_(self.settings.password) + if self.settings.use_google_oauth and self.settings.google_refresh_token: + connect_google_oauth( + self.pop, + self.settings.email_account, + self.settings.username, + self.settings.google_access_token, + self.settings.google_refresh_token, + ) + else: + self.pop.user(self.settings.username) + self.pop.pass_(self.settings.password) # connection established! return True diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 1211419de1..cd31aa0d5c 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -5,6 +5,7 @@ import smtplib import frappe from frappe import _ +from frappe.email.utils import connect_google_oauth from frappe.utils import cint, cstr @@ -43,13 +44,29 @@ def send(email, append_to=None, retry=1): class SMTPServer: - def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None): + def __init__( + self, + server, + login=None, + email_account=None, + password=None, + port=None, + use_tls=None, + use_ssl=None, + use_google_oauth=0, + google_refresh_token=None, + google_access_token=None, + ): self.login = login + self.email_account = email_account self.password = password self._server = server self._port = port self.use_tls = use_tls self.use_ssl = use_ssl + self.use_google_oauth = use_google_oauth + self.google_refresh_token = google_refresh_token + self.google_access_token = google_access_token self._session = None if not self.server: @@ -91,7 +108,17 @@ class SMTPServer: ) self.secure_session(_session) - if self.login and self.password: + + if self.use_google_oauth and self.google_refresh_token: + connect_google_oauth( + _session, + self.email_account, + self.login, + self.google_access_token, + self.google_refresh_token, + ) + + elif self.password: res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly diff --git a/frappe/email/utils.py b/frappe/email/utils.py index 147284a625..97cfaaa1e7 100644 --- a/frappe/email/utils.py +++ b/frappe/email/utils.py @@ -1,8 +1,13 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +import base64 import imaplib import poplib +import smtplib +from typing import Union +from frappe import _, db, log_error, throw +from frappe.integrations.google_oauth import GoogleAuthenticationError, GoogleOAuth from frappe.utils import cint @@ -15,3 +20,56 @@ def get_port(doc): doc.incoming_port = poplib.POP3_SSL_PORT if doc.use_ssl else poplib.POP3_PORT return cint(doc.incoming_port) + + +def connect_google_oauth( + connection_obj: Union[imaplib.IMAP4, poplib.POP3, smtplib.SMTP], + email_account: str, + email: str, + google_access_token: str, + google_refresh_token: str, + retry: int = 0, +) -> None: + auth_string = "user=%s\1auth=Bearer %s\1\1" % (email, google_access_token) + mechanism = "XOAUTH2" + _func = lambda x: auth_string # noqa: E731 + + try: + if isinstance(connection_obj, poplib.POP3): + # poplib doesn't have AUTH command implementation + res = connection_obj._shortcmd( + "AUTH {0} {1}".format(mechanism, base64.b64encode(bytes(auth_string, "utf-8")).decode("utf-8")) + ) + + if not res.startswith(b"+OK"): + raise + + elif isinstance(connection_obj, imaplib.IMAP4): + connection_obj.authenticate(mechanism, _func) + + else: + # SMTP + connection_obj.auth(mechanism, _func, initial_response_ok=False) + + except Exception: + # maybe the access token expired - refreshing + access_token = refresh_google_access_token(email_account, google_refresh_token) + + if not access_token or retry > 0: + throw( + _("Google Authentication Failed. Please Check and Update the credentials."), + GoogleAuthenticationError, + _("Google Authentication Error"), + ) + + connect_google_oauth( + connection_obj, email_account, email, access_token, google_refresh_token, retry + 1 + ) + + +def refresh_google_access_token(email_account: str, google_refresh_token: str) -> str: + oauth_obj = GoogleOAuth("mail") + res = oauth_obj.refresh_access_token(google_refresh_token) + db.set_value("Email Account", email_account, "google_access_token", res.get("access_token")) + + return res.get("access_token") diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index f8731a1e6a..da67f2ccf5 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -3,7 +3,7 @@ from typing import Dict, Union from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from requests import post, get +from requests import get, post import frappe @@ -12,22 +12,30 @@ _SCOPES = { "mail": ("https://mail.google.com/"), "contacts": ("https://www.googleapis.com/auth/contacts"), "drive": ("https://www.googleapis.com/auth/drive"), - "indexing": ("https://www.googleapis.com/auth/indexing") + "indexing": ("https://www.googleapis.com/auth/indexing"), } _SERVICES = { "contacts": ("people", "v1"), "drive": ("drive", "v3"), - "indexing": ("indexing", "v3") + "indexing": ("indexing", "v3"), } +class GoogleAuthenticationError(Exception): + pass + + class GoogleOAuth: OAUTH_URL = "https://oauth2.googleapis.com/token" - def __init__(self, domain: str, validate: bool=True): + def __init__(self, domain: str, validate: bool = True): self.google_settings = frappe.get_single("Google Settings") self.domain = domain.lower() - self.scopes = " ".join(_SCOPES[self.domain]) if isinstance(_SCOPES[self.domain], (list, tuple)) else _SCOPES[self.domain] + self.scopes = ( + " ".join(_SCOPES[self.domain]) + if isinstance(_SCOPES[self.domain], (list, tuple)) + else _SCOPES[self.domain] + ) if validate: self.validate_google_settings() @@ -132,8 +140,7 @@ def handle_response( def is_valid_access_token(access_token: str) -> bool: response = get( - "https://oauth2.googleapis.com/tokeninfo", - params={'access_token': access_token} + "https://oauth2.googleapis.com/tokeninfo", params={"access_token": access_token} ).json() if "error" in response: diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index 31e452c03f..bd634b4f32 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -95,7 +95,6 @@ class WebsiteSettings(Document): res = oauth_obj.refresh_access_token( self.get_password(fieldname="indexing_refresh_token", raise_exception=False) ) - print(res) return res.get("access_token") From cc878377be039e6f92d3044088d8fb83613b8c01 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 30 May 2022 16:04:08 +0530 Subject: [PATCH 096/201] fix: allow adding new emails with google oauth checked kinda works in a similar way as awaiting_password for new emails --- frappe/email/doctype/email_account/email_account.js | 3 +-- frappe/email/doctype/email_account/email_account.json | 4 ++-- frappe/email/doctype/email_account/email_account.py | 10 +++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index af6947c07c..4392e1db1d 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -135,7 +135,7 @@ frappe.ui.form.on("Email Account", { show_gmail_message_for_less_secure_apps: function(frm) { frm.dashboard.clear_headline(); - let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password."); + let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use Google OAuth."); let cta = __("Read the step by step guide here."); msg += ` ${cta}`; if (frm.doc.service==="GMail") { @@ -152,7 +152,6 @@ frappe.ui.form.on("Email Account", { }, callback: function(r) { if (!r.exc) { - frm.save(); window.open(r.message.url); } } diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 34572c600d..0f62e103e8 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -590,7 +590,7 @@ "label": "Use Google OAuth" }, { - "depends_on": "eval: doc.service === \"GMail\" && doc.use_google_oauth", + "depends_on": "eval: doc.service === \"GMail\" && doc.use_google_oauth && !doc.__islocal", "fieldname": "authorize_google_api_access", "fieldtype": "Button", "label": "Authorize API Access" @@ -613,7 +613,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-05-29 18:11:06.463553", + "modified": "2022-05-30 15:45:05.282867", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index f73aa02183..7779dea38c 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -100,7 +100,8 @@ class EmailAccount(Document): else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): if self.use_google_oauth: - frappe.throw(_("Please Authorize Google by using `Authorize API access` button")) + if not self.is_new(): + frappe.throw(_("Please Authorize Google by using `Authorize API access` button")) else: frappe.throw(_("Password is required or select Awaiting Password")) @@ -156,6 +157,13 @@ class EmailAccount(Document): enable_outgoing=self.enable_outgoing, ) + def after_insert(self): + if self.use_google_oauth and not self.google_refresh_token: + frappe.msgprint( + _("Please Authorize Google by using `Authorize API access` button"), + indicator="orange" + ) + def there_must_be_only_one_default(self): """If current Email Account is default, un-default all other accounts.""" for field in ("default_incoming", "default_outgoing"): From b930cb923b91c80fd3b33538c15ae161f863b76c Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 30 May 2022 17:25:08 +0530 Subject: [PATCH 097/201] fix: set X-Original-From header before replacing sender email/name This will help in knowing the original email id and sender of email and also bring consistency. --- frappe/email/email_body.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index b493ca2cb5..20f81cb89b 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -267,6 +267,7 @@ class EMail: validate_email_address(strip(self.sender), True) self.reply_to = validate_email_address(strip(self.reply_to) or self.sender, True) + self.set_header("X-Original-From", self.sender) self.replace_sender() self.replace_sender_name() @@ -279,16 +280,14 @@ class EMail: def replace_sender(self): if cint(self.email_account.always_use_account_email_id_as_sender): - self.set_header("X-Original-From", self.sender) - sender_name, sender_email = parse_addr(self.sender) + sender_name, _ = parse_addr(self.sender) self.sender = email.utils.formataddr( (str(Header(sender_name or self.email_account.name, "utf-8")), self.email_account.email_id) ) def replace_sender_name(self): if cint(self.email_account.always_use_account_name_as_sender_name): - self.set_header("X-Original-From", self.sender) - sender_name, sender_email = parse_addr(self.sender) + _, sender_email = parse_addr(self.sender) self.sender = email.utils.formataddr( (str(Header(self.email_account.name, "utf-8")), sender_email) ) From ebc586121054f3762be66d3f072da92158c8bc9c Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 01:14:04 +0530 Subject: [PATCH 098/201] feat: generic OAuth for email --- .../doctype/email_account/email_account.js | 7 +- .../doctype/email_account/email_account.json | 24 ++--- .../doctype/email_account/email_account.py | 47 ++++---- frappe/email/oauth.py | 100 ++++++++++++++++++ frappe/email/receive.py | 26 +++-- frappe/email/smtp.py | 27 ++--- frappe/email/utils.py | 59 +---------- 7 files changed, 176 insertions(+), 114 deletions(-) create mode 100644 frappe/email/oauth.py diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 4392e1db1d..4d84f69395 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -135,7 +135,7 @@ frappe.ui.form.on("Email Account", { show_gmail_message_for_less_secure_apps: function(frm) { frm.dashboard.clear_headline(); - let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use Google OAuth."); + let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use OAuth."); let cta = __("Read the step by step guide here."); msg += ` ${cta}`; if (frm.doc.service==="GMail") { @@ -143,11 +143,12 @@ frappe.ui.form.on("Email Account", { } }, - authorize_google_api_access: function(frm) { + authorize_api_access: function(frm) { frappe.call({ - method: "frappe.email.doctype.email_account.email_account.authorize_google_access", + method: "frappe.email.doctype.email_account.email_account.oauth_access", args: { "email_account": frm.doc.name, + "service": frm.doc.service, "reauthorize": frm.doc.refresh_token ? 1 : 0 }, callback: function(r) { diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 0f62e103e8..72be4f2e42 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -18,10 +18,10 @@ "awaiting_password", "ascii_encode_password", "column_break_10", - "use_google_oauth", - "authorize_google_api_access", - "google_refresh_token", - "google_access_token", + "use_oauth", + "authorize_api_access", + "refresh_token", + "access_token", "login_id_is_different", "login_id", "mailbox_settings", @@ -585,28 +585,28 @@ { "default": "0", "depends_on": "eval: doc.service === \"GMail\"", - "fieldname": "use_google_oauth", + "fieldname": "use_oauth", "fieldtype": "Check", - "label": "Use Google OAuth" + "label": "Use OAuth" }, { - "depends_on": "eval: doc.service === \"GMail\" && doc.use_google_oauth && !doc.__islocal", - "fieldname": "authorize_google_api_access", + "depends_on": "eval: doc.service === \"GMail\" && doc.use_oauth && !doc.__islocal", + "fieldname": "authorize_api_access", "fieldtype": "Button", "label": "Authorize API Access" }, { - "fieldname": "google_refresh_token", + "fieldname": "refresh_token", "fieldtype": "Data", "hidden": 1, - "label": "Google Refresh Token", + "label": "Refresh Token", "read_only": 1 }, { - "fieldname": "google_access_token", + "fieldname": "access_token", "fieldtype": "Small Text", "hidden": 1, - "label": "Google Access Token", + "label": "Access Token", "read_only": 1 } ], diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 7779dea38c..b2f8e1ed64 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -89,8 +89,11 @@ class EmailAccount(Document): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - if not frappe.local.flags.in_install and (self.use_google_oauth or not self.awaiting_password): - if self.google_refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): + if getattr(self, "service", "") != "GMail" and self.use_oauth: + self.use_oauth = 0 + + if not frappe.local.flags.in_install and (self.use_oauth or not self.awaiting_password): + if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -99,9 +102,9 @@ class EmailAccount(Document): self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - if self.use_google_oauth: + if self.use_oauth: if not self.is_new(): - frappe.throw(_("Please Authorize Google by using `Authorize API access` button")) + frappe.throw(_("Please Enable OAuth by using `Authorize API access` button")) else: frappe.throw(_("Password is required or select Awaiting Password")) @@ -158,9 +161,9 @@ class EmailAccount(Document): ) def after_insert(self): - if self.use_google_oauth and not self.google_refresh_token: + if self.use_oauth and not self.refresh_token: frappe.msgprint( - _("Please Authorize Google by using `Authorize API access` button"), + _("Please Enable OAuth by using `Authorize API access` button"), indicator="orange" ) @@ -211,13 +214,14 @@ class EmailAccount(Document): "host": self.email_server, "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, + "service": getattr(self, "service", None), "use_imap": self.use_imap, "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, - "use_google_oauth": self.use_google_oauth or 0, - "google_refresh_token": getattr(self, "google_refresh_token", None), - "google_access_token": getattr(self, "google_access_token", None), + "use_oauth": self.use_oauth or 0, + "refresh_token": getattr(self, "refresh_token", None), + "access_token": getattr(self, "access_token", None), } ) @@ -285,7 +289,7 @@ class EmailAccount(Document): @property def _password(self): raise_exception = not ( - self.use_google_oauth or self.no_smtp_authentication or frappe.flags.in_test + self.use_oauth or self.no_smtp_authentication or frappe.flags.in_test ) return self.get_password(raise_exception=raise_exception) @@ -424,9 +428,10 @@ class EmailAccount(Document): "password": self._password, "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), - "use_google_oauth": self.use_google_oauth or 0, - "google_refresh_token": getattr(self, "google_refresh_token", None), - "google_access_token": getattr(self, "google_access_token", None), + "service": getattr(self, "service", None), + "use_oauth": self.use_oauth or 0, + "refresh_token": getattr(self, "refresh_token", None), + "access_token": getattr(self, "access_token", None), } def get_smtp_server(self): @@ -799,7 +804,7 @@ def pull(now=False): for email_account in frappe.get_list( "Email Account", filters={"enable_incoming": 1}, - or_filters={"awaiting_password": 0, "use_google_oauth": 1}, + or_filters={"awaiting_password": 0, "use_oauth": 1}, ): if now: pull_from_email_account(email_account.name) @@ -930,9 +935,15 @@ def set_email_password(email_account, user, password): @frappe.whitelist(methods=["POST"]) -def authorize_google_access(email_account, reauthorize=False, code=None): +def oauth_access(email_account: str, reauthorize: bool = False, service: str = None): doctype = "Email Account" - refresh_token = frappe.db.get_value(doctype, email_account, "google_refresh_token") + refresh_token = frappe.db.get_value(doctype, email_account, "refresh_token") + + if service == "GMail": + return authorize_google_access(email_account, reauthorize, refresh_token, doctype) + + +def authorize_google_access(email_account, reauthorize: bool = False, refresh_token: str = None, doctype: str = "Email Account", code: str = None): oauth_obj = GoogleOAuth("mail") if not (refresh_token or code) or reauthorize: @@ -946,5 +957,5 @@ def authorize_google_access(email_account, reauthorize=False, code=None): ) res = oauth_obj.authorize(code, get_request_site_address(True)) - frappe.db.set_value(doctype, email_account, "google_refresh_token", res.get("refresh_token")) - frappe.db.set_value(doctype, email_account, "google_access_token", res.get("access_token")) + frappe.db.set_value(doctype, email_account, "refresh_token", res.get("refresh_token")) + frappe.db.set_value(doctype, email_account, "access_token", res.get("access_token")) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py new file mode 100644 index 0000000000..b4efac2e07 --- /dev/null +++ b/frappe/email/oauth.py @@ -0,0 +1,100 @@ +import base64 +from typing import Callable, Union +from imaplib import IMAP4 +from poplib import POP3 +from smtplib import SMTP + +import frappe +from frappe.integrations.google_oauth import GoogleOAuth + + +class OAuthenticationError(Exception): + pass + + +class Oauth: + def __init__(self, + conn: Union[IMAP4, POP3, SMTP], + email_account: str, + email: str, + access_token: str, + refresh_token: str, + service: str, + mechanism: str = "XOAUTH2", + ) -> None: + + self.email_account = email_account + self.email = email + self.service = service + self._mechanism = mechanism + self._conn = conn + self._access_token = access_token + self._refresh_token = refresh_token + + self.validate_implementation() + + def validate_implementation(self) -> None: + if self.service != "GMail": + raise NotImplementedError(f"Service {self.service} currently doesn't have oauth implementation.") + + @property + def _auth_string(self) -> str: + return "user=%s\1auth=Bearer %s\1\1" % (self.email, self._access_token) + + def connect(self, _retry: int = 0) -> None: + try: + if isinstance(self._conn, POP3): + res = self._connect_pop() + + if not res.startswith(b"+OK"): + raise + + elif isinstance(self._conn, IMAP4): + self._connect_imap() + + else: + # SMTP + self._connect_smtp() + + except Exception: + # maybe the access token expired - refreshing + access_token = self._refresh_access_token() + print(self._auth_string) + + if not access_token or _retry > 0: + frappe.throw( + frappe._("Authentication Failed. Please Check and Update the credentials."), + OAuthenticationError, + frappe._("OAuth Error"), + ) + + self._access_token = access_token + self.connect(_retry + 1) + + def _connect_pop(self) -> bytes: + # poplib doesn't have AUTH command implementation + res = self._conn._shortcmd( + "AUTH {0} {1}".format(self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")) + ) + + return res + + def _connect_imap(self) -> None: + self._conn.authenticate(self._mechanism, lambda x: self._auth_string) + + def _connect_smtp(self) -> None: + self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False) + + def _refresh_access_token(self) -> str: + service_obj = self._get_service_object() + access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None) + + # set the new access token in db + frappe.db.set_value("Email Account", self.email_account, "access_token", access_token) + frappe.db.commit() + return access_token + + def _get_service_object(self): + return { + "GMail": GoogleOAuth("mail"), + }[self.service] diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 81ef7927c1..074ec8e77d 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -18,7 +18,7 @@ from email_reply_parser import EmailReplyParser import frappe from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file import MaxFileSizeReachedError, get_random_filename -from frappe.email.utils import connect_google_oauth +from frappe.email.oauth import Oauth from frappe.utils import ( add_days, cint, @@ -100,14 +100,16 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - if self.settings.use_google_oauth and self.settings.google_refresh_token: - connect_google_oauth( + if self.settings.use_oauth and self.settings.refresh_token: + Oauth( self.imap, self.settings.email_account, self.settings.username, - self.settings.google_access_token, - self.settings.google_refresh_token, - ) + self.settings.access_token, + self.settings.refresh_token, + self.settings.service + ).connect() + else: self.imap.login(self.settings.username, self.settings.password) @@ -131,14 +133,16 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - if self.settings.use_google_oauth and self.settings.google_refresh_token: - connect_google_oauth( + if self.settings.use_oauth and self.settings.refresh_token: + Oauth( self.pop, self.settings.email_account, self.settings.username, - self.settings.google_access_token, - self.settings.google_refresh_token, - ) + self.settings.access_token, + self.settings.refresh_token, + self.settings.service + ).connect() + else: self.pop.user(self.settings.username) self.pop.pass_(self.settings.password) diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index cd31aa0d5c..df88ddb986 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -5,7 +5,7 @@ import smtplib import frappe from frappe import _ -from frappe.email.utils import connect_google_oauth +from frappe.email.oauth import Oauth from frappe.utils import cint, cstr @@ -53,9 +53,10 @@ class SMTPServer: port=None, use_tls=None, use_ssl=None, - use_google_oauth=0, - google_refresh_token=None, - google_access_token=None, + use_oauth=0, + refresh_token=None, + access_token=None, + service=None ): self.login = login self.email_account = email_account @@ -64,9 +65,10 @@ class SMTPServer: self._port = port self.use_tls = use_tls self.use_ssl = use_ssl - self.use_google_oauth = use_google_oauth - self.google_refresh_token = google_refresh_token - self.google_access_token = google_access_token + self.use_oauth = use_oauth + self.refresh_token = refresh_token + self.access_token = access_token + self.service = service self._session = None if not self.server: @@ -109,14 +111,15 @@ class SMTPServer: self.secure_session(_session) - if self.use_google_oauth and self.google_refresh_token: - connect_google_oauth( + if self.use_oauth and self.refresh_token: + Oauth( _session, self.email_account, self.login, - self.google_access_token, - self.google_refresh_token, - ) + self.access_token, + self.refresh_token, + self.service + ).connect() elif self.password: res = _session.login(str(self.login or ""), str(self.password or "")) diff --git a/frappe/email/utils.py b/frappe/email/utils.py index 97cfaaa1e7..7fc2e0ff89 100644 --- a/frappe/email/utils.py +++ b/frappe/email/utils.py @@ -1,13 +1,9 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import base64 + import imaplib import poplib -import smtplib -from typing import Union -from frappe import _, db, log_error, throw -from frappe.integrations.google_oauth import GoogleAuthenticationError, GoogleOAuth from frappe.utils import cint @@ -20,56 +16,3 @@ def get_port(doc): doc.incoming_port = poplib.POP3_SSL_PORT if doc.use_ssl else poplib.POP3_PORT return cint(doc.incoming_port) - - -def connect_google_oauth( - connection_obj: Union[imaplib.IMAP4, poplib.POP3, smtplib.SMTP], - email_account: str, - email: str, - google_access_token: str, - google_refresh_token: str, - retry: int = 0, -) -> None: - auth_string = "user=%s\1auth=Bearer %s\1\1" % (email, google_access_token) - mechanism = "XOAUTH2" - _func = lambda x: auth_string # noqa: E731 - - try: - if isinstance(connection_obj, poplib.POP3): - # poplib doesn't have AUTH command implementation - res = connection_obj._shortcmd( - "AUTH {0} {1}".format(mechanism, base64.b64encode(bytes(auth_string, "utf-8")).decode("utf-8")) - ) - - if not res.startswith(b"+OK"): - raise - - elif isinstance(connection_obj, imaplib.IMAP4): - connection_obj.authenticate(mechanism, _func) - - else: - # SMTP - connection_obj.auth(mechanism, _func, initial_response_ok=False) - - except Exception: - # maybe the access token expired - refreshing - access_token = refresh_google_access_token(email_account, google_refresh_token) - - if not access_token or retry > 0: - throw( - _("Google Authentication Failed. Please Check and Update the credentials."), - GoogleAuthenticationError, - _("Google Authentication Error"), - ) - - connect_google_oauth( - connection_obj, email_account, email, access_token, google_refresh_token, retry + 1 - ) - - -def refresh_google_access_token(email_account: str, google_refresh_token: str) -> str: - oauth_obj = GoogleOAuth("mail") - res = oauth_obj.refresh_access_token(google_refresh_token) - db.set_value("Email Account", email_account, "google_access_token", res.get("access_token")) - - return res.get("access_token") From 06c5a7226d5c74f627ad8c2a400d47b264685a78 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 11:24:45 +0530 Subject: [PATCH 099/201] chore: fix linter --- .../email/doctype/email_account/email_account.py | 15 +++++++++------ frappe/email/oauth.py | 13 +++++++++---- frappe/email/receive.py | 4 ++-- frappe/email/smtp.py | 9 ++------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index b2f8e1ed64..9d011d3671 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -163,8 +163,7 @@ class EmailAccount(Document): def after_insert(self): if self.use_oauth and not self.refresh_token: frappe.msgprint( - _("Please Enable OAuth by using `Authorize API access` button"), - indicator="orange" + _("Please Enable OAuth by using `Authorize API access` button"), indicator="orange" ) def there_must_be_only_one_default(self): @@ -288,9 +287,7 @@ class EmailAccount(Document): @property def _password(self): - raise_exception = not ( - self.use_oauth or self.no_smtp_authentication or frappe.flags.in_test - ) + raise_exception = not (self.use_oauth or self.no_smtp_authentication or frappe.flags.in_test) return self.get_password(raise_exception=raise_exception) @property @@ -943,7 +940,13 @@ def oauth_access(email_account: str, reauthorize: bool = False, service: str = N return authorize_google_access(email_account, reauthorize, refresh_token, doctype) -def authorize_google_access(email_account, reauthorize: bool = False, refresh_token: str = None, doctype: str = "Email Account", code: str = None): +def authorize_google_access( + email_account, + reauthorize: bool = False, + refresh_token: str = None, + doctype: str = "Email Account", + code: str = None, +): oauth_obj = GoogleOAuth("mail") if not (refresh_token or code) or reauthorize: diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index b4efac2e07..740aefc6a6 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -1,8 +1,8 @@ import base64 -from typing import Callable, Union from imaplib import IMAP4 from poplib import POP3 from smtplib import SMTP +from typing import Union import frappe from frappe.integrations.google_oauth import GoogleOAuth @@ -13,7 +13,8 @@ class OAuthenticationError(Exception): class Oauth: - def __init__(self, + def __init__( + self, conn: Union[IMAP4, POP3, SMTP], email_account: str, email: str, @@ -35,7 +36,9 @@ class Oauth: def validate_implementation(self) -> None: if self.service != "GMail": - raise NotImplementedError(f"Service {self.service} currently doesn't have oauth implementation.") + raise NotImplementedError( + f"Service {self.service} currently doesn't have oauth implementation." + ) @property def _auth_string(self) -> str: @@ -74,7 +77,9 @@ class Oauth: def _connect_pop(self) -> bytes: # poplib doesn't have AUTH command implementation res = self._conn._shortcmd( - "AUTH {0} {1}".format(self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")) + "AUTH {0} {1}".format( + self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8") + ) ) return res diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 074ec8e77d..8bfd177579 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -107,7 +107,7 @@ class EmailServer: self.settings.username, self.settings.access_token, self.settings.refresh_token, - self.settings.service + self.settings.service, ).connect() else: @@ -140,7 +140,7 @@ class EmailServer: self.settings.username, self.settings.access_token, self.settings.refresh_token, - self.settings.service + self.settings.service, ).connect() else: diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index df88ddb986..f5e2af7194 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -56,7 +56,7 @@ class SMTPServer: use_oauth=0, refresh_token=None, access_token=None, - service=None + service=None, ): self.login = login self.email_account = email_account @@ -113,12 +113,7 @@ class SMTPServer: if self.use_oauth and self.refresh_token: Oauth( - _session, - self.email_account, - self.login, - self.access_token, - self.refresh_token, - self.service + _session, self.email_account, self.login, self.access_token, self.refresh_token, self.service ).connect() elif self.password: From 064ffef8b9f202c1d3db257a26bac722cecc9dec Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 12:25:28 +0530 Subject: [PATCH 100/201] minor: throw exception if refresh_token is not present --- frappe/email/oauth.py | 11 +++++++++-- frappe/email/receive.py | 4 ++-- frappe/email/smtp.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 740aefc6a6..c432bcf45a 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -32,14 +32,21 @@ class Oauth: self._access_token = access_token self._refresh_token = refresh_token - self.validate_implementation() + self.validate() - def validate_implementation(self) -> None: + def validate(self) -> None: if self.service != "GMail": raise NotImplementedError( f"Service {self.service} currently doesn't have oauth implementation." ) + if not self._refresh_token: + frappe.throw( + frappe._("Please Authorize OAuth."), + OAuthenticationError, + frappe._("OAuth Error"), + ) + @property def _auth_string(self) -> str: return "user=%s\1auth=Bearer %s\1\1" % (self.email, self._access_token) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 8bfd177579..e26748dd07 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -100,7 +100,7 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - if self.settings.use_oauth and self.settings.refresh_token: + if self.settings.use_oauth: Oauth( self.imap, self.settings.email_account, @@ -133,7 +133,7 @@ class EmailServer: self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - if self.settings.use_oauth and self.settings.refresh_token: + if self.settings.use_oauth: Oauth( self.pop, self.settings.email_account, diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index f5e2af7194..687b8318b6 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -111,7 +111,7 @@ class SMTPServer: self.secure_session(_session) - if self.use_oauth and self.refresh_token: + if self.use_oauth: Oauth( _session, self.email_account, self.login, self.access_token, self.refresh_token, self.service ).connect() From 67730b7b268728c16f0df11cf92c9be3ef1befb1 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 12:43:24 +0530 Subject: [PATCH 101/201] chore: fix sider --- frappe/email/doctype/email_account/email_account.js | 2 +- frappe/email/doctype/email_account/email_account.py | 4 ++-- frappe/email/oauth.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 4d84f69395..97d93a63c2 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -148,7 +148,7 @@ frappe.ui.form.on("Email Account", { method: "frappe.email.doctype.email_account.email_account.oauth_access", args: { "email_account": frm.doc.name, - "service": frm.doc.service, + "service": frm.doc.service || "", "reauthorize": frm.doc.refresh_token ? 1 : 0 }, callback: function(r) { diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 9d011d3671..2cf8ec69fb 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -213,7 +213,7 @@ class EmailAccount(Document): "host": self.email_server, "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, - "service": getattr(self, "service", None), + "service": getattr(self, "service", ""), "use_imap": self.use_imap, "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), @@ -425,7 +425,7 @@ class EmailAccount(Document): "password": self._password, "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), - "service": getattr(self, "service", None), + "service": getattr(self, "service", ""), "use_oauth": self.use_oauth or 0, "refresh_token": getattr(self, "refresh_token", None), "access_token": getattr(self, "access_token", None), diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index c432bcf45a..3dfbfaf97d 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -69,7 +69,6 @@ class Oauth: except Exception: # maybe the access token expired - refreshing access_token = self._refresh_access_token() - print(self._auth_string) if not access_token or _retry > 0: frappe.throw( From de6f1326f79345e58641dfafdc701e010d32ac0a Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 19:04:04 +0530 Subject: [PATCH 102/201] minor: move oauth access functions from email_account --- .../doctype/email_account/email_account.js | 2 +- .../doctype/email_account/email_account.py | 36 ---------------- frappe/email/oauth.py | 42 ++++++++++++++++++- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 97d93a63c2..5cb7f1e497 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -145,7 +145,7 @@ frappe.ui.form.on("Email Account", { authorize_api_access: function(frm) { frappe.call({ - method: "frappe.email.doctype.email_account.email_account.oauth_access", + method: "frappe.email.oauth.oauth_access", args: { "email_account": frm.doc.name, "service": frm.doc.service || "", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 2cf8ec69fb..ea19c01f4a 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -8,7 +8,6 @@ import socket import time from datetime import datetime, timedelta from poplib import error_proto -from urllib.parse import quote import frappe from frappe import _, are_emails_muted, safe_encode @@ -16,13 +15,11 @@ from frappe.desk.form import assign_to from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from frappe.email.smtp import SMTPServer from frappe.email.utils import get_port -from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document from frappe.utils import ( cint, comma_or, cstr, - get_request_site_address, parse_addr, validate_email_address, ) @@ -929,36 +926,3 @@ def set_email_password(email_account, user, password): return False return True - - -@frappe.whitelist(methods=["POST"]) -def oauth_access(email_account: str, reauthorize: bool = False, service: str = None): - doctype = "Email Account" - refresh_token = frappe.db.get_value(doctype, email_account, "refresh_token") - - if service == "GMail": - return authorize_google_access(email_account, reauthorize, refresh_token, doctype) - - -def authorize_google_access( - email_account, - reauthorize: bool = False, - refresh_token: str = None, - doctype: str = "Email Account", - code: str = None, -): - oauth_obj = GoogleOAuth("mail") - - if not (refresh_token or code) or reauthorize: - return oauth_obj.get_authentication_url( - get_request_site_address(True), - state={ - "method": "frappe.email.doctype.email_account.email_account.authorize_google_access", - "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), - "email_account": email_account, - }, - ) - - res = oauth_obj.authorize(code, get_request_site_address(True)) - frappe.db.set_value(doctype, email_account, "refresh_token", res.get("refresh_token")) - frappe.db.set_value(doctype, email_account, "access_token", res.get("access_token")) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 3dfbfaf97d..5ebb231cf6 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -3,9 +3,11 @@ from imaplib import IMAP4 from poplib import POP3 from smtplib import SMTP from typing import Union +from urllib.parse import quote import frappe from frappe.integrations.google_oauth import GoogleOAuth +from frappe.utils import get_request_site_address class OAuthenticationError(Exception): @@ -107,5 +109,43 @@ class Oauth: def _get_service_object(self): return { - "GMail": GoogleOAuth("mail"), + "GMail": GoogleOAuth("mail", validate=False), }[self.service] + + +@frappe.whitelist(methods=["POST"]) +def oauth_access(email_account: str, reauthorize: bool = False, service: str = None): + doctype = "Email Account" + refresh_token = frappe.db.get_value(doctype, email_account, "refresh_token") + + # NOTE: setting this here, since we redirect to the service's auth page, + # we lose the use_oauth value in the emal account form + frappe.db.set_value(doctype, email_account, "use_oauth", 1) + + if service: + if service == "GMail": + return authorize_google_access(email_account, reauthorize, refresh_token, doctype) + + +def authorize_google_access( + email_account, + reauthorize: bool = False, + refresh_token: str = None, + doctype: str = "Email Account", + code: str = None, +): + oauth_obj = GoogleOAuth("mail") + + if not (refresh_token or code) or reauthorize: + return oauth_obj.get_authentication_url( + get_request_site_address(True), + state={ + "method": "frappe.email.oauth.authorize_google_access", + "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), + "email_account": email_account, + }, + ) + + res = oauth_obj.authorize(code, get_request_site_address(True)) + frappe.db.set_value(doctype, email_account, "refresh_token", res.get("refresh_token")) + frappe.db.set_value(doctype, email_account, "access_token", res.get("access_token")) From ed0a255353637069fe7d8f181a6263b1281fe9cd Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 31 May 2022 21:00:11 +0530 Subject: [PATCH 103/201] minor: fetch oauth fields from site config --- frappe/email/doctype/email_account/email_account.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index ea19c01f4a..18c13ed2c4 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -403,6 +403,9 @@ class EmailAccount(Document): "default": 0, }, "name": {"conf_names": ("email_sender_name",), "default": "Frappe"}, + "use_oauth": {"conf_names": ("use_oauth"), "default": 0}, + "access_token": {"conf_names": ("mail_access_token")}, + "refresh_token": {"conf_names": ("mail_refresh_token")}, "from_site_config": {"default": True}, } From e58afca3f6470d0cb505c61f96e0777cde91e7bd Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 1 Jun 2022 00:44:38 +0530 Subject: [PATCH 104/201] minor: simplify authorize_google_access --- .../email/doctype/email_account/email_account.js | 3 +-- .../email/doctype/email_account/email_account.py | 8 +------- frappe/email/oauth.py | 15 ++++----------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 5cb7f1e497..fda4966b90 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -148,8 +148,7 @@ frappe.ui.form.on("Email Account", { method: "frappe.email.oauth.oauth_access", args: { "email_account": frm.doc.name, - "service": frm.doc.service || "", - "reauthorize": frm.doc.refresh_token ? 1 : 0 + "service": frm.doc.service || "" }, callback: function(r) { if (!r.exc) { diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 18c13ed2c4..e17c021e52 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -16,13 +16,7 @@ from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError from frappe.email.smtp import SMTPServer from frappe.email.utils import get_port from frappe.model.document import Document -from frappe.utils import ( - cint, - comma_or, - cstr, - parse_addr, - validate_email_address, -) +from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address from frappe.utils.background_jobs import enqueue, get_jobs from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 5ebb231cf6..ccb3cac3f0 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -114,9 +114,8 @@ class Oauth: @frappe.whitelist(methods=["POST"]) -def oauth_access(email_account: str, reauthorize: bool = False, service: str = None): +def oauth_access(email_account: str, service: str = None): doctype = "Email Account" - refresh_token = frappe.db.get_value(doctype, email_account, "refresh_token") # NOTE: setting this here, since we redirect to the service's auth page, # we lose the use_oauth value in the emal account form @@ -124,19 +123,13 @@ def oauth_access(email_account: str, reauthorize: bool = False, service: str = N if service: if service == "GMail": - return authorize_google_access(email_account, reauthorize, refresh_token, doctype) + return authorize_google_access(email_account, doctype) -def authorize_google_access( - email_account, - reauthorize: bool = False, - refresh_token: str = None, - doctype: str = "Email Account", - code: str = None, -): +def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): oauth_obj = GoogleOAuth("mail") - if not (refresh_token or code) or reauthorize: + if not code: return oauth_obj.get_authentication_url( get_request_site_address(True), state={ From 221423c71808636d60e4b3da228785f9d6fea368 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 1 Jun 2022 14:49:53 +0530 Subject: [PATCH 105/201] chore: added docstrings --- frappe/email/oauth.py | 23 ++++++++++++++++++----- frappe/integrations/google_oauth.py | 20 +++++++++++++++++++- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index ccb3cac3f0..44e69d8d85 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -34,9 +34,9 @@ class Oauth: self._access_token = access_token self._refresh_token = refresh_token - self.validate() + self._validate() - def validate(self) -> None: + def _validate(self) -> None: if self.service != "GMail": raise NotImplementedError( f"Service {self.service} currently doesn't have oauth implementation." @@ -54,6 +54,7 @@ class Oauth: return "user=%s\1auth=Bearer %s\1\1" % (self.email, self._access_token) def connect(self, _retry: int = 0) -> None: + """Connection method with retry on exception for Oauth""" try: if isinstance(self._conn, POP3): res = self._connect_pop() @@ -99,6 +100,7 @@ class Oauth: self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False) def _refresh_access_token(self) -> str: + """Refreshes access token via calling `refresh_access_token` method of oauth service object""" service_obj = self._get_service_object() access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None) @@ -108,6 +110,8 @@ class Oauth: return access_token def _get_service_object(self): + """Get Oauth service object""" + return { "GMail": GoogleOAuth("mail", validate=False), }[self.service] @@ -115,18 +119,27 @@ class Oauth: @frappe.whitelist(methods=["POST"]) def oauth_access(email_account: str, service: str = None): + """Used as a default endpoint/caller for all oauth services. + Returns authorization url for redirection""" + + if not service: + frappe.throw(frappe._("No Service is selected. Please select one and try again!")) + doctype = "Email Account" # NOTE: setting this here, since we redirect to the service's auth page, # we lose the use_oauth value in the emal account form frappe.db.set_value(doctype, email_account, "use_oauth", 1) - if service: - if service == "GMail": - return authorize_google_access(email_account, doctype) + if service == "GMail": + return authorize_google_access(email_account, doctype) def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): + """Facilitates google oauth for email. + This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url + and second time for setting the refresh and access token in db when google redirects back with oauth code.""" + oauth_obj = GoogleOAuth("mail") if not code: diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index da67f2ccf5..c720a22d28 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -48,6 +48,12 @@ class GoogleOAuth: frappe.throw(frappe._("Please update Google Settings before continuing.")) def authorize(self, oauth_code: str, site_address: str) -> Dict[str, Union[str, int]]: + """Returns a dict with access and refresh token. + + :param oauth_code: code got back from google upon successful auhtorization + :param site_address: side address from which the request is being made + """ + data = { "code": oauth_code, "client_id": self.google_settings.client_id, @@ -66,6 +72,8 @@ class GoogleOAuth: ) def refresh_access_token(self, refresh_token: str) -> Dict[str, Union[str, int]]: + """Refreshes google access token using refresh token""" + data = { "client_id": self.google_settings.client_id, "client_secret": self.google_settings.get_password( @@ -86,7 +94,11 @@ class GoogleOAuth: def get_authentication_url( self, site_address: str, state: Dict[str, str] = None ) -> Dict[str, str]: - """Return authentication url with the client id and redirect uri.""" + """Returns google authentication url. + + :param site_address: side address from which the request is being made (for redirect back to site) + :param state: [optional] dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc) + """ state = json.dumps(state) callback_url = site_address + CALLBACK_METHOD @@ -100,6 +112,8 @@ class GoogleOAuth: } def get_google_service_object(self, access_token: str, refresh_token: str): + """Returns google service object""" + credentials_dict = { "token": access_token, "refresh_token": refresh_token, @@ -151,6 +165,10 @@ def is_valid_access_token(access_token: str) -> bool: @frappe.whitelist(methods=["GET"]) def callback(state: str, code: str = None, error: str = None) -> None: + """Common callback for google integrations. + Invokes functions using `frappe.get_attr` and also adds required (keyworded) arguments + along with committing and redirecting us back to frappe site.""" + state = json.loads(state) redirect = state.pop("redirect", "/app") From ab9a577474cb26f22559392d83c2c7070c0b45bf Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 1 Jun 2022 17:57:51 +0530 Subject: [PATCH 106/201] minor: better oauth flow --- .../doctype/email_account/email_account.js | 43 +++++++++++++------ .../doctype/email_account/email_account.json | 2 +- .../doctype/email_account/email_account.py | 21 +++++---- frappe/integrations/google_oauth.py | 2 +- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index fda4966b90..d553b58e28 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -67,6 +67,21 @@ frappe.email_defaults_pop = { }; +function oauth_access(frm) { + return frappe.call({ + method: "frappe.email.oauth.oauth_access", + args: { + "email_account": frm.doc.name, + "service": frm.doc.service || "" + }, + callback: function(r) { + if (!r.exc) { + window.open(r.message.url); + } + } + }); +} + frappe.ui.form.on("Email Account", { service: function(frm) { $.each(frappe.email_defaults[frm.doc.service], function(key, value) { @@ -126,6 +141,7 @@ frappe.ui.form.on("Email Account", { frm.events.enable_incoming(frm); frm.events.notify_if_unreplied(frm); frm.events.show_gmail_message_for_less_secure_apps(frm); + frm.events.show_oauth_authorization_message(frm); if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; @@ -133,6 +149,12 @@ frappe.ui.form.on("Email Account", { } }, + after_save(frm) { + if (frm.doc.use_oauth && !frm.doc.refresh_token) { + oauth_access(frm); + } + }, + show_gmail_message_for_less_secure_apps: function(frm) { frm.dashboard.clear_headline(); let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use OAuth."); @@ -143,19 +165,16 @@ frappe.ui.form.on("Email Account", { } }, + show_oauth_authorization_message(frm) { + let msg = __("Oauth Enabled but not Authorized. Please use `Authorize API Access` Button to do the same."); + if (frm.doc.use_oauth && !frm.doc.refresh_token) { + frm.dashboard.clear_headline(); + frm.dashboard.set_headline_alert(msg, "yellow"); + } + }, + authorize_api_access: function(frm) { - frappe.call({ - method: "frappe.email.oauth.oauth_access", - args: { - "email_account": frm.doc.name, - "service": frm.doc.service || "" - }, - callback: function(r) { - if (!r.exc) { - window.open(r.message.url); - } - } - }); + oauth_access(frm); }, email_id:function(frm) { diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 72be4f2e42..f76bacd044 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -613,7 +613,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-05-30 15:45:05.282867", + "modified": "2022-06-01 17:51:37.878446", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index e17c021e52..71cc16bb21 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -83,8 +83,16 @@ class EmailAccount(Document): if getattr(self, "service", "") != "GMail" and self.use_oauth: self.use_oauth = 0 + if not self.use_oauth and self.refresh_token: + # clear access & refresh token + self.refresh_token = self.access_token = None + if not frappe.local.flags.in_install and (self.use_oauth or not self.awaiting_password): - if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): + if ( + (self.use_oauth and self.refresh_token) + or self.password + or self.smtp_server in ("127.0.0.1", "localhost") + ): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -93,10 +101,7 @@ class EmailAccount(Document): self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - if self.use_oauth: - if not self.is_new(): - frappe.throw(_("Please Enable OAuth by using `Authorize API access` button")) - else: + if not self.use_oauth: frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: @@ -151,12 +156,6 @@ class EmailAccount(Document): enable_outgoing=self.enable_outgoing, ) - def after_insert(self): - if self.use_oauth and not self.refresh_token: - frappe.msgprint( - _("Please Enable OAuth by using `Authorize API access` button"), indicator="orange" - ) - def there_must_be_only_one_default(self): """If current Email Account is default, un-default all other accounts.""" for field in ("default_incoming", "default_outgoing"): diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index c720a22d28..c920fc31fa 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -145,7 +145,7 @@ def handle_response( ) if raise_err: - frappe.throw(frappe._(error_title), frappe._(error_message)) + frappe.throw(frappe._(error_title), GoogleAuthenticationError, frappe._(error_message)) return {} From 834410a2a56b44c280cb5c6f0a375c8dcc238a8b Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 1 Jun 2022 18:01:27 +0530 Subject: [PATCH 107/201] fix: dont update modified timestamp on generating and refreshing tokens --- frappe/email/oauth.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 44e69d8d85..2e328dc2b6 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -105,7 +105,9 @@ class Oauth: access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None) # set the new access token in db - frappe.db.set_value("Email Account", self.email_account, "access_token", access_token) + frappe.db.set_value( + "Email Account", self.email_account, "access_token", access_token, update_modified=True + ) frappe.db.commit() return access_token @@ -129,7 +131,7 @@ def oauth_access(email_account: str, service: str = None): # NOTE: setting this here, since we redirect to the service's auth page, # we lose the use_oauth value in the emal account form - frappe.db.set_value(doctype, email_account, "use_oauth", 1) + frappe.db.set_value(doctype, email_account, "use_oauth", 1, update_modified=False) if service == "GMail": return authorize_google_access(email_account, doctype) @@ -153,5 +155,9 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: ) res = oauth_obj.authorize(code, get_request_site_address(True)) - frappe.db.set_value(doctype, email_account, "refresh_token", res.get("refresh_token")) - frappe.db.set_value(doctype, email_account, "access_token", res.get("access_token")) + frappe.db.set_value( + doctype, email_account, "refresh_token", res.get("refresh_token"), update_modified=False + ) + frappe.db.set_value( + doctype, email_account, "access_token", res.get("access_token"), update_modified=False + ) From 4d5dec4048d5b4d83a5fd3c5c9ece2ab2bd39df2 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 1 Jun 2022 18:39:07 +0530 Subject: [PATCH 108/201] fix: open authorization url on the current browsing context --- frappe/email/doctype/email_account/email_account.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index d553b58e28..5d65a8efe6 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -76,7 +76,7 @@ function oauth_access(frm) { }, callback: function(r) { if (!r.exc) { - window.open(r.message.url); + window.open(r.message.url, "_self"); } } }); From 5521abd40cf8f70d6e0826a6c86d2283f665e799 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 2 Jun 2022 13:06:48 +0530 Subject: [PATCH 109/201] fix: use safe decode for uid --- frappe/email/doctype/email_account/email_account.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 71cc16bb21..4eb230c372 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -502,11 +502,10 @@ class EmailAccount(Document): def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): uid = messages["uid_list"][index] if messages.get("uid_list") else None - uid = uid.decode() if isinstance(uid, bytes) else uid seen_status = messages.get("seen_status", {}).get(uid) if self.email_sync_option != "UNSEEN" or seen_status != "SEEN": # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' - mails.append(InboundMail(message, self, uid, seen_status, append_to)) + mails.append(InboundMail(message, self, frappe.safe_decode(uid), seen_status, append_to)) if not self.enable_incoming: return [] From 6d3dfca2148dbdde73cc00c413063474df2e8410 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 5 Jun 2022 22:34:02 +0530 Subject: [PATCH 110/201] fix: consider oauth usage as well for asking/updating user email password --- frappe/core/doctype/user/user.py | 4 +-- .../core/doctype/user_email/user_email.json | 15 +++++++++-- .../doctype/email_account/email_account.py | 26 +++++++++++++------ frappe/public/js/frappe/desk.js | 1 - 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index eb3c11a4db..12a48afe7e 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -766,7 +766,7 @@ def get_email_awaiting(user): return frappe.get_all( "User Email", fields=["email_account", "email_id"], - filters={"awaiting_password": 1, "parent": user}, + filters={"awaiting_password": 1, "parent": user, "used_oauth": 0}, ) @@ -775,7 +775,7 @@ def ask_pass_update(): from frappe.utils import set_default password_list = frappe.get_all( - "User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True + "User Email", filters={"awaiting_password": 1, "used_oauth": 0}, pluck="parent", distinct=True ) set_default("email_user_password", ",".join(password_list)) diff --git a/frappe/core/doctype/user_email/user_email.json b/frappe/core/doctype/user_email/user_email.json index b106ed4a19..43dc3ace8d 100644 --- a/frappe/core/doctype/user_email/user_email.json +++ b/frappe/core/doctype/user_email/user_email.json @@ -9,6 +9,7 @@ "email_id", "column_break_3", "awaiting_password", + "used_oauth", "enable_outgoing" ], "fields": [ @@ -48,16 +49,26 @@ "fieldtype": "Check", "label": "Enable Outgoing", "read_only": 1 + }, + { + "default": "0", + "fetch_from": "email_account.use_oauth", + "fieldname": "used_oauth", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Used OAuth", + "read_only": 1 } ], "istable": 1, "links": [], - "modified": "2020-04-06 19:19:12.130246", + "modified": "2022-06-03 14:25:46.944733", "modified_by": "Administrator", "module": "Core", "name": "User Email", "owner": "Administrator", "permissions": [], "sort_field": "modified", - "sort_order": "DESC" + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 4eb230c372..278aeb698b 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -83,13 +83,17 @@ class EmailAccount(Document): if getattr(self, "service", "") != "GMail" and self.use_oauth: self.use_oauth = 0 - if not self.use_oauth and self.refresh_token: + if self.use_oauth: + # no need for awaiting password for oauth + self.awaiting_password = 0 + + elif self.refresh_token: # clear access & refresh token self.refresh_token = self.access_token = None if not frappe.local.flags.in_install and (self.use_oauth or not self.awaiting_password): if ( - (self.use_oauth and self.refresh_token) + self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost") ): @@ -154,6 +158,7 @@ class EmailAccount(Document): awaiting_password=self.awaiting_password, email_id=self.email_id, enable_outgoing=self.enable_outgoing, + used_oauth=self.use_oauth, ) def there_must_be_only_one_default(self): @@ -839,7 +844,7 @@ def get_max_email_uid(email_account): return max_uid -def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing): +def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing, used_oauth): """setup email inbox for user""" from frappe.core.doctype.user.user import ask_pass_update @@ -850,6 +855,7 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou row.email_id = email_id row.email_account = email_account row.awaiting_password = awaiting_password or 0 + row.used_oauth = used_oauth or 0 row.enable_outgoing = enable_outgoing or 0 user.save(ignore_permissions=True) @@ -881,8 +887,12 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou if update_user_email_settings: UserEmail = frappe.qb.DocType("User Email") - frappe.qb.update(UserEmail).set(UserEmail.awaiting_password, (awaiting_password or 0)).set( - UserEmail.enable_outgoing, enable_outgoing + frappe.qb.update(UserEmail).set( + UserEmail.awaiting_password, (awaiting_password or 0) + ).set( + UserEmail.enable_outgoing, (enable_outgoing or 0) + ).set( + UserEmail.used_oauth, (used_oauth or 0) ).where(UserEmail.email_account == email_account).run() else: @@ -908,10 +918,10 @@ def remove_user_email_inbox(email_account): doc.save(ignore_permissions=True) -@frappe.whitelist(allow_guest=False) -def set_email_password(email_account, user, password): +@frappe.whitelist() +def set_email_password(email_account, password): account = frappe.get_doc("Email Account", email_account) - if account.awaiting_password: + if account.awaiting_password and not account.use_oauth: account.awaiting_password = 0 account.password = password try: diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index a8cbe020f3..f1a5d00dfd 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -251,7 +251,6 @@ frappe.Application = class Application { method: 'frappe.email.doctype.email_account.email_account.set_email_password', args: { "email_account": email_account[i]["email_account"], - "user": user, "password": d.get_value("password") }, callback: function(passed) { From 6848c93770e5d15fada4e5c6a4eaca6890d7ad19 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 13 Jun 2022 13:25:20 +0530 Subject: [PATCH 111/201] chore: remove GET method whitelisting from google integrations --- frappe/integrations/doctype/google_contacts/google_contacts.py | 2 +- frappe/integrations/doctype/google_drive/google_drive.py | 2 +- frappe/website/doctype/website_settings/google_indexing.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 51ebcdd730..8a4fc7d9bb 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -29,7 +29,7 @@ class GoogleContacts(Document): return r.get("access_token") -@frappe.whitelist(methods=["POST", "GET"]) +@frappe.whitelist(methods=["POST"]) def authorize_access(g_contact, reauthorize=False, code=None): """ If no Authorization code get it from Google and then request for Refresh Token. diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index b9416e6669..dd3d87ea71 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -40,7 +40,7 @@ class GoogleDrive(Document): return r.get("access_token") -@frappe.whitelist(methods=["POST", "GET"]) +@frappe.whitelist(methods=["POST"]) def authorize_access(reauthorize=False, code=None): """ If no Authorization code get it from Google and then request for Refresh Token. diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index b89918dff2..25779967e4 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -12,7 +12,7 @@ from frappe.integrations.google_oauth import GoogleOAuth from frappe.utils import get_request_site_address -@frappe.whitelist(methods=["POST", "GET"]) +@frappe.whitelist(methods=["POST"]) def authorize_access(reauthorize=False, code=None): """If no Authorization code get it from Google and then request for Refresh Token.""" From 484758d6e0b38160be84702ac7b8ac1e761cf2b3 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 13 Jun 2022 13:31:57 +0530 Subject: [PATCH 112/201] chore: remove additional/unnecessary set_value calls --- frappe/email/oauth.py | 8 ++++---- .../doctype/google_contacts/google_contacts.py | 8 +++++--- frappe/integrations/doctype/google_drive/google_drive.py | 7 +++++-- .../website/doctype/website_settings/google_indexing.py | 5 +++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 2e328dc2b6..5793d84a13 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -156,8 +156,8 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: res = oauth_obj.authorize(code, get_request_site_address(True)) frappe.db.set_value( - doctype, email_account, "refresh_token", res.get("refresh_token"), update_modified=False - ) - frappe.db.set_value( - doctype, email_account, "access_token", res.get("access_token"), update_modified=False + doctype, + email_account, + {"refresh_token": res.get("refresh_token"), "access_token": res.get("access_token")}, + update_modified=False, ) diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 8a4fc7d9bb..1b309a9e16 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -51,10 +51,12 @@ def authorize_access(g_contact, reauthorize=False, code=None): }, ) - frappe.db.set_value("Google Contacts", g_contact, "authorization_code", oauth_code) r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) - if r: - frappe.db.set_value("Google Contacts", g_contact, "refresh_token", r.get("refresh_token")) + frappe.db.set_value( + "Google Contacts", + g_contact, + {"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")}, + ) def get_google_contacts_object(g_contact): diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index dd3d87ea71..22b3759266 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -63,9 +63,12 @@ def authorize_access(reauthorize=False, code=None): }, ) - frappe.db.set_value("Google Drive", None, "authorization_code", oauth_code) r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) - frappe.db.set_value("Google Drive", "Google Drive", "refresh_token", r.get("refresh_token")) + frappe.db.set_value( + "Google Drive", + "Google Drive", + {"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")}, + ) def get_google_drive_object(): diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 25779967e4..3b36bb5360 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -33,10 +33,11 @@ def authorize_access(reauthorize=False, code=None): }, ) - frappe.db.set_value("Website Settings", None, "indexing_authorization_code", oauth_code) res = oauth_obj.authorize(oauth_code, get_request_site_address(True)) frappe.db.set_value( - "Website Settings", "Website Settings", "indexing_refresh_token", res.get("refresh_token") + "Website Settings", + "Website Settings", + {"indexing_authorization_code": oauth_code, "indexing_refresh_token": res.get("refresh_token")}, ) From 84ad7b74b950c69f419724eec54eeed78f2c422d Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 30 Jun 2022 16:50:30 +0530 Subject: [PATCH 113/201] fix: don't show authorize api access button if the form is unsaved --- frappe/email/doctype/email_account/email_account.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index f76bacd044..e8540c90d9 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -590,7 +590,7 @@ "label": "Use OAuth" }, { - "depends_on": "eval: doc.service === \"GMail\" && doc.use_oauth && !doc.__islocal", + "depends_on": "eval: doc.service === \"GMail\" && doc.use_oauth && !doc.__islocal && !doc.__unsaved", "fieldname": "authorize_api_access", "fieldtype": "Button", "label": "Authorize API Access" From 431afaeee43d8dc8f50ab49bfb22610e0a80b533 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 30 Jun 2022 17:00:06 +0530 Subject: [PATCH 114/201] fix: remove commit * chore: raise not implemented error for services other than gmail * chore: use fstring for _auth_string property --- frappe/email/oauth.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 5793d84a13..d5e74afc9f 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -37,11 +37,6 @@ class Oauth: self._validate() def _validate(self) -> None: - if self.service != "GMail": - raise NotImplementedError( - f"Service {self.service} currently doesn't have oauth implementation." - ) - if not self._refresh_token: frappe.throw( frappe._("Please Authorize OAuth."), @@ -51,7 +46,7 @@ class Oauth: @property def _auth_string(self) -> str: - return "user=%s\1auth=Bearer %s\1\1" % (self.email, self._access_token) + return f"user={self.email}\1auth=Bearer {self._access_token}\1\1" def connect(self, _retry: int = 0) -> None: """Connection method with retry on exception for Oauth""" @@ -106,9 +101,8 @@ class Oauth: # set the new access token in db frappe.db.set_value( - "Email Account", self.email_account, "access_token", access_token, update_modified=True + "Email Account", self.email_account, "access_token", access_token, update_modified=False ) - frappe.db.commit() return access_token def _get_service_object(self): @@ -120,7 +114,7 @@ class Oauth: @frappe.whitelist(methods=["POST"]) -def oauth_access(email_account: str, service: str = None): +def oauth_access(email_account: str, service: str): """Used as a default endpoint/caller for all oauth services. Returns authorization url for redirection""" @@ -136,6 +130,8 @@ def oauth_access(email_account: str, service: str = None): if service == "GMail": return authorize_google_access(email_account, doctype) + raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.") + def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None): """Facilitates google oauth for email. From 8b38fcb4383831b90725695045515b13d2a9d0f8 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 30 Jun 2022 17:12:25 +0530 Subject: [PATCH 115/201] chore: move getting site address to GoogleOAuth --- frappe/email/oauth.py | 6 ++---- .../doctype/google_contacts/google_contacts.py | 6 ++---- .../integrations/doctype/google_drive/google_drive.py | 7 +++---- frappe/integrations/google_oauth.py | 11 +++++------ .../doctype/website_settings/google_indexing.py | 6 ++---- 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index d5e74afc9f..cebb370b1f 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -7,7 +7,6 @@ from urllib.parse import quote import frappe from frappe.integrations.google_oauth import GoogleOAuth -from frappe.utils import get_request_site_address class OAuthenticationError(Exception): @@ -142,15 +141,14 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: if not code: return oauth_obj.get_authentication_url( - get_request_site_address(True), - state={ + { "method": "frappe.email.oauth.authorize_google_access", "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), "email_account": email_account, }, ) - res = oauth_obj.authorize(code, get_request_site_address(True)) + res = oauth_obj.authorize(code) frappe.db.set_value( doctype, email_account, diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 1b309a9e16..eb2a8af647 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -8,7 +8,6 @@ import frappe from frappe import _ from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document -from frappe.utils import get_request_site_address class GoogleContacts(Document): @@ -43,15 +42,14 @@ def authorize_access(g_contact, reauthorize=False, code=None): if not oauth_code or reauthorize: return oauth_obj.get_authentication_url( - get_request_site_address(True), - state={ + { "method": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", "g_contact": g_contact, "redirect": "/app/Form/Google%20Contacts/{}".format(g_contact), }, ) - r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + r = oauth_obj.authorize(oauth_code) frappe.db.set_value( "Google Contacts", g_contact, diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index 22b3759266..f4a6f1f46e 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -16,7 +16,7 @@ from frappe.integrations.offsite_backup_utils import ( validate_file_size, ) from frappe.model.document import Document -from frappe.utils import get_backups_path, get_bench_path, get_request_site_address +from frappe.utils import get_backups_path, get_bench_path from frappe.utils.background_jobs import enqueue from frappe.utils.backups import new_backup @@ -56,14 +56,13 @@ def authorize_access(reauthorize=False, code=None): if reauthorize: frappe.db.set_value("Google Drive", None, "backup_folder_id", "") return oauth_obj.get_authentication_url( - get_request_site_address(True), - state={ + { "method": "frappe.integrations.doctype.google_drive.google_drive.authorize_access", "redirect": "/app/Form/{0}".format(quote("Google Drive")), }, ) - r = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + r = oauth_obj.authorize(oauth_code) frappe.db.set_value( "Google Drive", "Google Drive", diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index c920fc31fa..2bb1aaca1f 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -6,6 +6,7 @@ from googleapiclient.discovery import build from requests import get, post import frappe +from frappe.utils import get_request_site_address CALLBACK_METHOD = "/api/method/frappe.integrations.google_oauth.callback" _SCOPES = { @@ -47,7 +48,7 @@ class GoogleOAuth: if not (self.google_settings.client_id and self.google_settings.client_secret): frappe.throw(frappe._("Please update Google Settings before continuing.")) - def authorize(self, oauth_code: str, site_address: str) -> Dict[str, Union[str, int]]: + def authorize(self, oauth_code: str) -> Dict[str, Union[str, int]]: """Returns a dict with access and refresh token. :param oauth_code: code got back from google upon successful auhtorization @@ -62,7 +63,7 @@ class GoogleOAuth: ), "grant_type": "authorization_code", "scope": self.scopes, - "redirect_uri": site_address + CALLBACK_METHOD, + "redirect_uri": get_request_site_address(True) + CALLBACK_METHOD, } return handle_response( @@ -91,9 +92,7 @@ class GoogleOAuth: raise_err=True, ) - def get_authentication_url( - self, site_address: str, state: Dict[str, str] = None - ) -> Dict[str, str]: + def get_authentication_url(self, state: Dict[str, str]) -> Dict[str, str]: """Returns google authentication url. :param site_address: side address from which the request is being made (for redirect back to site) @@ -101,7 +100,7 @@ class GoogleOAuth: """ state = json.dumps(state) - callback_url = site_address + CALLBACK_METHOD + callback_url = get_request_site_address(True) + CALLBACK_METHOD return { "url": "https://accounts.google.com/o/oauth2/v2/auth?" diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 3b36bb5360..3577f688f5 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -9,7 +9,6 @@ from googleapiclient.errors import HttpError import frappe from frappe import _ from frappe.integrations.google_oauth import GoogleOAuth -from frappe.utils import get_request_site_address @frappe.whitelist(methods=["POST"]) @@ -26,14 +25,13 @@ def authorize_access(reauthorize=False, code=None): if not oauth_code or reauthorize: return oauth_obj.get_authentication_url( - get_request_site_address(True), - state={ + { "method": "frappe.website.doctype.website_settings.google_indexing.authorize_access", "redirect": "/app/Form/{0}".format(quote("Website Settings")), }, ) - res = oauth_obj.authorize(oauth_code, get_request_site_address(True)) + res = oauth_obj.authorize(oauth_code) frappe.db.set_value( "Website Settings", "Website Settings", From 5bf26819a8d44dd3619f35aa8011104ee87f616d Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 1 Jul 2022 11:00:06 +0530 Subject: [PATCH 116/201] fix: better/reduced exception handling for email oauth Since the places where connection methods are called already have a lot of exception handling, we can just raise and let them handle all the probable cases. --- frappe/email/oauth.py | 11 ++++++----- frappe/email/smtp.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index cebb370b1f..9d456daf81 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -63,16 +63,17 @@ class Oauth: # SMTP self._connect_smtp() - except Exception: + except Exception as e: # maybe the access token expired - refreshing access_token = self._refresh_access_token() if not access_token or _retry > 0: - frappe.throw( - frappe._("Authentication Failed. Please Check and Update the credentials."), - OAuthenticationError, - frappe._("OAuth Error"), + frappe.log_error( + "OAuth Error - Authentication Failed", str(e), "Email Account", self.email_account ) + # raising a bare exception here as we have a lot of exception handling present + # where the connect method is called from - hence just logging and raising. + raise self._access_token = access_token self.connect(_retry + 1) diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 687b8318b6..10eb2f7681 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -147,7 +147,7 @@ class SMTPServer: @classmethod def throw_invalid_credentials_exception(cls): frappe.throw( - _("Incorrect email or password. Please check your login credentials."), + _("Please check your email login credentials."), title=_("Invalid Credentials"), exc=InvalidEmailCredentials, ) From 2907571098e81875916542e902a559c09bf56bd3 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 11 Jul 2022 16:27:52 +0530 Subject: [PATCH 117/201] minor: encrypt email oauth refresh and access token --- .../doctype/email_account/email_account.json | 4 +-- .../doctype/email_account/email_account.py | 36 ++++++++++--------- frappe/email/oauth.py | 12 ++++--- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index e8540c90d9..bd4023d62b 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -597,7 +597,7 @@ }, { "fieldname": "refresh_token", - "fieldtype": "Data", + "fieldtype": "Small Text", "hidden": 1, "label": "Refresh Token", "read_only": 1 @@ -613,7 +613,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-06-01 17:51:37.878446", + "modified": "2022-07-11 15:11:15.196935", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 278aeb698b..d3273ab89d 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -20,6 +20,7 @@ from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_addres from frappe.utils.background_jobs import enqueue, get_jobs from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template +from frappe.utils.password import decrypt, encrypt from frappe.utils.user import get_system_managers @@ -92,11 +93,7 @@ class EmailAccount(Document): self.refresh_token = self.access_token = None if not frappe.local.flags.in_install and (self.use_oauth or not self.awaiting_password): - if ( - self.refresh_token - or self.password - or self.smtp_server in ("127.0.0.1", "localhost") - ): + if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -214,8 +211,8 @@ class EmailAccount(Document): "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, "use_oauth": self.use_oauth or 0, - "refresh_token": getattr(self, "refresh_token", None), - "access_token": getattr(self, "access_token", None), + "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, + "access_token": decrypt(self.access_token) if self.access_token else None, } ) @@ -411,7 +408,12 @@ class EmailAccount(Document): for doc_field_name, d in field_to_conf_name_map.items(): conf_names, default = d.get("conf_names") or [], d.get("default") value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)] - account_details[doc_field_name] = (value and value[0]) or default + + if doc_field_name in ("refresh_token", "access_token"): + account_details[doc_field_name] = value and encrypt(value[0]) + else: + account_details[doc_field_name] = (value and value[0]) or default + return account_details def sendmail_config(self): @@ -425,8 +427,8 @@ class EmailAccount(Document): "use_tls": cint(self.use_tls), "service": getattr(self, "service", ""), "use_oauth": self.use_oauth or 0, - "refresh_token": getattr(self, "refresh_token", None), - "access_token": getattr(self, "access_token", None), + "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, + "access_token": decrypt(self.access_token) if self.access_token else None, } def get_smtp_server(self): @@ -844,7 +846,9 @@ def get_max_email_uid(email_account): return max_uid -def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing, used_oauth): +def setup_user_email_inbox( + email_account, awaiting_password, email_id, enable_outgoing, used_oauth +): """setup email inbox for user""" from frappe.core.doctype.user.user import ask_pass_update @@ -887,13 +891,11 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou if update_user_email_settings: UserEmail = frappe.qb.DocType("User Email") - frappe.qb.update(UserEmail).set( - UserEmail.awaiting_password, (awaiting_password or 0) - ).set( + frappe.qb.update(UserEmail).set(UserEmail.awaiting_password, (awaiting_password or 0)).set( UserEmail.enable_outgoing, (enable_outgoing or 0) - ).set( - UserEmail.used_oauth, (used_oauth or 0) - ).where(UserEmail.email_account == email_account).run() + ).set(UserEmail.used_oauth, (used_oauth or 0)).where( + UserEmail.email_account == email_account + ).run() else: users = " and ".join([frappe.bold(user.get("name")) for user in user_names]) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 9d456daf81..cb9dcbbc85 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -7,6 +7,7 @@ from urllib.parse import quote import frappe from frappe.integrations.google_oauth import GoogleOAuth +from frappe.utils.password import encrypt class OAuthenticationError(Exception): @@ -16,7 +17,7 @@ class OAuthenticationError(Exception): class Oauth: def __init__( self, - conn: Union[IMAP4, POP3, SMTP], + conn: IMAP4 | POP3 | SMTP, email_account: str, email: str, access_token: str, @@ -81,7 +82,7 @@ class Oauth: def _connect_pop(self) -> bytes: # poplib doesn't have AUTH command implementation res = self._conn._shortcmd( - "AUTH {0} {1}".format( + "AUTH {} {}".format( self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8") ) ) @@ -144,7 +145,7 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: return oauth_obj.get_authentication_url( { "method": "frappe.email.oauth.authorize_google_access", - "redirect": "/app/Form/{0}/{1}".format(quote(doctype), quote(email_account)), + "redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}", "email_account": email_account, }, ) @@ -153,6 +154,9 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: frappe.db.set_value( doctype, email_account, - {"refresh_token": res.get("refresh_token"), "access_token": res.get("access_token")}, + { + "refresh_token": encrypt(res.get("refresh_token")), + "access_token": encrypt(res.get("access_token")), + }, update_modified=False, ) From 2b7bd4eef08591e098068f4d08f6de8f6e8a00de Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 11 Jul 2022 17:47:45 +0530 Subject: [PATCH 118/201] minor(ux): select field for alternating between basic and oauth authentication --- .../doctype/email_account/email_account.js | 17 +++++++++-- .../doctype/email_account/email_account.json | 25 +++++++++------- .../doctype/email_account/email_account.py | 29 +++++++++++-------- frappe/email/oauth.py | 5 ---- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 5d65a8efe6..9fd9c1a0bf 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -93,6 +93,7 @@ frappe.ui.form.on("Email Account", { }); } frm.events.show_gmail_message_for_less_secure_apps(frm); + frm.events.toggle_auth_method(frm); }, use_imap: function(frm) { @@ -134,6 +135,7 @@ frappe.ui.form.on("Email Account", { frm.add_child("imap_folder", {"folder_name": "INBOX"}); frm.refresh_field("imap_folder"); } + frm.toggle_display(['auth_method'], frm.doc.service === "GMail"); }, refresh: function(frm) { @@ -150,11 +152,20 @@ frappe.ui.form.on("Email Account", { }, after_save(frm) { - if (frm.doc.use_oauth && !frm.doc.refresh_token) { + if (frm.doc.auth_method === "Oauth" && !frm.doc.refresh_token) { oauth_access(frm); } }, + toggle_auth_method: function(frm) { + if (frm.doc.service !== "GMail") { + frm.toggle_display(['auth_method'], false); + frm.doc.auth_method = "Basic"; + } else { + frm.toggle_display(['auth_method'], true); + } + }, + show_gmail_message_for_less_secure_apps: function(frm) { frm.dashboard.clear_headline(); let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use OAuth."); @@ -166,8 +177,8 @@ frappe.ui.form.on("Email Account", { }, show_oauth_authorization_message(frm) { - let msg = __("Oauth Enabled but not Authorized. Please use `Authorize API Access` Button to do the same."); - if (frm.doc.use_oauth && !frm.doc.refresh_token) { + if (frm.doc.auth_method === "Oauth" && !frm.doc.refresh_token) { + let msg = __("Oauth Enabled but not Authorized. Please use Authorize API Access Button to do the same."); frm.dashboard.clear_headline(); frm.dashboard.set_headline_alert(msg, "yellow"); } diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index bd4023d62b..2c42010897 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -14,12 +14,12 @@ "domain", "service", "authentication_column", + "auth_method", + "authorize_api_access", "password", "awaiting_password", "ascii_encode_password", "column_break_10", - "use_oauth", - "authorize_api_access", "refresh_token", "access_token", "login_id_is_different", @@ -103,6 +103,7 @@ "label": "Email Login ID" }, { + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "password", "fieldtype": "Password", "hide_days": 1, @@ -111,6 +112,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "awaiting_password", "fieldtype": "Check", "hide_days": 1, @@ -119,6 +121,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "ascii_encode_password", "fieldtype": "Check", "hide_days": 1, @@ -583,14 +586,7 @@ "label": "IMAP Details" }, { - "default": "0", - "depends_on": "eval: doc.service === \"GMail\"", - "fieldname": "use_oauth", - "fieldtype": "Check", - "label": "Use OAuth" - }, - { - "depends_on": "eval: doc.service === \"GMail\" && doc.use_oauth && !doc.__islocal && !doc.__unsaved", + "depends_on": "eval: doc.service === \"GMail\" && doc.auth_method === \"Oauth\" && !doc.__islocal && !doc.__unsaved", "fieldname": "authorize_api_access", "fieldtype": "Button", "label": "Authorize API Access" @@ -608,12 +604,19 @@ "hidden": 1, "label": "Access Token", "read_only": 1 + }, + { + "default": "Basic", + "fieldname": "auth_method", + "fieldtype": "Select", + "label": "Method", + "options": "Basic\nOauth" } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-07-11 15:11:15.196935", + "modified": "2022-07-11 17:29:13.651583", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index d3273ab89d..a3d4b3ba32 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -81,10 +81,13 @@ class EmailAccount(Document): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - if getattr(self, "service", "") != "GMail" and self.use_oauth: - self.use_oauth = 0 + use_oauth = self.auth_method == "Oauth" - if self.use_oauth: + if getattr(self, "service", "") != "GMail" and use_oauth: + self.auth_method = "Basic" + use_oauth = False + + if use_oauth: # no need for awaiting password for oauth self.awaiting_password = 0 @@ -92,7 +95,7 @@ class EmailAccount(Document): # clear access & refresh token self.refresh_token = self.access_token = None - if not frappe.local.flags.in_install and (self.use_oauth or not self.awaiting_password): + if not frappe.local.flags.in_install and (use_oauth or not self.awaiting_password): if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() @@ -102,7 +105,7 @@ class EmailAccount(Document): self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - if not self.use_oauth: + if not use_oauth: frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: @@ -155,7 +158,7 @@ class EmailAccount(Document): awaiting_password=self.awaiting_password, email_id=self.email_id, enable_outgoing=self.enable_outgoing, - used_oauth=self.use_oauth, + used_oauth=self.auth_method == "Oauth", ) def there_must_be_only_one_default(self): @@ -210,7 +213,7 @@ class EmailAccount(Document): "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, - "use_oauth": self.use_oauth or 0, + "use_oauth": self.auth_method == "Oauth", "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, "access_token": decrypt(self.access_token) if self.access_token else None, } @@ -279,7 +282,9 @@ class EmailAccount(Document): @property def _password(self): - raise_exception = not (self.use_oauth or self.no_smtp_authentication or frappe.flags.in_test) + raise_exception = not ( + self.auth_method == "Oauth" or self.no_smtp_authentication or frappe.flags.in_test + ) return self.get_password(raise_exception=raise_exception) @property @@ -398,7 +403,7 @@ class EmailAccount(Document): "default": 0, }, "name": {"conf_names": ("email_sender_name",), "default": "Frappe"}, - "use_oauth": {"conf_names": ("use_oauth"), "default": 0}, + "auth_method": {"conf_names": ("auth_method"), "default": "Basic"}, "access_token": {"conf_names": ("mail_access_token")}, "refresh_token": {"conf_names": ("mail_refresh_token")}, "from_site_config": {"default": True}, @@ -426,7 +431,7 @@ class EmailAccount(Document): "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), "service": getattr(self, "service", ""), - "use_oauth": self.use_oauth or 0, + "use_oauth": self.auth_method == "Oauth", "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, "access_token": decrypt(self.access_token) if self.access_token else None, } @@ -800,7 +805,7 @@ def pull(now=False): for email_account in frappe.get_list( "Email Account", filters={"enable_incoming": 1}, - or_filters={"awaiting_password": 0, "use_oauth": 1}, + or_filters={"awaiting_password": 0, "auth_method": "Oauth"}, ): if now: pull_from_email_account(email_account.name) @@ -923,7 +928,7 @@ def remove_user_email_inbox(email_account): @frappe.whitelist() def set_email_password(email_account, password): account = frappe.get_doc("Email Account", email_account) - if account.awaiting_password and not account.use_oauth: + if account.awaiting_password and not account.auth_method == "Oauth": account.awaiting_password = 0 account.password = password try: diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index cb9dcbbc85..140da6afed 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -2,7 +2,6 @@ import base64 from imaplib import IMAP4 from poplib import POP3 from smtplib import SMTP -from typing import Union from urllib.parse import quote import frappe @@ -124,10 +123,6 @@ def oauth_access(email_account: str, service: str): doctype = "Email Account" - # NOTE: setting this here, since we redirect to the service's auth page, - # we lose the use_oauth value in the emal account form - frappe.db.set_value(doctype, email_account, "use_oauth", 1, update_modified=False) - if service == "GMail": return authorize_google_access(email_account, doctype) From a826f4cc5320d62f782c8d2c5f0c15d8bb05909c Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 11 Jul 2022 17:51:56 +0530 Subject: [PATCH 119/201] fix(ux): consistent field names for ssl and server * keep server and port together --- .../email/doctype/email_account/email_account.js | 4 ++++ .../email/doctype/email_account/email_account.json | 14 +++++++------- frappe/email/oauth.py | 5 +++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 9fd9c1a0bf..46c3742fde 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -145,6 +145,10 @@ frappe.ui.form.on("Email Account", { frm.events.show_gmail_message_for_less_secure_apps(frm); frm.events.show_oauth_authorization_message(frm); + if (frm.doc.service !== "Gmail") { + frm.doc.auth_method = "Basic"; + } + if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; delete locals['User'][frappe.route_flags.linked_user]; diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 2c42010897..feb1a998f6 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -48,9 +48,9 @@ "send_notification_to", "outgoing_mail_settings", "enable_outgoing", - "smtp_server", "use_tls", "use_ssl_for_outgoing", + "smtp_server", "smtp_port", "column_break_38", "default_outgoing", @@ -92,7 +92,7 @@ "fieldtype": "Check", "hide_days": 1, "hide_seconds": 1, - "label": "Use different login" + "label": "Use different Email ID" }, { "depends_on": "login_id_is_different", @@ -100,7 +100,7 @@ "fieldtype": "Data", "hide_days": 1, "hide_seconds": 1, - "label": "Email Login ID" + "label": "Alternative Email ID" }, { "depends_on": "eval: doc.auth_method === \"Basic\"", @@ -190,7 +190,7 @@ "fieldtype": "Data", "hide_days": 1, "hide_seconds": 1, - "label": "Email Server" + "label": "Incoming Server" }, { "default": "0", @@ -312,7 +312,7 @@ "fieldtype": "Data", "hide_days": 1, "hide_seconds": 1, - "label": "SMTP Server" + "label": "Outgoing Server" }, { "default": "0", @@ -532,7 +532,7 @@ "fieldtype": "Check", "hide_days": 1, "hide_seconds": 1, - "label": "Use SSL for Outgoing" + "label": "Use SSL" }, { "default": "1", @@ -616,7 +616,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-07-11 17:29:13.651583", + "modified": "2022-07-11 18:34:06.945668", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 140da6afed..74708bf2ec 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -36,6 +36,11 @@ class Oauth: self._validate() def _validate(self) -> None: + if self.service != "GMail": + raise NotImplementedError( + f"Service {self.service} currently doesn't have oauth implementation." + ) + if not self._refresh_token: frappe.throw( frappe._("Please Authorize OAuth."), From da33f6e6d8102a26ca92ee33021f97165c4f7caf Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 11 Jul 2022 18:08:19 +0530 Subject: [PATCH 120/201] fix: remove fetch-from from user email for used_oauth --- frappe/core/doctype/user_email/user_email.json | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/core/doctype/user_email/user_email.json b/frappe/core/doctype/user_email/user_email.json index 43dc3ace8d..6e3f813035 100644 --- a/frappe/core/doctype/user_email/user_email.json +++ b/frappe/core/doctype/user_email/user_email.json @@ -52,7 +52,6 @@ }, { "default": "0", - "fetch_from": "email_account.use_oauth", "fieldname": "used_oauth", "fieldtype": "Check", "in_list_view": 1, From 5ea642ef7fe23a502d88a69954ee751632c4f776 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 12 Jul 2022 15:11:51 +0530 Subject: [PATCH 121/201] fix(ux): update form header upon successful authorization * minor: simplified validations for email account --- frappe/email/doctype/email_account/email_account.js | 13 ++++++------- frappe/email/doctype/email_account/email_account.py | 13 +++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 46c3742fde..9ce72b055c 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -145,10 +145,6 @@ frappe.ui.form.on("Email Account", { frm.events.show_gmail_message_for_less_secure_apps(frm); frm.events.show_oauth_authorization_message(frm); - if (frm.doc.service !== "Gmail") { - frm.doc.auth_method = "Basic"; - } - if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; delete locals['User'][frappe.route_flags.linked_user]; @@ -181,10 +177,13 @@ frappe.ui.form.on("Email Account", { }, show_oauth_authorization_message(frm) { - if (frm.doc.auth_method === "Oauth" && !frm.doc.refresh_token) { - let msg = __("Oauth Enabled but not Authorized. Please use Authorize API Access Button to do the same."); + if (frm.doc.auth_method === "Oauth") { + let msg = { + message: !frm.doc.refresh_token ? "Oauth Enabled but not Authorized. Please use Authorize API Access Button to do the same." : "Oauth Authorized. Re-Authorization can be done using Authorize API Access Button.", + indicator: !frm.doc.refresh_token ? "yellow" : "green" + } frm.dashboard.clear_headline(); - frm.dashboard.set_headline_alert(msg, "yellow"); + frm.dashboard.set_headline_alert(__(msg.message), msg.indicator); } }, diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index a3d4b3ba32..8f2107e165 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -95,7 +95,7 @@ class EmailAccount(Document): # clear access & refresh token self.refresh_token = self.access_token = None - if not frappe.local.flags.in_install and (use_oauth or not self.awaiting_password): + if not frappe.local.flags.in_install and not self.awaiting_password: if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() @@ -114,11 +114,12 @@ class EmailAccount(Document): for e in self.get_unreplied_notification_emails(): validate_email_address(e, True) - for folder in self.imap_folder: - if self.enable_incoming and folder.append_to: - valid_doctypes = [d[0] for d in get_append_to()] - if folder.append_to not in valid_doctypes: - frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) + if self.enable_incoming: + for folder in self.imap_folder: + if folder.append_to: + valid_doctypes = [d[0] for d in get_append_to()] + if folder.append_to not in valid_doctypes: + frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) def validate_smtp_conn(self): if not self.smtp_server: From a50568596acaa5a1820c5ff093784c531483c5d2 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 12 Jul 2022 16:22:45 +0530 Subject: [PATCH 122/201] minor: fetch attachment size from get_max_file_size api --- .../email/doctype/email_account/email_account.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 9ce72b055c..eabc8c19aa 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -82,6 +82,19 @@ function oauth_access(frm) { }); } +function set_default_max_attachment_size(frm, field) { + if (frm.doc.__islocal && !frm.doc[field]) { + frappe.call({ + method: "frappe.core.api.file.get_max_file_size", + callback: function(r) { + if (!r.exc) { + frm.set_value(field, Number(r.message)/(1024*1024)); + } + }, + }); + } +} + frappe.ui.form.on("Email Account", { service: function(frm) { $.each(frappe.email_defaults[frm.doc.service], function(key, value) { @@ -136,6 +149,7 @@ frappe.ui.form.on("Email Account", { frm.refresh_field("imap_folder"); } frm.toggle_display(['auth_method'], frm.doc.service === "GMail"); + set_default_max_attachment_size(frm, "attachment_limit"); }, refresh: function(frm) { From 81bb9c411ad384163fa0200a04e409d631eb1c2a Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 12 Jul 2022 16:50:02 +0530 Subject: [PATCH 123/201] chore: fix sider --- frappe/email/doctype/email_account/email_account.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index eabc8c19aa..f86ec12344 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -195,7 +195,7 @@ frappe.ui.form.on("Email Account", { let msg = { message: !frm.doc.refresh_token ? "Oauth Enabled but not Authorized. Please use Authorize API Access Button to do the same." : "Oauth Authorized. Re-Authorization can be done using Authorize API Access Button.", indicator: !frm.doc.refresh_token ? "yellow" : "green" - } + }; frm.dashboard.clear_headline(); frm.dashboard.set_headline_alert(__(msg.message), msg.indicator); } From d1a199258daa62d2a79d99ccde23e6013b14302d Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 12 Jul 2022 17:47:47 +0530 Subject: [PATCH 124/201] fix: pull from accounts for oauth whose refresh_token is not null * chore: rename Oauth to OAuth --- .../doctype/email_account/email_account.js | 6 ++-- .../doctype/email_account/email_account.json | 4 +-- .../doctype/email_account/email_account.py | 31 ++++++++++++------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index f86ec12344..34d6e6acd8 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -166,7 +166,7 @@ frappe.ui.form.on("Email Account", { }, after_save(frm) { - if (frm.doc.auth_method === "Oauth" && !frm.doc.refresh_token) { + if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) { oauth_access(frm); } }, @@ -191,9 +191,9 @@ frappe.ui.form.on("Email Account", { }, show_oauth_authorization_message(frm) { - if (frm.doc.auth_method === "Oauth") { + if (frm.doc.auth_method === "OAuth") { let msg = { - message: !frm.doc.refresh_token ? "Oauth Enabled but not Authorized. Please use Authorize API Access Button to do the same." : "Oauth Authorized. Re-Authorization can be done using Authorize API Access Button.", + message: !frm.doc.refresh_token ? "OAuth Enabled but not Authorized. Please use Authorize API Access Button to do the same." : "OAuth Authorized. Re-Authorization can be done using Authorize API Access Button.", indicator: !frm.doc.refresh_token ? "yellow" : "green" }; frm.dashboard.clear_headline(); diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index feb1a998f6..ecb5af7378 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -586,7 +586,7 @@ "label": "IMAP Details" }, { - "depends_on": "eval: doc.service === \"GMail\" && doc.auth_method === \"Oauth\" && !doc.__islocal && !doc.__unsaved", + "depends_on": "eval: doc.service === \"GMail\" && doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved", "fieldname": "authorize_api_access", "fieldtype": "Button", "label": "Authorize API Access" @@ -610,7 +610,7 @@ "fieldname": "auth_method", "fieldtype": "Select", "label": "Method", - "options": "Basic\nOauth" + "options": "Basic\nOAuth" } ], "icon": "fa fa-inbox", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 8f2107e165..589ddf42f0 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -81,7 +81,7 @@ class EmailAccount(Document): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - use_oauth = self.auth_method == "Oauth" + use_oauth = self.auth_method == "OAuth" if getattr(self, "service", "") != "GMail" and use_oauth: self.auth_method = "Basic" @@ -159,7 +159,7 @@ class EmailAccount(Document): awaiting_password=self.awaiting_password, email_id=self.email_id, enable_outgoing=self.enable_outgoing, - used_oauth=self.auth_method == "Oauth", + used_oauth=self.auth_method == "OAuth", ) def there_must_be_only_one_default(self): @@ -214,7 +214,7 @@ class EmailAccount(Document): "email_sync_rule": email_sync_rule, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, - "use_oauth": self.auth_method == "Oauth", + "use_oauth": self.auth_method == "OAuth", "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, "access_token": decrypt(self.access_token) if self.access_token else None, } @@ -284,7 +284,7 @@ class EmailAccount(Document): @property def _password(self): raise_exception = not ( - self.auth_method == "Oauth" or self.no_smtp_authentication or frappe.flags.in_test + self.auth_method == "OAuth" or self.no_smtp_authentication or frappe.flags.in_test ) return self.get_password(raise_exception=raise_exception) @@ -432,7 +432,7 @@ class EmailAccount(Document): "use_ssl": cint(self.use_ssl_for_outgoing), "use_tls": cint(self.use_tls), "service": getattr(self, "service", ""), - "use_oauth": self.auth_method == "Oauth", + "use_oauth": self.auth_method == "OAuth", "refresh_token": decrypt(self.refresh_token) if self.refresh_token else None, "access_token": decrypt(self.access_token) if self.access_token else None, } @@ -802,12 +802,18 @@ def pull(now=False): else: return - queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] - for email_account in frappe.get_list( - "Email Account", - filters={"enable_incoming": 1}, - or_filters={"awaiting_password": 0, "auth_method": "Oauth"}, - ): + doctype = frappe.qb.DocType("Email Account") + email_accounts = ( + frappe.qb.from_(doctype) + .select(doctype.name) + .where(doctype.enable_incoming == 1) + .where( + (doctype.awaiting_password == 0) + | ((doctype.auth_method == "OAuth") & (doctype.refresh_token.isnotnull())) + ) + .run(as_dict=1) + ) + for email_account in email_accounts: if now: pull_from_email_account(email_account.name) @@ -815,6 +821,7 @@ def pull(now=False): # job_name is used to prevent duplicates in queue job_name = f"pull_from_email_account|{email_account.name}" + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] if job_name not in queued_jobs: enqueue( pull_from_email_account, @@ -929,7 +936,7 @@ def remove_user_email_inbox(email_account): @frappe.whitelist() def set_email_password(email_account, password): account = frappe.get_doc("Email Account", email_account) - if account.awaiting_password and not account.auth_method == "Oauth": + if account.awaiting_password and not account.auth_method == "OAuth": account.awaiting_password = 0 account.password = password try: From 5b7d37477ee64e0012a2d59b04a3500903f6ea69 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 12 Jul 2022 17:48:49 +0530 Subject: [PATCH 125/201] chore: add link for google settings when throwing error --- frappe/integrations/google_oauth.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index 2bb1aaca1f..9ae4461cc1 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -42,13 +42,15 @@ class GoogleOAuth: self.validate_google_settings() def validate_google_settings(self): + google_settings = "Google Settings" + if not self.google_settings.enable: - frappe.throw(frappe._("Please enable Google Settings before continuing.")) + frappe.throw(frappe._("Please enable {} before continuing.").format(google_settings)) if not (self.google_settings.client_id and self.google_settings.client_secret): - frappe.throw(frappe._("Please update Google Settings before continuing.")) + frappe.throw(frappe._("Please update {} before continuing.").format(google_settings)) - def authorize(self, oauth_code: str) -> Dict[str, Union[str, int]]: + def authorize(self, oauth_code: str) -> dict[str, str | int]: """Returns a dict with access and refresh token. :param oauth_code: code got back from google upon successful auhtorization @@ -72,7 +74,7 @@ class GoogleOAuth: "Something went wrong during the authorization.", ) - def refresh_access_token(self, refresh_token: str) -> Dict[str, Union[str, int]]: + def refresh_access_token(self, refresh_token: str) -> dict[str, str | int]: """Refreshes google access token using refresh token""" data = { @@ -92,7 +94,7 @@ class GoogleOAuth: raise_err=True, ) - def get_authentication_url(self, state: Dict[str, str]) -> Dict[str, str]: + def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]: """Returns google authentication url. :param site_address: side address from which the request is being made (for redirect back to site) @@ -105,7 +107,7 @@ class GoogleOAuth: return { "url": "https://accounts.google.com/o/oauth2/v2/auth?" + "access_type=offline&response_type=code&prompt=consent&include_granted_scopes=true&" - + "client_id={0}&scope={1}&redirect_uri={2}&state={3}".format( + + "client_id={}&scope={}&redirect_uri={}&state={}".format( self.google_settings.client_id, self.scopes, callback_url, state ) } @@ -133,7 +135,7 @@ class GoogleOAuth: def handle_response( - response: Dict[str, Union[str, int]], + response: dict[str, str | int], error_title: str, error_message: str, raise_err: bool = False, From 01a1860491201bb5d942a0a175fae4281cf3e5a6 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 13 Jul 2022 11:31:11 +0530 Subject: [PATCH 126/201] fix(ux): better ux for successful oauthorization --- .../doctype/email_account/email_account.js | 18 ++++++++++++------ frappe/email/oauth.py | 2 ++ frappe/integrations/google_oauth.py | 5 ++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 34d6e6acd8..093c2a3820 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -137,6 +137,15 @@ frappe.ui.form.on("Email Account", { }, onload: function(frm) { + let oauth_authorization_state = frappe.utils.get_query_params().successful_authorization + if (oauth_authorization_state === '1') { + frappe.show_alert("Successfully Authorized"); + // FIXME: find better alternative + window.history.replaceState(null, "", window.location.pathname); + } else if (oauth_authorization_state === '0') { + window.history.replaceState(null, "", window.location.pathname); + } + frm.set_df_property("append_to", "only_select", true); frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to"); frm.set_query("append_to", "imap_folder", function() { @@ -191,13 +200,10 @@ frappe.ui.form.on("Email Account", { }, show_oauth_authorization_message(frm) { - if (frm.doc.auth_method === "OAuth") { - let msg = { - message: !frm.doc.refresh_token ? "OAuth Enabled but not Authorized. Please use Authorize API Access Button to do the same." : "OAuth Authorized. Re-Authorization can be done using Authorize API Access Button.", - indicator: !frm.doc.refresh_token ? "yellow" : "green" - }; + if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) { + let msg = __('OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.') frm.dashboard.clear_headline(); - frm.dashboard.set_headline_alert(__(msg.message), msg.indicator); + frm.dashboard.set_headline_alert(msg, "yellow"); } }, diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 74708bf2ec..8cf7a42ab5 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -146,6 +146,8 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: { "method": "frappe.email.oauth.authorize_google_access", "redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}", + "success_query_param": "successful_authorization=1", + "failure_query_param": "successful_authorization=0", "email_account": email_account, }, ) diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index 9ae4461cc1..f88a47ba6a 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -1,5 +1,4 @@ import json -from typing import Dict, Union from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -172,6 +171,8 @@ def callback(state: str, code: str = None, error: str = None) -> None: state = json.loads(state) redirect = state.pop("redirect", "/app") + success_query_param = state.pop("success_query_param", "") + failure_query_param = state.pop("failure_query_param", "") if not error: state.update({"code": code}) @@ -180,5 +181,7 @@ def callback(state: str, code: str = None, error: str = None) -> None: # GET request, hence using commit to persist changes frappe.db.commit() + redirect = f"{redirect}?{failure_query_param if error else success_query_param}" + frappe.local.response["type"] = "redirect" frappe.local.response["location"] = redirect From 31c5f260d7504d8c2bd151d98f3331b78806f016 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 13 Jul 2022 12:29:25 +0530 Subject: [PATCH 127/201] chore: use f-strings --- .../integrations/doctype/google_contacts/google_contacts.py | 4 +++- frappe/integrations/doctype/google_drive/google_drive.py | 2 +- frappe/website/doctype/website_settings/google_indexing.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index eb2a8af647..c1f445b599 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -2,6 +2,8 @@ # License: MIT. See LICENSE +from urllib.parse import quote + from googleapiclient.errors import HttpError import frappe @@ -45,7 +47,7 @@ def authorize_access(g_contact, reauthorize=False, code=None): { "method": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", "g_contact": g_contact, - "redirect": "/app/Form/Google%20Contacts/{}".format(g_contact), + "redirect": f"/app/Form/{quote('Google Contacts')}/{quote(g_contact)}", }, ) diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index f4a6f1f46e..62100ae7c5 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -58,7 +58,7 @@ def authorize_access(reauthorize=False, code=None): return oauth_obj.get_authentication_url( { "method": "frappe.integrations.doctype.google_drive.google_drive.authorize_access", - "redirect": "/app/Form/{0}".format(quote("Google Drive")), + "redirect": f"/app/Form/{quote('Google Drive')}", }, ) diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 3577f688f5..4f67949f86 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -27,7 +27,7 @@ def authorize_access(reauthorize=False, code=None): return oauth_obj.get_authentication_url( { "method": "frappe.website.doctype.website_settings.google_indexing.authorize_access", - "redirect": "/app/Form/{0}".format(quote("Website Settings")), + "redirect": f"/app/Form/{quote('Website Settings')}", }, ) From e762fe9ce168ea7bb144ff08ad30fa0281b05627 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Jul 2022 12:29:41 +0530 Subject: [PATCH 128/201] test: clear filters on ToDo before running test (#17494) --- cypress/integration/form.js | 3 ++- cypress/integration/list_paging.js | 1 + cypress/integration/list_view.js | 2 ++ cypress/integration/timeline.js | 5 +++-- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cypress/integration/form.js b/cypress/integration/form.js index 4d50a5f66a..b395ff77b2 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -17,7 +17,8 @@ context('Form', () => { cy.get('.primary-action').click(); cy.wait('@form_save').its('response.statusCode').should('eq', 200); - cy.visit('/app/todo'); + cy.go_to_list('ToDo'); + cy.clear_filters() cy.get('.page-head').findByTitle('To Do').should('exist'); cy.get('.list-row').should('contain', 'this is a test todo'); }); diff --git a/cypress/integration/list_paging.js b/cypress/integration/list_paging.js index 4a59024a7b..0cf6f2e565 100644 --- a/cypress/integration/list_paging.js +++ b/cypress/integration/list_paging.js @@ -9,6 +9,7 @@ context('List Paging', () => { it('test load more with count selection buttons', () => { cy.visit('/app/todo/view/report'); + cy.clear_filters() cy.get('.list-paging-area .list-count').should('contain.text', '20 of'); cy.get('.list-paging-area .btn-more').click(); diff --git a/cypress/integration/list_view.js b/cypress/integration/list_view.js index 3e0d1c9d50..ee12b37638 100644 --- a/cypress/integration/list_view.js +++ b/cypress/integration/list_view.js @@ -9,6 +9,7 @@ context('List View', () => { it('Keep checkbox checked after Refresh', () => { cy.go_to_list('ToDo'); + cy.clear_filters() cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true }); cy.get('.actions-btn-group button').contains('Actions').should('be.visible'); cy.intercept('/api/method/frappe.desk.reportview.get').as('list-refresh'); @@ -21,6 +22,7 @@ context('List View', () => { it('enables "Actions" button', () => { const actions = ['Approve', 'Reject', 'Edit', 'Export', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete']; cy.go_to_list('ToDo'); + cy.clear_filters() cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ multiple: true, force: true }); cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click(); cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 9).each((el, index) => { diff --git a/cypress/integration/timeline.js b/cypress/integration/timeline.js index cb4d43a96a..e7308fbaa7 100644 --- a/cypress/integration/timeline.js +++ b/cypress/integration/timeline.js @@ -12,7 +12,8 @@ context('Timeline', () => { cy.get('[data-fieldname="description"] .ql-editor.ql-blank').type('Test ToDo', {force: true}).wait(200); cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); - cy.visit('/app/todo'); + cy.go_to_list('ToDo'); + cy.clear_filters() cy.click_listview_row_item(0); //To check if the comment box is initially empty and tying some text into it @@ -79,4 +80,4 @@ context('Timeline', () => { cy.get('.page-actions .actions-btn-group [data-label="Delete"]').click(); cy.click_modal_primary_button('Yes'); }); -}); \ No newline at end of file +}); From 9090b0fe3a50cfb612e1d1a1dc0c3f3b118157d8 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 13 Jul 2022 12:41:32 +0530 Subject: [PATCH 129/201] chore: remove unnecessary failure_query_param for email oauth --- frappe/email/doctype/email_account/email_account.js | 5 +---- frappe/email/oauth.py | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 093c2a3820..155c4775fb 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -137,13 +137,10 @@ frappe.ui.form.on("Email Account", { }, onload: function(frm) { - let oauth_authorization_state = frappe.utils.get_query_params().successful_authorization - if (oauth_authorization_state === '1') { + if (frappe.utils.get_query_params().successful_authorization === '1') { frappe.show_alert("Successfully Authorized"); // FIXME: find better alternative window.history.replaceState(null, "", window.location.pathname); - } else if (oauth_authorization_state === '0') { - window.history.replaceState(null, "", window.location.pathname); } frm.set_df_property("append_to", "only_select", true); diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 8cf7a42ab5..46d1565275 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -147,7 +147,6 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: "method": "frappe.email.oauth.authorize_google_access", "redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}", "success_query_param": "successful_authorization=1", - "failure_query_param": "successful_authorization=0", "email_account": email_account, }, ) From 7d5262f5e0c6b9b1c6ae0d7a467982bbe26490b9 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 13 Jul 2022 12:50:04 +0530 Subject: [PATCH 130/201] chore: translate authorization message and add nosemgrep --- frappe/email/doctype/email_account/email_account.js | 2 +- frappe/integrations/google_oauth.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 155c4775fb..d22009963d 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -138,7 +138,7 @@ frappe.ui.form.on("Email Account", { onload: function(frm) { if (frappe.utils.get_query_params().successful_authorization === '1') { - frappe.show_alert("Successfully Authorized"); + frappe.show_alert(__("Successfully Authorized")); // FIXME: find better alternative window.history.replaceState(null, "", window.location.pathname); } diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index f88a47ba6a..edce63493e 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -179,7 +179,7 @@ def callback(state: str, code: str = None, error: str = None) -> None: frappe.get_attr(state.pop("method"))(**state) # GET request, hence using commit to persist changes - frappe.db.commit() + frappe.db.commit() # nosemgrep redirect = f"{redirect}?{failure_query_param if error else success_query_param}" From 843f241c136dac748c0c0d30d5df8e98316c363e Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 13 Jul 2022 09:41:37 +0000 Subject: [PATCH 131/201] fix: copy global defaults before updating to avoid cache mutation (#17497) Co-authored-by: Ankush Menat --- frappe/defaults.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frappe/defaults.py b/frappe/defaults.py index d0a49ef692..02076b1fda 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -85,18 +85,19 @@ def get_user_permissions(user=None): def get_defaults(user=None): - globald = get_defaults_for() + global_defaults = get_defaults_for() if not user: user = frappe.session.user if frappe.session else "Guest" - if user: - userd = {} - userd.update(get_defaults_for(user)) - userd.update({"user": user, "owner": user}) - globald.update(userd) + if not user: + return global_defaults - return globald + defaults = global_defaults.copy() + defaults.update(get_defaults_for(user)) + defaults.update(user=user, owner=user) + + return defaults def clear_user_default(key, user=None): From 468a5c55a32f92763c68a64cf16141e17c1eb5bd Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Jul 2022 18:08:20 +0530 Subject: [PATCH 132/201] ci: fix weird version check in cypress tests (#17499) --- frappe/commands/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 1b63914030..585478c0ff 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -872,7 +872,6 @@ def run_ui_tests( and os.path.exists(real_events_plugin_path) and os.path.exists(testing_library_path) and os.path.exists(coverage_plugin_path) - and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6 ): # install cypress click.secho("Installing Cypress...", fg="yellow") From 2791066bb2f6801713734b1f21e633212efa2c81 Mon Sep 17 00:00:00 2001 From: uepselon <49870752+uepselon@users.noreply.github.com> Date: Wed, 13 Jul 2022 16:05:41 +0200 Subject: [PATCH 133/201] fix: allow System Manager to reset OTP secret * squashed: Change Admin based OTP reset to role based reset (System Manager) * fix: show `Reset OTP Secret` button only if applicable * chore: flatten code, use `only_for` API Co-authored-by: Leonard Goertz Co-authored-by: Sagar Vora --- frappe/core/doctype/user/user.js | 18 +++++----- frappe/twofactor.py | 58 +++++++++++++++++--------------- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 001aae4da0..41b7e7fb38 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -173,14 +173,16 @@ frappe.ui.form.on('User', { }); } - frm.add_custom_button(__("Reset OTP Secret"), function() { - frappe.call({ - method: "frappe.twofactor.reset_otp_secret", - args: { - "user": frm.doc.name - } - }); - }, __("Password")); + if (frappe.session.user == doc.name || frappe.user.has_role("System Manager")) { + frm.add_custom_button(__("Reset OTP Secret"), function() { + frappe.call({ + method: "frappe.twofactor.reset_otp_secret", + args: { + "user": frm.doc.name + } + }); + }, __("Password")); + } frm.trigger('enabled'); diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 55c27e2bac..26fc3ad619 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -461,33 +461,35 @@ def disable(): @frappe.whitelist() def reset_otp_secret(user): + if frappe.session.user != user: + frappe.only_for("System Manager", message=True) + otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") user_email = frappe.db.get_value("User", user, "email") - if frappe.session.user in ["Administrator", user]: - frappe.defaults.clear_default(user + "_otplogin") - frappe.defaults.clear_default(user + "_otpsecret") - email_args = { - "recipients": user_email, - "sender": None, - "subject": _("OTP Secret Reset - {0}").format(otp_issuer or "Frappe Framework"), - "message": _( - "

Your OTP secret on {0} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.

" - ).format(otp_issuer or "Frappe Framework"), - "delayed": False, - "retry": 3, - } - enqueue( - method=frappe.sendmail, - queue="short", - timeout=300, - event=None, - is_async=True, - job_name=None, - now=False, - **email_args, - ) - return frappe.msgprint( - _("OTP Secret has been reset. Re-registration will be required on next login.") - ) - else: - return frappe.throw(_("OTP secret can only be reset by the Administrator.")) + + frappe.defaults.clear_default(user + "_otplogin") + frappe.defaults.clear_default(user + "_otpsecret") + + email_args = { + "recipients": user_email, + "sender": None, + "subject": _("OTP Secret Reset - {0}").format(otp_issuer or "Frappe Framework"), + "message": _( + "

Your OTP secret on {0} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.

" + ).format(otp_issuer or "Frappe Framework"), + "delayed": False, + "retry": 3, + } + + enqueue( + method=frappe.sendmail, + queue="short", + timeout=300, + event=None, + is_async=True, + job_name=None, + now=False, + **email_args, + ) + + frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login.")) From 5a55507e6d8b6401e5d3a5fd9e3e4853969bdf41 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Jul 2022 20:17:31 +0530 Subject: [PATCH 134/201] fix: pass docfield for custom indicator formatter (#17501) --- frappe/public/js/frappe/form/formatters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 3bf36c86af..5cf5a2f4f3 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -26,7 +26,7 @@ frappe.form.formatters = { if (df) { const std_df = frappe.meta.docfield_map[df.parent] && frappe.meta.docfield_map[df.parent][df.fieldname]; if (std_df && std_df.formatter && typeof std_df.formatter==='function') { - value = std_df.formatter(value); + value = std_df.formatter(value, df); } } return value; From 99d3fe3893ebef7732c8f7bad01925f90d9b5b68 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 19:59:42 +0530 Subject: [PATCH 135/201] fix(dark theme): background color on version page --- frappe/core/doctype/version/version_view.html | 11 +++++----- frappe/public/scss/common/css_variables.scss | 4 ++++ frappe/public/scss/desk/dark.scss | 4 ++++ frappe/public/scss/desk/global.scss | 20 ++++++------------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/version/version_view.html b/frappe/core/doctype/version/version_view.html index 67f005ed4c..a17460ccc7 100644 --- a/frappe/core/doctype/version/version_view.html +++ b/frappe/core/doctype/version/version_view.html @@ -18,8 +18,8 @@ {% for item in data.changed %} {{ frappe.meta.get_label(doc.ref_doctype, item[0]) }} - {{ item[1] }} - {{ item[2] }} + {{ item[1] }} + {{ item[2] }} {% endfor %} @@ -43,8 +43,7 @@ {% for item in values %} {{ frappe.meta.get_label(doc.ref_doctype, item[0]) }} - + {% var item_keys = Object.keys(item[1]).sort(); %} @@ -86,8 +85,8 @@ - - + + {% endfor %} {% endfor %} diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index ab52c10e45..efcbe06920 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -262,6 +262,10 @@ $input-height: 28px !default; --checkbox-focus-shadow: 0 0 0 2px var(--gray-300); --checkbox-gradient: linear-gradient(180deg, #4AC3F8 -124.51%, var(--primary) 100%); + // "diff" colors + --diff-added: var(--green-100); + --diff-removed: var(--red-100); + --right-arrow-svg: url("data: image/svg+xml;utf8, "); --left-arrow-svg: url("data: image/svg+xml;utf8, "); } diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index c627d88f89..3d05ecd237 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -93,6 +93,10 @@ --shadow-base: 0px 4px 8px rgba(114, 176, 233, 0.06), 0px 0px 4px rgba(112, 172, 228, 0.12); + // "diff" colors + --diff-added: var(--green-800); + --diff-removed: var(--red-800); + // input --input-disabled-bg: none; diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 6a85dea6dd..7466bdc874 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -579,22 +579,14 @@ details > summary:focus { color: var(--text-color); } - -.diffview .insert { - background-color: var(--green-100); +.diffview .insert, +.diff-add { + background-color: var(--diff-added); } -.diffview .delete { - background-color: var(--red-100); -} - -[data-theme="dark"] { - .diffview .insert { - background-color: var(--green-800); - } - .diffview .delete { - background-color: var(--red-800); - } +.diffview .delete, +.diff-remove { + background-color: var(--diff-removed); } // REDESIGN TODO: Handling of broken images? From 3a7c7ff18234bef055cc1c732a7c934fbe220746 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 20:23:34 +0530 Subject: [PATCH 136/201] ci: ignore HTML files for server tests --- .github/helper/roulette.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index c240443e9a..554f4ae5f5 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -46,7 +46,7 @@ def is_ci(file): return ".github" in file def is_frontend_code(file): - return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue")) + return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html")) def is_docs(file): regex = re.compile(r'\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE') From 2c36874554e6c70fe08bf9d3a9c7b9d5d2be64f1 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 13 Jul 2022 12:39:43 +0530 Subject: [PATCH 137/201] style: add config for CSS and SCSS --- .editorconfig | 2 +- frappe/public/scss/common/css_variables.scss | 8 ++++---- frappe/public/scss/desk/dark.scss | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.editorconfig b/.editorconfig index d76f67cd7f..f4c7f1528c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,6 +9,6 @@ trim_trailing_whitespace = true charset = utf-8 # python, js indentation settings -[{*.py,*.js,*.vue}] +[{*.py,*.js,*.vue,*.css,*.scss,*.html}] indent_style = tab indent_size = 4 diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index efcbe06920..0b6da84222 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -134,7 +134,7 @@ $input-height: 28px !default; --shadow-xs: rgba(0, 0, 0, 0.05) 0px 0.5px 0px 0px, rgba(0, 0, 0, 0.08) 0px 0px 0px 1px, rgba(0, 0, 0, 0.05) 0px 2px 4px 0px; --shadow-sm: 0px 1px 2px rgba(25, 39, 52, 0.05), 0px 0px 4px rgba(25, 39, 52, 0.1); - --shadow-base: 0px 4px 8px rgba(25, 39, 52, 0.06), 0px 0px 4px rgba(25, 39, 52, 0.12); + --shadow-base: 0px 4px 8px rgba(25, 39, 52, 0.06), 0px 0px 4px rgba(25, 39, 52, 0.12); --shadow-md: 0px 8px 14px rgba(25, 39, 52, 0.08), 0px 2px 6px rgba(25, 39, 52, 0.04); --shadow-lg: 0px 18px 22px rgba(25, 39, 52, 0.1), 0px 1px 10px rgba(0, 0, 0, 0.06), 0px 0.5px 5px rgba(25, 39, 52, 0.04); @@ -262,9 +262,9 @@ $input-height: 28px !default; --checkbox-focus-shadow: 0 0 0 2px var(--gray-300); --checkbox-gradient: linear-gradient(180deg, #4AC3F8 -124.51%, var(--primary) 100%); - // "diff" colors - --diff-added: var(--green-100); - --diff-removed: var(--red-100); + // "diff" colors + --diff-added: var(--green-100); + --diff-removed: var(--red-100); --right-arrow-svg: url("data: image/svg+xml;utf8, "); --left-arrow-svg: url("data: image/svg+xml;utf8, "); diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index 3d05ecd237..731ff4525d 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -91,11 +91,11 @@ --highlight-shadow: 1px 1px 10px var(--blue-900), 0px 0px 4px var(--blue-500); - --shadow-base: 0px 4px 8px rgba(114, 176, 233, 0.06), 0px 0px 4px rgba(112, 172, 228, 0.12); + --shadow-base: 0px 4px 8px rgba(114, 176, 233, 0.06), 0px 0px 4px rgba(112, 172, 228, 0.12); - // "diff" colors - --diff-added: var(--green-800); - --diff-removed: var(--red-800); + // "diff" colors + --diff-added: var(--green-800); + --diff-removed: var(--red-800); // input --input-disabled-bg: none; From 95f67b8de85267d6620f9ea311bf07bd45a1fc91 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 14 Jul 2022 13:14:58 +0530 Subject: [PATCH 138/201] fix: ignore empty part in naming series (#17508) on v13 doc.get("") returns entire doc dictionary, this gets strigified and becomes a problem for naming. --- frappe/model/naming.py | 21 +++++++++++++++++++-- frappe/tests/test_naming.py | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 0ce6704c39..49a58da314 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import datetime import re from typing import TYPE_CHECKING, Callable, Optional @@ -23,6 +24,17 @@ NAMING_SERIES_PATTERN = re.compile(r"^[\w\- \/.#{}]+$", re.UNICODE) BRACED_PARAMS_PATTERN = re.compile(r"(\{[\w | #]+\})") +# Types that can be using in naming series fields +NAMING_SERIES_PART_TYPES = ( + int, + str, + datetime.datetime, + datetime.date, + datetime.time, + datetime.timedelta, +) + + class InvalidNamingSeriesError(frappe.ValidationError): pass @@ -298,6 +310,9 @@ def parse_naming_series( series_set = False today = now_datetime() for e in parts: + if not e: + continue + part = "" if e.startswith("#"): if not series_set: @@ -320,14 +335,16 @@ def parse_naming_series( part = frappe.defaults.get_user_default("fiscal_year") elif e.startswith("{") and doc: e = e.replace("{", "").replace("}", "") - part = (cstr(doc.get(e)) or "").strip() + part = doc.get(e) elif doc and doc.get(e): - part = (cstr(doc.get(e)) or "").strip() + part = doc.get(e) else: part = e if isinstance(part, str): name += part + elif isinstance(part, NAMING_SERIES_PART_TYPES): + name += cstr(part).strip() return name diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 5feaad6f9b..9f5b46a4d1 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -9,6 +9,7 @@ from frappe.model.naming import ( append_number_if_name_exists, determine_consecutive_week_number, getseries, + parse_naming_series, revert_series_if_last, ) from frappe.tests.utils import FrappeTestCase @@ -342,6 +343,32 @@ class TestNaming(FrappeTestCase): name.startswith("KOOH-on_update"), f"incorrect name generated {name}, missing field value" ) + def test_naming_with_empty_part(self): + # check naming with empty part (duplicate dots) + + webhook = frappe.new_doc("Webhook") + webhook.webhook_docevent = "on_update" + + series = "KOOH-..{webhook_docevent}.-.####" + + name = parse_naming_series(series, doc=webhook) + self.assertTrue( + name.startswith("KOOH-on_update"), f"incorrect name generated {name}, missing field value" + ) + + def test_naming_with_unsupported_part(self): + # check naming with empty part (duplicate dots) + + webhook = frappe.new_doc("Webhook") + webhook.webhook_docevent = {"dict": "not supported"} + + series = "KOOH-..{webhook_docevent}.-.####" + + name = parse_naming_series(series, doc=webhook) + self.assertTrue( + name.startswith("KOOH-"), f"incorrect name generated {name}, missing field value" + ) + def make_invalid_todo(): frappe.get_doc({"doctype": "ToDo", "description": "Test"}).insert(set_name="ToDo") From a8f86abbd8fa16c18eb3b621e90c5f77eb69d1d0 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 14 Jul 2022 14:09:00 +0530 Subject: [PATCH 139/201] refactor: Replaced blog's feedback with comment's comment_type='Like' (#17479) --- frappe/core/doctype/comment/comment.json | 14 +++- frappe/core/doctype/feedback/feedback.js | 8 -- frappe/core/doctype/feedback/feedback.json | 73 ------------------ frappe/core/doctype/feedback/feedback.py | 9 --- frappe/core/doctype/feedback/test_feedback.py | 42 ---------- frappe/patches.txt | 1 + .../v14_0/setup_likes_from_feedback.py | 30 ++++++++ .../templates/includes/comments/comment.html | 2 +- .../templates/includes/comments/comments.html | 7 +- .../templates/includes/comments/comments.py | 8 +- .../templates/includes/feedback/__init__.py | 0 .../templates/includes/feedback/feedback.py | 55 ------------- .../includes/likes}/__init__.py | 0 .../feedback.html => likes/likes.html} | 24 +++--- frappe/templates/includes/likes/likes.py | 77 +++++++++++++++++++ .../website/doctype/blog_post/blog_post.json | 18 ++--- frappe/website/doctype/blog_post/blog_post.py | 52 ++++++------- .../blog_post/templates/blog_post.html | 19 ++++- .../doctype/blog_post/test_blog_post.py | 23 ++++++ .../doctype/blog_settings/blog_settings.json | 18 ++--- .../doctype/blog_settings/blog_settings.py | 4 +- 21 files changed, 221 insertions(+), 263 deletions(-) delete mode 100644 frappe/core/doctype/feedback/feedback.js delete mode 100644 frappe/core/doctype/feedback/feedback.json delete mode 100644 frappe/core/doctype/feedback/feedback.py delete mode 100644 frappe/core/doctype/feedback/test_feedback.py create mode 100644 frappe/patches/v14_0/setup_likes_from_feedback.py delete mode 100644 frappe/templates/includes/feedback/__init__.py delete mode 100644 frappe/templates/includes/feedback/feedback.py rename frappe/{core/doctype/feedback => templates/includes/likes}/__init__.py (100%) rename frappe/templates/includes/{feedback/feedback.html => likes/likes.html} (78%) create mode 100644 frappe/templates/includes/likes/likes.py diff --git a/frappe/core/doctype/comment/comment.json b/frappe/core/doctype/comment/comment.json index fe465f46bd..9f27e7e7be 100644 --- a/frappe/core/doctype/comment/comment.json +++ b/frappe/core/doctype/comment/comment.json @@ -1,4 +1,5 @@ { + "actions": [], "creation": "2019-02-07 10:10:46.845678", "doctype": "DocType", "editable_grid": 1, @@ -17,7 +18,8 @@ "link_name", "reference_owner", "section_break_10", - "content" + "content", + "ip_address" ], "fields": [ { @@ -102,9 +104,16 @@ "ignore_xss_filter": 1, "in_list_view": 1, "label": "Content" + }, + { + "fieldname": "ip_address", + "fieldtype": "Data", + "hidden": 1, + "label": "IP Address" } ], - "modified": "2019-09-02 21:00:10.784787", + "links": [], + "modified": "2022-07-12 17:35:31.774137", "modified_by": "Administrator", "module": "Core", "name": "Comment", @@ -138,6 +147,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "comment_type", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/feedback/feedback.js b/frappe/core/doctype/feedback/feedback.js deleted file mode 100644 index 131f0e19d8..0000000000 --- a/frappe/core/doctype/feedback/feedback.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2021, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Feedback', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/feedback/feedback.json b/frappe/core/doctype/feedback/feedback.json deleted file mode 100644 index f8380cfda6..0000000000 --- a/frappe/core/doctype/feedback/feedback.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "actions": [], - "creation": "2021-06-03 19:02:55.328423", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "reference_name", - "column_break_3", - "like", - "ip_address" - ], - "fields": [ - { - "fieldname": "column_break_3", - "fieldtype": "Column Break" - }, - { - "fieldname": "reference_doctype", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Reference Document Type", - "options": "\nBlog Post" - }, - { - "fieldname": "reference_name", - "fieldtype": "Dynamic Link", - "in_list_view": 1, - "label": "Reference Name", - "options": "reference_doctype", - "reqd": 1 - }, - { - "fieldname": "ip_address", - "fieldtype": "Data", - "hidden": 1, - "label": "IP Address", - "read_only": 1 - }, - { - "default": "0", - "fieldname": "like", - "fieldtype": "Check", - "label": "Like" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2021-11-10 20:53:21.255593", - "modified_by": "Administrator", - "module": "Core", - "name": "Feedback", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "reference_name", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/feedback/feedback.py b/frappe/core/doctype/feedback/feedback.py deleted file mode 100644 index c616787e4b..0000000000 --- a/frappe/core/doctype/feedback/feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class Feedback(Document): - pass diff --git a/frappe/core/doctype/feedback/test_feedback.py b/frappe/core/doctype/feedback/test_feedback.py deleted file mode 100644 index e8e29e75ae..0000000000 --- a/frappe/core/doctype/feedback/test_feedback.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE - -import unittest - -import frappe - - -class TestFeedback(unittest.TestCase): - def tearDown(self): - frappe.form_dict.reference_doctype = None - frappe.form_dict.reference_name = None - frappe.form_dict.like = None - frappe.local.request_ip = None - - def test_feedback_creation_updation(self): - from frappe.website.doctype.blog_post.test_blog_post import make_test_blog - - test_blog = make_test_blog() - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - from frappe.templates.includes.feedback.feedback import give_feedback - - frappe.form_dict.reference_doctype = "Blog Post" - frappe.form_dict.reference_name = test_blog.name - frappe.form_dict.like = True - frappe.local.request_ip = "127.0.0.1" - - feedback = give_feedback() - - self.assertEqual(feedback.like, True) - - frappe.form_dict.like = False - - updated_feedback = give_feedback() - - self.assertEqual(updated_feedback.like, False) - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - test_blog.delete() diff --git a/frappe/patches.txt b/frappe/patches.txt index 425468f06c..d9b827931c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -192,6 +192,7 @@ frappe.patches.v14_0.reset_creation_datetime frappe.patches.v14_0.remove_is_first_startup frappe.patches.v14_0.clear_long_pending_stale_logs frappe.patches.v14_0.log_settings_migration +frappe.patches.v14_0.setup_likes_from_feedback [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v14_0/setup_likes_from_feedback.py b/frappe/patches/v14_0/setup_likes_from_feedback.py new file mode 100644 index 0000000000..d88f69ce4b --- /dev/null +++ b/frappe/patches/v14_0/setup_likes_from_feedback.py @@ -0,0 +1,30 @@ +import frappe + + +def execute(): + frappe.reload_doctype("Comment") + + if frappe.db.count("Feedback") > 20000: + frappe.db.auto_commit_on_many_writes = True + + for feedback in frappe.get_all("Feedback", fields=["*"]): + if feedback.like: + new_comment = frappe.new_doc("Comment") + new_comment.comment_type = "Like" + new_comment.comment_email = feedback.owner + new_comment.content = "Liked by: " + feedback.owner + new_comment.reference_doctype = feedback.reference_doctype + new_comment.reference_name = feedback.reference_name + new_comment.creation = feedback.creation + new_comment.modified = feedback.modified + new_comment.owner = feedback.owner + new_comment.modified_by = feedback.modified_by + new_comment.ip_address = feedback.ip_address + new_comment.db_insert() + + if frappe.db.auto_commit_on_many_writes: + frappe.db.auto_commit_on_many_writes = False + + # clean up + frappe.db.delete("Feedback") + frappe.db.commit() diff --git a/frappe/templates/includes/comments/comment.html b/frappe/templates/includes/comments/comment.html index 4713ee498d..64de9e5943 100644 --- a/frappe/templates/includes/comments/comment.html +++ b/frappe/templates/includes/comments/comment.html @@ -13,6 +13,6 @@ {{ frappe.utils.pretty_date(comment.creation) }} -
{{ frappe.utils.strip_html(comment.content) | markdown }}
+
{{ frappe.utils.strip_html(comment.content) | markdown }}
\ No newline at end of file diff --git a/frappe/templates/includes/comments/comments.html b/frappe/templates/includes/comments/comments.html index 0007f56934..63ec6a21bd 100644 --- a/frappe/templates/includes/comments/comments.html +++ b/frappe/templates/includes/comments/comments.html @@ -57,7 +57,7 @@ {% endblock %} diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index d202d15642..3ea447d90c 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -152,6 +152,29 @@ class TestBlogPost(FrappeTestCase): frappe.delete_doc("Blog Post", blog.name) frappe.delete_doc("Blog Category", blog.blog_category) + def test_like_dislike(self): + test_blog = make_test_blog() + + frappe.db.delete("Comment", {"comment_type": "Like", "reference_doctype": "Blog Post"}) + + from frappe.templates.includes.likes.likes import like + + frappe.form_dict.reference_doctype = "Blog Post" + frappe.form_dict.reference_name = test_blog.name + frappe.form_dict.like = True + frappe.local.request_ip = "127.0.0.1" + + liked = like() + self.assertEqual(liked, True) + + frappe.form_dict.like = False + + disliked = like() + self.assertEqual(disliked, False) + + frappe.db.delete("Comment", {"comment_type": "Like", "reference_doctype": "Blog Post"}) + test_blog.delete() + def scrub(text): return WebsiteGenerator.scrub(None, text) diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index aed1e77969..4e89af5c8e 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -19,7 +19,7 @@ "cta_label", "cta_url", "section_break_12", - "feedback_limit", + "like_limit", "column_break_14", "comment_limit" ], @@ -89,13 +89,6 @@ "fieldname": "section_break_12", "fieldtype": "Section Break" }, - { - "default": "5", - "description": "Feedback limit per hour", - "fieldname": "feedback_limit", - "fieldtype": "Int", - "label": "Feedback limit" - }, { "default": "5", "description": "Comment limit per hour", @@ -118,13 +111,20 @@ "fieldname": "browse_by_category", "fieldtype": "Check", "label": "Browse by category" + }, + { + "default": "5", + "description": "Like limit per hour", + "fieldname": "like_limit", + "fieldtype": "Int", + "label": "Like limit" } ], "icon": "fa fa-cog", "idx": 1, "issingle": 1, "links": [], - "modified": "2021-12-20 13:40:32.312459", + "modified": "2022-07-12 17:45:49.108398", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", diff --git a/frappe/website/doctype/blog_settings/blog_settings.py b/frappe/website/doctype/blog_settings/blog_settings.py index ed22f64fd7..6b1d7b6323 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.py +++ b/frappe/website/doctype/blog_settings/blog_settings.py @@ -15,8 +15,8 @@ class BlogSettings(Document): clear_cache("writers") -def get_feedback_limit(): - return frappe.db.get_single_value("Blog Settings", "feedback_limit") or 5 +def get_like_limit(): + return frappe.db.get_single_value("Blog Settings", "like_limit") or 5 def get_comment_limit(): From 0a41c4051c2f8bb63bce2b11f45cf105200f0daf Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 14 Jul 2022 18:01:47 +0530 Subject: [PATCH 140/201] fix: Do not relay email to standard users Co-authored-by: Ritwik Puri --- frappe/core/doctype/communication/mixins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 695b8bebae..af3b0a4661 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -73,7 +73,8 @@ class CommunicationEmailMixin: if include_sender: cc.append(self.sender_mailid) if is_inbound_mail_communcation: - cc.append(self.get_owner()) + if (doc_owner := self.get_owner()) not in frappe.STANDARD_USERS: + cc.append(doc_owner) cc = set(cc) - {self.sender_mailid} cc.update(self.get_assignees()) From 1f0e019e89a887bb14c59d3566b9fdd9d4659b2b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 14 Jul 2022 19:02:19 +0530 Subject: [PATCH 141/201] fix(UX): correct message for empty prepared report (#17517) --- frappe/public/js/frappe/views/reports/query_report.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 80b251e5ec..e15cc339ae 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -623,6 +623,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { if (data.prepared_report) { this.prepared_report = true; + this.prepared_report_document = data.doc // If query_string contains prepared_report_name then set filters // to match the mentioned prepared report doc and disable editing if (query_params.prepared_report_name) { @@ -1800,7 +1801,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { } toggle_nothing_to_show(flag) { - let message = this.prepared_report + let message = (this.prepared_report && !this.prepared_report_document) ? __('This is a background report. Please set the appropriate filters and then generate a new one.') : this.get_no_result_message(); From c200f5b3ae8431d03ccdf18c0c3f53dd8782f2dd Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Jul 2022 11:59:04 +0530 Subject: [PATCH 142/201] ci: check build requirement before setting up python [skip ci] --- .github/workflows/patch-mariadb-tests.yml | 24 ++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 73e0dda5de..1e21ae8549 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -30,17 +30,6 @@ jobs: - name: Clone uses: actions/checkout@v3 - - name: Setup Python - uses: "gabrielfalcao/pyenv-action@v10" - with: - versions: 3.10:latest, 3.7:latest - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: 14 - check-latest: true - - name: Check if build should be run id: check-build run: | @@ -50,6 +39,19 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} + - name: Setup Python + if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: "gabrielfalcao/pyenv-action@v10" + with: + versions: 3.10:latest, 3.7:latest + + - name: Setup Node + if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: actions/setup-node@v3 + with: + node-version: 14 + check-latest: true + - name: Add to Hosts if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts From ddbbc0ef8df16f57a921f2e0f2c634535448bd80 Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 15 Jul 2022 14:23:05 +0530 Subject: [PATCH 143/201] fix: add default system manager role when changing from child to non-child table doctype --- frappe/core/doctype/doctype/doctype.js | 12 +++++++++--- frappe/core/doctype/doctype/doctype.py | 7 +++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 514e3a9455..3a9b1f63dc 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -46,9 +46,7 @@ frappe.ui.form.on('DocType', { } if(frm.is_new()) { - if (!(frm.doc.permissions && frm.doc.permissions.length)) { - frm.add_child('permissions', {role: 'System Manager'}); - } + frm.events.set_default_permission(frm); } else { frm.toggle_enable("engine", 0); } @@ -65,6 +63,14 @@ frappe.ui.form.on('DocType', { if (frm.doc.istable && frm.is_new()) { frm.set_value('autoname', 'autoincrement'); frm.set_value('allow_rename', 0); + } else if (!frm.doc.istable && !frm.is_new()) { + frm.events.set_default_permission(frm); + } + }, + + set_default_permission: (frm) => { + if (!(frm.doc.permissions && frm.doc.permissions.length)) { + frm.add_child('permissions', {role: 'System Manager'}); } }, }); diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index dbbbbc521a..2b28373384 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -181,10 +181,6 @@ class DocType(Document): ) ) - def after_insert(self): - # clear user cache so that on the next reload this doctype is included in boot - clear_user_cache(frappe.session.user) - def set_defaults_for_single_and_table(self): if self.issingle: self.allow_import = 0 @@ -412,6 +408,9 @@ class DocType(Document): delete_notification_count_for(doctype=self.name) frappe.clear_cache(doctype=self.name) + # clear user cache so that on the next reload this doctype is included in boot + clear_user_cache(frappe.session.user) + if not frappe.flags.in_install and hasattr(self, "before_update"): self.sync_global_search() From 5738676b461128d77c016c7125ff0bceb2d007ca Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 15 Jul 2022 19:23:59 +0530 Subject: [PATCH 144/201] fix: use on_update hook instead of after_save and after_insert after_save hook doesn;t exist so using that triggers nothing --- frappe/hooks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/hooks.py b/frappe/hooks.py index a337d8e0d3..66820ecd0f 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -180,12 +180,10 @@ doc_events = { "on_update": "frappe.integrations.doctype.google_contacts.google_contacts.update_contacts_to_google_contacts", }, "DocType": { - "after_insert": "frappe.cache_manager.build_domain_restriced_doctype_cache", - "after_save": "frappe.cache_manager.build_domain_restriced_doctype_cache", + "on_update": "frappe.cache_manager.build_domain_restriced_doctype_cache", }, "Page": { - "after_insert": "frappe.cache_manager.build_domain_restriced_page_cache", - "after_save": "frappe.cache_manager.build_domain_restriced_page_cache", + "on_update": "frappe.cache_manager.build_domain_restriced_page_cache", }, } From 454766c26379d36ecf3c7cfe7b30fcdc9b6e8a34 Mon Sep 17 00:00:00 2001 From: Amin Ahmed Date: Fri, 15 Jul 2022 21:17:34 +0300 Subject: [PATCH 145/201] Update datepicker_i18n.js --- frappe/public/js/frappe/form/controls/datepicker_i18n.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/datepicker_i18n.js b/frappe/public/js/frappe/form/controls/datepicker_i18n.js index f010325c2e..a5b825072d 100644 --- a/frappe/public/js/frappe/form/controls/datepicker_i18n.js +++ b/frappe/public/js/frappe/form/controls/datepicker_i18n.js @@ -22,10 +22,10 @@ import "air-datepicker/dist/js/i18n/datepicker.zh.js"; months: ['يناير', 'فبراير', 'مارس', 'أبريل', 'مايو', 'يونيو', 'يوليو', 'أغسطس', 'سبتمبر', 'اكتوبر', 'نوفمبر', 'ديسمبر'], monthsShort: ['يناير', 'فبراير', 'مارس', 'أبريل', 'مايو', 'يونيو', 'يوليو', 'أغسطس', 'سبتمبر', 'اكتوبر', 'نوفمبر', 'ديسمبر'], today: 'اليوم', - clear: 'Clear', + clear: 'حذف', dateFormat: 'dd/mm/yyyy', timeFormat: 'hh:ii aa', - firstDay: 0 + firstDay: 6 }; })(jQuery); From 55fb8acafa3e63d2982850331f5cd8c6976d937d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 17 Jul 2022 20:07:05 +0530 Subject: [PATCH 146/201] perf(DX): add watchdog as developer dependency Werkzeug reloader is right now using an inefficient `stat` based reloader which is horrible on large codebases with low-powered devices. Difference: - `stat` based reloader basically checks each and every file if they have changed or not. - watchdog subscribes to platform specific change events on kernel (like kqueue, fsevents or inotify ) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a1706ac33e..5eeb6f46dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,3 +109,4 @@ coverage = "~=6.4.1" Faker = "~=13.12.1" pyngrok = "~=5.0.5" unittest-xml-reporting = "~=3.0.4" +watchdog = "~=2.1.9" From 8edae2ce098f6a7fc08034726570aba9d221aea7 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 17 Jul 2022 10:29:57 +0530 Subject: [PATCH 147/201] fix: encrypt access_token when setting in db after refreshing access_token --- frappe/email/oauth.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 46d1565275..9adeb2f9de 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -102,12 +102,18 @@ class Oauth: def _refresh_access_token(self) -> str: """Refreshes access token via calling `refresh_access_token` method of oauth service object""" service_obj = self._get_service_object() - access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None) + access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token") + + if access_token: + # set the new access token in db + frappe.db.set_value( + "Email Account", + self.email_account, + "access_token", + encrypt(access_token), + update_modified=False, + ) - # set the new access token in db - frappe.db.set_value( - "Email Account", self.email_account, "access_token", access_token, update_modified=False - ) return access_token def _get_service_object(self): From f679dc3fdd64e8515a03f0d2d855a24bf41cd6b1 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 17 Jul 2022 11:14:01 +0530 Subject: [PATCH 148/201] fix(security): restrict the god google callback the common google callback can be used to trigger any method in the whole codebase restrict it by only allowing domain specific callback method and raise an error if the domain is not found --- frappe/email/oauth.py | 1 - .../google_contacts/google_contacts.py | 1 - .../doctype/google_drive/google_drive.py | 1 - frappe/integrations/google_oauth.py | 30 ++++++++++++++----- .../website_settings/google_indexing.py | 1 - 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py index 9adeb2f9de..89b6df15d8 100644 --- a/frappe/email/oauth.py +++ b/frappe/email/oauth.py @@ -150,7 +150,6 @@ def authorize_google_access(email_account, doctype: str = "Email Account", code: if not code: return oauth_obj.get_authentication_url( { - "method": "frappe.email.oauth.authorize_google_access", "redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}", "success_query_param": "successful_authorization=1", "email_account": email_account, diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index c1f445b599..9a20d5e905 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -45,7 +45,6 @@ def authorize_access(g_contact, reauthorize=False, code=None): if not oauth_code or reauthorize: return oauth_obj.get_authentication_url( { - "method": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", "g_contact": g_contact, "redirect": f"/app/Form/{quote('Google Contacts')}/{quote(g_contact)}", }, diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index 62100ae7c5..6b1e03eccb 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -57,7 +57,6 @@ def authorize_access(reauthorize=False, code=None): frappe.db.set_value("Google Drive", None, "backup_folder_id", "") return oauth_obj.get_authentication_url( { - "method": "frappe.integrations.doctype.google_drive.google_drive.authorize_access", "redirect": f"/app/Form/{quote('Google Drive')}", }, ) diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index edce63493e..1d5ed3723f 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -19,6 +19,12 @@ _SERVICES = { "drive": ("drive", "v3"), "indexing": ("indexing", "v3"), } +_DOMAIN_CALLBACK_METHODS = { + "mail": "frappe.email.oauth.authorize_google_access", + "contacts": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access", + "drive": "frappe.integrations.doctype.google_drive.google_drive.authorize_access", + "indexing": "frappe.website.doctype.website_settings.google_indexing.authorize_access", +} class GoogleAuthenticationError(Exception): @@ -100,6 +106,7 @@ class GoogleOAuth: :param state: [optional] dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc) """ + state.update({"domain": self.domain}) state = json.dumps(state) callback_url = get_request_site_address(True) + CALLBACK_METHOD @@ -175,13 +182,22 @@ def callback(state: str, code: str = None, error: str = None) -> None: failure_query_param = state.pop("failure_query_param", "") if not error: - state.update({"code": code}) - frappe.get_attr(state.pop("method"))(**state) + if (domain := state.pop("domain")) in _DOMAIN_CALLBACK_METHODS: + state.update({"code": code}) + frappe.get_attr(_DOMAIN_CALLBACK_METHODS[domain])(**state) - # GET request, hence using commit to persist changes - frappe.db.commit() # nosemgrep - - redirect = f"{redirect}?{failure_query_param if error else success_query_param}" + # GET request, hence using commit to persist changes + frappe.db.commit() # nosemgrep + else: + return frappe.respond_as_web_page( + "Invalid Google Callback", + "The callback domain provided is not valid for Google Authentication", + http_status_code=400, + indicator_color="red", + width=640, + ) frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = redirect + frappe.local.response[ + "location" + ] = f"{redirect}?{failure_query_param if error else success_query_param}" diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 4f67949f86..9dbd415b02 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -26,7 +26,6 @@ def authorize_access(reauthorize=False, code=None): if not oauth_code or reauthorize: return oauth_obj.get_authentication_url( { - "method": "frappe.website.doctype.website_settings.google_indexing.authorize_access", "redirect": f"/app/Form/{quote('Website Settings')}", }, ) From 404e9ab4e60661367ba17c14dbf9b7431dbc9fc7 Mon Sep 17 00:00:00 2001 From: phot0n Date: Sun, 17 Jul 2022 14:52:36 +0530 Subject: [PATCH 149/201] minor: add auto fetch for updating used_oauth field in User Email doctype (when added a user email manually) --- frappe/core/doctype/user/user.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 41b7e7fb38..05a1102c03 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -287,6 +287,18 @@ frappe.ui.form.on('User', { } }); + +frappe.ui.form.on('User Email', { + email_account(frm, cdt, cdn) { + let child_row = locals[cdt][cdn]; + frappe.model.get_value("Email Account", child_row.email_account, "auth_method", (value) => { + child_row.used_oauth = value.auth_method === "OAuth"; + frm.refresh_field("user_emails", cdn, "used_oauth"); + }); + } +}); + + function has_access_to_edit_user() { return has_common(frappe.user_roles, get_roles_for_editing_user()); } From 1fd47f1c10b9c9adf6e56d35bc1e8be87dc6baf7 Mon Sep 17 00:00:00 2001 From: phot0n Date: Wed, 13 Jul 2022 13:07:32 +0530 Subject: [PATCH 150/201] chore: remove unused no_remaining field from email account doctype --- .../core/doctype/communication/test_communication.py | 1 - frappe/email/doctype/email_account/email_account.js | 2 -- .../email/doctype/email_account/email_account.json | 12 +----------- frappe/email/doctype/email_account/test_records.json | 1 - 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index 77f83b7f91..2621fe3c6b 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -358,7 +358,6 @@ def create_email_account(): "send_notification_to": "test_comm@example.com", "pop3_server": "pop.test.example.com", "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], - "no_remaining": "0", "enable_automatic_linking": 1, } ).insert(ignore_permissions=True) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index d22009963d..3faf83800d 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -123,8 +123,6 @@ frappe.ui.form.on("Email Account", { }, enable_incoming: function(frm) { - frm.doc.no_remaining = null; //perform full sync - //frm.set_df_property("append_to", "reqd", frm.doc.enable_incoming); frm.trigger("warn_autoreply_on_incoming"); }, diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index ecb5af7378..9395526fe4 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -70,7 +70,6 @@ "brand_logo", "uidvalidity", "uidnext", - "no_remaining", "no_failed" ], "fields": [ @@ -472,15 +471,6 @@ "label": "UIDNEXT", "no_copy": 1 }, - { - "fieldname": "no_remaining", - "fieldtype": "Data", - "hidden": 1, - "hide_days": 1, - "hide_seconds": 1, - "label": "No of emails remaining to be synced", - "no_copy": 1 - }, { "fieldname": "no_failed", "fieldtype": "Int", @@ -616,7 +606,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-07-11 18:34:06.945668", + "modified": "2022-07-13 13:05:45.445572", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index 66eb5a9b2e..2e204e5277 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -18,7 +18,6 @@ "unreplied_for_mins": 20, "send_notification_to": "test_unreplied@example.com", "pop3_server": "pop.test.example.com", - "no_remaining":"0", "append_to": "ToDo", "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}], "track_email_status": 1 From 29c855b02807107813676bc4c2db49e4944462a0 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Jul 2022 15:10:49 +0530 Subject: [PATCH 151/201] fix: db.get_value -> db.get_single_value (#17531) db.get_value for singles returns string type always, this is confusing behaviour, db.get_single_value should be used instead. semgrep rule: https://github.com/frappe/semgrep-rules/pull/16 --- .../core/doctype/sms_settings/sms_settings.py | 2 +- .../system_settings/system_settings.py | 2 +- frappe/core/doctype/user/user.py | 6 ++--- .../dropbox_settings/dropbox_settings.py | 6 ++--- .../doctype/google_drive/google_drive.py | 2 +- .../oauth_provider_settings.py | 4 +++- .../s3_backup_settings/s3_backup_settings.py | 4 ++-- .../patches/v11_0/set_dropbox_file_backup.py | 2 +- .../pages/integrations/razorpay_checkout.py | 2 +- frappe/twofactor.py | 22 ++++++++----------- frappe/utils/data.py | 2 +- .../doctype/web_page_view/web_page_view.py | 2 +- .../website_settings/google_indexing.py | 2 +- frappe/www/contact.py | 2 +- 14 files changed, 29 insertions(+), 31 deletions(-) diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index 686890514a..0a5536eb9b 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -63,7 +63,7 @@ def send_sms(receiver_list, msg, sender_name="", success_msg=True): "success_msg": success_msg, } - if frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): send_via_gateway(arg) else: msgprint(_("Please Update SMS Settings")) diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index e4d36b7fc7..fbdc188742 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -28,7 +28,7 @@ class SystemSettings(Document): if self.enable_two_factor_auth: if self.two_factor_method == "SMS": - if not frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if not frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): frappe.throw( _("Please setup SMS before setting it as an authentication method, via SMS Settings") ) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 12a48afe7e..6d0de186a5 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -611,10 +611,10 @@ class User(Document): """ login_with_mobile = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number") + frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number") ) login_with_username = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name") + frappe.db.get_single_value("System Settings", "allow_login_using_user_name") ) or_filters = [{"name": user_name}] @@ -861,7 +861,7 @@ def sign_up(email, full_name, redirect_to): user.insert() # set default signup role as per Portal Settings - default_role = frappe.db.get_value("Portal Settings", None, "default_role") + default_role = frappe.db.get_single_value("Portal Settings", "default_role") if default_role: user.add_roles(default_role) diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py index 50c5fa8fe6..dc9db2ccda 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py @@ -62,21 +62,21 @@ def take_backups_weekly(): def take_backups_if(freq): - if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq: + if frappe.db.get_single_value("Dropbox Settings", "backup_frequency") == freq: take_backup_to_dropbox() def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): did_not_upload, error_log = [], [] try: - if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")): + if cint(frappe.db.get_single_value("Dropbox Settings", "enabled")): validate_file_size() did_not_upload, error_log = backup_to_dropbox(upload_db_backup) if did_not_upload: raise Exception - if cint(frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup")): + if cint(frappe.db.get_single_value("Dropbox Settings", "send_email_for_successful_backup")): send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to") except JobTimeoutException: if retry_count < 2: diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index 62100ae7c5..c24d797086 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -48,7 +48,7 @@ def authorize_access(reauthorize=False, code=None): """ oauth_code = ( - frappe.db.get_value("Google Drive", "Google Drive", "authorization_code") if not code else code + frappe.db.get_single_value("Google Drive", "authorization_code") if not code else code ) oauth_obj = GoogleOAuth("drive") diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py index 984382df9d..5a918db587 100644 --- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py +++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py @@ -14,7 +14,9 @@ def get_oauth_settings(): """Returns oauth settings""" out = frappe._dict( { - "skip_authorization": frappe.db.get_value("OAuth Provider Settings", None, "skip_authorization") + "skip_authorization": frappe.db.get_single_value( + "OAuth Provider Settings", "skip_authorization" + ) } ) diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index 1c2d39be10..568ff71b00 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -76,8 +76,8 @@ def take_backups_monthly(): def take_backups_if(freq): - if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")): - if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq: + if cint(frappe.db.get_single_value("S3 Backup Settings", "enabled")): + if frappe.db.get_single_value("S3 Backup Settings", "frequency") == freq: take_backups_s3() diff --git a/frappe/patches/v11_0/set_dropbox_file_backup.py b/frappe/patches/v11_0/set_dropbox_file_backup.py index c9dec31414..396491e8b3 100644 --- a/frappe/patches/v11_0/set_dropbox_file_backup.py +++ b/frappe/patches/v11_0/set_dropbox_file_backup.py @@ -4,6 +4,6 @@ from frappe.utils import cint def execute(): frappe.reload_doctype("Dropbox Settings") - check_dropbox_enabled = cint(frappe.db.get_value("Dropbox Settings", None, "enabled")) + check_dropbox_enabled = cint(frappe.db.get_single_value("Dropbox Settings", "enabled")) if check_dropbox_enabled == 1: frappe.db.set_value("Dropbox Settings", None, "file_backup", 1) diff --git a/frappe/templates/pages/integrations/razorpay_checkout.py b/frappe/templates/pages/integrations/razorpay_checkout.py index b4f9e74a03..d0e77f6d8a 100644 --- a/frappe/templates/pages/integrations/razorpay_checkout.py +++ b/frappe/templates/pages/integrations/razorpay_checkout.py @@ -51,7 +51,7 @@ def get_context(context): def get_api_key(): - api_key = frappe.db.get_value("Razorpay Settings", None, "api_key") + api_key = frappe.db.get_single_value("Razorpay Settings", "api_key") if cint(frappe.form_dict.get("use_sandbox")): api_key = frappe.conf.sandbox_api_key diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 26fc3ad619..528e7f8c8b 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -27,10 +27,10 @@ def toggle_two_factor_auth(state, roles=None): def two_factor_is_enabled(user=None): """Returns True if 2FA is enabled.""" - enabled = int(frappe.db.get_value("System Settings", None, "enable_two_factor_auth") or 0) + enabled = int(frappe.db.get_single_value("System Settings", "enable_two_factor_auth") or 0) if enabled: bypass_two_factor_auth = int( - frappe.db.get_value("System Settings", None, "bypass_2fa_for_retricted_ip_users") or 0 + frappe.db.get_single_value("System Settings", "bypass_2fa_for_retricted_ip_users") or 0 ) if bypass_two_factor_auth and user: user_doc = frappe.get_doc("User", user) @@ -127,7 +127,7 @@ def get_otpsecret_for_(user): def get_verification_method(): - return frappe.db.get_value("System Settings", None, "two_factor_method") + return frappe.db.get_single_value("System Settings", "two_factor_method") def confirm_otp_token(login_manager, otp=None, tmp_id=None): @@ -173,7 +173,7 @@ def confirm_otp_token(login_manager, otp=None, tmp_id=None): def get_verification_obj(user, token, otp_secret): - otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + otp_issuer = frappe.db.get_single_value("System Settings", "otp_issuer_name") verification_method = get_verification_method() verification_obj = None if verification_method == "SMS": @@ -249,7 +249,7 @@ def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"): def get_email_subject_for_2fa(kwargs_dict): """Get email subject for 2fa.""" subject_template = _("Login Verification Code from {}").format( - frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + frappe.db.get_single_value("System Settings", "otp_issuer_name") ) subject = frappe.render_template(subject_template, kwargs_dict) return subject @@ -269,7 +269,7 @@ def get_email_body_for_2fa(kwargs_dict): def get_email_subject_for_qr_code(kwargs_dict): """Get QRCode email subject.""" subject_template = _("One Time Password (OTP) Registration Code from {}").format( - frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + frappe.db.get_single_value("System Settings", "otp_issuer_name") ) subject = frappe.render_template(subject_template, kwargs_dict) return subject @@ -289,9 +289,7 @@ def get_link_for_qrcode(user, totp_uri): key = frappe.generate_hash(length=20) key_user = f"{key}_user" key_uri = f"{key}_uri" - lifespan = ( - int(frappe.db.get_value("System Settings", "System Settings", "lifespan_qrcode_image")) or 240 - ) + lifespan = int(frappe.db.get_single_value("System Settings", "lifespan_qrcode_image")) or 240 frappe.cache().set_value(key_uri, totp_uri, expires_in_sec=lifespan) frappe.cache().set_value(key_user, user, expires_in_sec=lifespan) return get_url(f"/qrcode?k={key}") @@ -447,9 +445,7 @@ def should_remove_barcode_image(barcode): """Check if it's time to delete barcode image from server.""" if isinstance(barcode, str): barcode = frappe.get_doc("File", barcode) - lifespan = ( - frappe.db.get_value("System Settings", "System Settings", "lifespan_qrcode_image") or 240 - ) + lifespan = frappe.db.get_single_value("System Settings", "lifespan_qrcode_image") or 240 if time_diff_in_seconds(get_datetime(), barcode.creation) > int(lifespan): return True return False @@ -464,7 +460,7 @@ def reset_otp_secret(user): if frappe.session.user != user: frappe.only_for("System Manager", message=True) - otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + otp_issuer = frappe.db.get_single_value("System Settings", "otp_issuer_name") user_email = frappe.db.get_value("User", user, "email") frappe.defaults.clear_default(user + "_otplogin") diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 81ca143682..8a970e57cc 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1545,7 +1545,7 @@ def get_url(uri: str | None = None, full_address: bool = False) -> str: host_name = protocol + frappe.local.site else: - host_name = frappe.db.get_value("Website Settings", "Website Settings", "subdomain") + host_name = frappe.db.get_single_value("Website Settings", "subdomain") if not host_name: host_name = "http://localhost" diff --git a/frappe/website/doctype/web_page_view/web_page_view.py b/frappe/website/doctype/web_page_view/web_page_view.py index 7417f2d290..c3aef62584 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.py +++ b/frappe/website/doctype/web_page_view/web_page_view.py @@ -49,4 +49,4 @@ def get_page_view_count(path): def is_tracking_enabled(): - return frappe.db.get_value("Website Settings", "Website Settings", "enable_view_tracking") + return frappe.db.get_single_value("Website Settings", "enable_view_tracking") diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 4f67949f86..fbd5fa7820 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -16,7 +16,7 @@ def authorize_access(reauthorize=False, code=None): """If no Authorization code get it from Google and then request for Refresh Token.""" oauth_code = ( - frappe.db.get_value("Website Settings", "Website Settings", "indexing_authorization_code") + frappe.db.get_single_value("Website Settings", "indexing_authorization_code") if not code else code ) diff --git a/frappe/www/contact.py b/frappe/www/contact.py index 11be5e86da..cf26539ff4 100644 --- a/frappe/www/contact.py +++ b/frappe/www/contact.py @@ -51,7 +51,7 @@ def send_message(subject="Website Query", message="", sender=""): return # send email - forward_to_email = frappe.db.get_value("Contact Us Settings", None, "forward_to_email") + forward_to_email = frappe.db.get_single_value("Contact Us Settings", "forward_to_email") if forward_to_email: frappe.sendmail(recipients=forward_to_email, sender=sender, content=message, subject=subject) From b73899e99dbd7ca224c8ae4212322459abb20787 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Jul 2022 19:56:15 +0530 Subject: [PATCH 152/201] feat: frappe.utils.debug.watch_property to debug JS This util adds a watcher on any JS object for purpose of debugging, whenever the property changes a console trace is generated. A custom callback function can also be passed if required. Usage: ```javascript filters = {} frappe.utils.debug.watch_property(filters, "company"); // any changes to company key on filters object from now on will print a console trace ``` only for debugging, make sure to not commit it in codebase! :) debug --- frappe/public/js/frappe/utils/utils.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 34374d7c85..877add95bf 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1537,4 +1537,29 @@ Object.assign(frappe.utils, { is_current_user(user) { return user === frappe.session.user; }, + + debug: { + watch_property(obj, prop, callback=console.trace) { + if (!frappe.boot.developer_mode) { + return; + } + console.warn("Adding property watcher, make sure to remove it after debugging."); + + // Adapted from https://stackoverflow.com/a/11658693 + // Reused under CC-BY-SA 4.0 + // changes: variable names are changed for consistency with our codebase + const private_prop = "$_" + prop + "_$"; + obj[private_prop] = obj[prop]; + + Object.defineProperty(obj, prop, { + get: function() { + return obj[private_prop]; + }, + set: function(value) { + callback(); + obj[private_prop] = value; + }, + }); + }, + } }); From 1a7a21bbe5a376084633b80aa412cbb3d8b760b4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Jul 2022 19:40:58 +0530 Subject: [PATCH 153/201] fix(UX)!: respect no filters set by user Currently if user resets a filter and comes back to same view default filters get populated. This is annoying behaviour especially since this is already saved in user setting. Fix: Always respect user's preference, if their last choice was no filters then load view without filters. --- frappe/public/js/frappe/list/base_list.js | 15 --------------- frappe/public/js/frappe/list/list_view.js | 5 +---- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index bbee90048b..e5272ccd91 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -764,10 +764,6 @@ class FilterArea { const doctype_fields = this.list_view.meta.fields; const title_field = this.list_view.meta.title_field; - const has_existing_filters = ( - this.list_view.filters - && this.list_view.filters.length > 0 - ); fields = fields.concat( doctype_fields @@ -803,23 +799,12 @@ class FilterArea { } } - let default_value; - - if (fieldtype === "Link" && !has_existing_filters) { - default_value = frappe.defaults.get_user_default(options); - } - - if (["__default", "__global"].includes(default_value)) { - default_value = null; - } - return { fieldtype: fieldtype, label: __(df.label), options: options, fieldname: df.fieldname, condition: condition, - default: default_value, onchange: () => this.refresh_list_view(), ignore_link_validation: fieldtype === "Dynamic Link", is_filter: 1, diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 94a3c29b27..cbeda50e53 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -87,10 +87,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.menu_items = this.menu_items.concat(this.get_menu_items()); // set filters from view_user_settings or list_settings - if ( - this.view_user_settings.filters && - this.view_user_settings.filters.length - ) { + if (Array.isArray(this.view_user_settings.filters)) { // Priority 1: view_user_settings const saved_filters = this.view_user_settings.filters; this.filters = this.validate_filters(saved_filters); From a98e47150f7f81ce4524e3e9d414d7ec4205f0d4 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Jul 2022 16:26:00 +0530 Subject: [PATCH 154/201] feat(tiny): frappe.log -> frappe.log for server scripts This it already whitelisted but in global scope. [skip ci] --- frappe/utils/safe_exec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 03f5d041ce..8e983f11b2 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -152,6 +152,7 @@ def get_safe_globals(): enqueue=safe_enqueue, sanitize_html=frappe.utils.sanitize_html, log_error=frappe.log_error, + log=frappe.log, db=NamespaceDict( get_list=frappe.get_list, get_all=frappe.get_all, From 59fff76cb34820d760d56b6b488e9a802dd2b308 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Jul 2022 16:47:36 +0530 Subject: [PATCH 155/201] fix(dx): word wrap in script diffview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helpful for devs who write >120 character long code. 💩 [skip ci] --- frappe/public/js/frappe/utils/diffview.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/diffview.js b/frappe/public/js/frappe/utils/diffview.js index a898a318a1..a326fd74bc 100644 --- a/frappe/public/js/frappe/utils/diffview.js +++ b/frappe/public/js/frappe/utils/diffview.js @@ -89,7 +89,7 @@ frappe.ui.DiffView = class DiffView { } else if (line.startsWith("-")) { line_class = "delete"; } - html += `
${line}
`; + html += `
${line}
`; }); return `
${html}
`; } From a588879094ec997cd1f134aac72e64bb409ab05d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 18 Jul 2022 17:30:04 +0530 Subject: [PATCH 156/201] refactor(minor): LDAP Settings Test Suite * Re-write blocks for better readability * De-indent everything * Add typing, etc --- .../ldap_settings/test_ldap_settings.py | 308 ++++++------------ 1 file changed, 101 insertions(+), 207 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index f53b5291b3..8587c6c656 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -1,15 +1,16 @@ -# Copyright (c) 2019, Frappe Technologies and Contributors +# Copyright (c) 2022, Frappe Technologies and Contributors # License: MIT. See LICENSE +import contextlib import functools import os import ssl -import unittest -from unittest import mock +from unittest import TestCase, mock import ldap3 from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, Connection, Server import frappe +from frappe.exceptions import MandatoryError, ValidationError from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings @@ -27,10 +28,9 @@ class LDAP_TestCase: def wrapped(self, *args, **kwargs): with mock.patch( - "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap" - ) as mock_connection: - mock_connection.return_value = self.connection - + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap", + return_value=self.connection, + ): self.test_class = LDAPSettings(self.doc) # Create a clean doc @@ -47,80 +47,64 @@ class LDAP_TestCase: return wrapped def clean_test_users(): - try: # clean up test user 1 + with contextlib.suppress(Exception): frappe.get_doc("User", "posix.user1@unit.testing").delete() - except Exception: - pass - - try: # clean up test user 2 + with contextlib.suppress(Exception): frappe.get_doc("User", "posix.user2@unit.testing").delete() - except Exception: - pass @classmethod - def setUpClass(self, ldapServer="OpenLDAP"): - - self.clean_test_users() + def setUpClass(cls): + cls.clean_test_users() # Save user data for restoration in tearDownClass() - self.user_ldap_settings = frappe.get_doc("LDAP Settings") + cls.user_ldap_settings = frappe.get_doc("LDAP Settings") # Create test user1 - self.user1doc = { + cls.user1doc = { "username": "posix.user", "email": "posix.user1@unit.testing", "first_name": "posix", + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": "System User", } - self.user1doc.update( - { - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": "System User", - } - ) - user = frappe.get_doc(self.user1doc) + user = frappe.get_doc(cls.user1doc) user.insert(ignore_permissions=True) - # Create test user1 - self.user2doc = { + cls.user2doc = { "username": "posix.user2", "email": "posix.user2@unit.testing", "first_name": "posix", + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": "System User", } - self.user2doc.update( - { - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": "System User", - } - ) - - user = frappe.get_doc(self.user2doc) + user = frappe.get_doc(cls.user2doc) user.insert(ignore_permissions=True) # Setup Mock OpenLDAP Directory - self.ldap_dc_path = "dc=unit,dc=testing" - self.ldap_user_path = "ou=users," + self.ldap_dc_path - self.ldap_group_path = "ou=groups," + self.ldap_dc_path - self.base_dn = "cn=base_dn_user," + self.ldap_dc_path - self.base_password = "my_password" - self.ldap_server = "ldap://my_fake_server:389" + cls.ldap_dc_path = "dc=unit,dc=testing" + cls.ldap_user_path = f"ou=users,{cls.ldap_dc_path}" + cls.ldap_group_path = f"ou=groups,{cls.ldap_dc_path}" + cls.base_dn = f"cn=base_dn_user,{cls.ldap_dc_path}" + cls.base_password = "my_password" + cls.ldap_server = "ldap://my_fake_server:389" - self.doc = { + cls.doc = { "doctype": "LDAP Settings", "enabled": True, - "ldap_directory_server": self.TEST_LDAP_SERVER, - "ldap_server_url": self.ldap_server, - "base_dn": self.base_dn, - "password": self.base_password, - "ldap_search_path_user": self.ldap_user_path, - "ldap_search_string": self.TEST_LDAP_SEARCH_STRING, - "ldap_search_path_group": self.ldap_group_path, + "ldap_directory_server": cls.TEST_LDAP_SERVER, + "ldap_server_url": cls.ldap_server, + "base_dn": cls.base_dn, + "password": cls.base_password, + "ldap_search_path_user": cls.ldap_user_path, + "ldap_search_string": cls.TEST_LDAP_SEARCH_STRING, + "ldap_search_path_group": cls.ldap_group_path, "ldap_user_creation_and_mapping_section": "", "ldap_email_field": "mail", - "ldap_username_field": self.LDAP_USERNAME_FIELD, + "ldap_username_field": cls.LDAP_USERNAME_FIELD, "ldap_first_name_field": "givenname", "ldap_middle_name_field": "", "ldap_last_name_field": "sn", @@ -135,50 +119,40 @@ class LDAP_TestCase: "ldap_group_objectclass": "", "ldap_group_member_attribute": "", "default_role": "Newsletter Manager", - "ldap_groups": self.DOCUMENT_GROUP_MAPPINGS, + "ldap_groups": cls.DOCUMENT_GROUP_MAPPINGS, "ldap_group_field": "", } - self.server = Server(host=self.ldap_server, port=389, get_info=self.LDAP_SCHEMA) - - self.connection = Connection( - self.server, - user=self.base_dn, - password=self.base_password, + cls.server = Server(host=cls.ldap_server, port=389, get_info=cls.LDAP_SCHEMA) + cls.connection = Connection( + cls.server, + user=cls.base_dn, + password=cls.base_password, read_only=True, client_strategy=MOCK_SYNC, ) - - self.connection.strategy.entries_from_json( - os.path.abspath(os.path.dirname(__file__)) + "/" + self.LDAP_LDIF_JSON + cls.connection.strategy.entries_from_json( + f"{os.path.abspath(os.path.dirname(__file__))}/{cls.LDAP_LDIF_JSON}" ) - - self.connection.bind() + cls.connection.bind() @classmethod - def tearDownClass(self): - try: + def tearDownClass(cls): + with contextlib.suppress(Exception): frappe.get_doc("LDAP Settings").delete() - except Exception: - pass - - try: - # return doc back to user data - self.user_ldap_settings.save() - - except Exception: - pass + # return doc back to user data + with contextlib.suppress(Exception): + cls.user_ldap_settings.save() # Clean-up test users - self.clean_test_users() + cls.clean_test_users() # Clear OpenLDAP connection - self.connection = None + cls.connection = None @mock_ldap_connection - def test_mandatory_fields(self): - + def test_mandatory_fields(self: TestCase): mandatory_fields = [ "ldap_server_url", "ldap_directory_server", @@ -195,26 +169,14 @@ class LDAP_TestCase: ] # fields that are required to have ldap functioning need to be mandatory for mandatory_field in mandatory_fields: - localdoc = self.doc.copy() localdoc[mandatory_field] = "" - try: - + with contextlib.suppress(MandatoryError, ValidationError): frappe.get_doc(localdoc).save() - self.fail(f"Document LDAP Settings field [{mandatory_field}] is not mandatory") - except frappe.exceptions.MandatoryError: - pass - - except frappe.exceptions.ValidationError: - if mandatory_field == "ldap_search_string": - # additional validation is done on this field, pass in this instance - pass - for non_mandatory_field in self.doc: # Ensure remaining fields have not been made mandatory - if non_mandatory_field == "doctype" or non_mandatory_field in mandatory_fields: continue @@ -222,15 +184,12 @@ class LDAP_TestCase: localdoc[non_mandatory_field] = "" try: - frappe.get_doc(localdoc).save() - - except frappe.exceptions.MandatoryError: + except MandatoryError: self.fail(f"Document LDAP Settings field [{non_mandatory_field}] should not be mandatory") @mock_ldap_connection - def test_validation_ldap_search_string(self): - + def test_validation_ldap_search_string(self: TestCase): invalid_ldap_search_strings = [ "", "uid={0}", @@ -242,19 +201,26 @@ class LDAP_TestCase: ] # ldap search string must be enclosed in '()' for ldap search to work for finding user and have the same number of opening and closing brackets. for invalid_search_string in invalid_ldap_search_strings: - localdoc = self.doc.copy() localdoc["ldap_search_string"] = invalid_search_string - try: + with contextlib.suppress(ValidationError): frappe.get_doc(localdoc).save() - self.fail(f"LDAP search string [{invalid_search_string}] should not validate") - except frappe.exceptions.ValidationError: - pass - - def test_connect_to_ldap(self): + def test_connect_to_ldap(self: TestCase): + # prevent these parameters for security or lack of the und user from being able to configure + prevent_connection_parameters = { + "mode": { + "IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure", + "IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure", + }, + "auto_bind": { + "NONE": "ldap3.Connection must autobind with base_dn", + "NO_TLS": "ldap3.Connection must have TLS", + "TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind", + }, + } # setup a clean doc with ldap disabled so no validation occurs (this is tested seperatly) local_doc = self.doc.copy() @@ -262,48 +228,25 @@ class LDAP_TestCase: self.test_class = LDAPSettings(self.doc) with mock.patch("ldap3.Server") as ldap3_server_method: - - with mock.patch("ldap3.Connection") as ldap3_connection_method: - ldap3_connection_method.return_value = self.connection - + with mock.patch("ldap3.Connection", return_value=self.connection) as ldap3_connection_method: with mock.patch("ldap3.Tls") as ldap3_Tls_method: - function_return = self.test_class.connect_to_ldap( base_dn=self.base_dn, password=self.base_password ) - args, kwargs = ldap3_connection_method.call_args - prevent_connection_parameters = { - # prevent these parameters for security or lack of the und user from being able to configure - "mode": { - "IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure", - "IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure", - }, - "auto_bind": { - "NONE": "ldap3.Connection must autobind with base_dn", - "NO_TLS": "ldap3.Connection must have TLS", - "TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind", - }, - } - for connection_arg in kwargs: - if ( connection_arg in prevent_connection_parameters and kwargs[connection_arg] in prevent_connection_parameters[connection_arg] ): - self.fail( - "ldap3.Connection was called with {}, failed reason: [{}]".format( - kwargs[connection_arg], - prevent_connection_parameters[connection_arg][kwargs[connection_arg]], - ) + f"ldap3.Connection was called with {kwargs[connection_arg]}, failed reason: [{prevent_connection_parameters[connection_arg][kwargs[connection_arg]]}]" ) + tls_version = ssl.PROTOCOL_TLS_CLIENT if local_doc["require_trusted_certificate"] == "Yes": tls_validate = ssl.CERT_REQUIRED - tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) self.assertTrue( @@ -313,7 +256,6 @@ class LDAP_TestCase: else: tls_validate = ssl.CERT_NONE - tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) self.assertTrue(kwargs["auto_bind"], "ldap3.Connection must autobind") @@ -347,7 +289,7 @@ class LDAP_TestCase: ) self.assertTrue( - type(function_return) is ldap3.core.connection.Connection, + type(function_return) is Connection, "The return type must be of ldap3.Connection", ) @@ -363,25 +305,21 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_get_ldap_client_settings(self): - + def test_get_ldap_client_settings(self: TestCase): result = self.test_class.get_ldap_client_settings() self.assertIsInstance(result, dict) - self.assertTrue(result["enabled"] == self.doc["enabled"]) # settings should match doc localdoc = self.doc.copy() localdoc["enabled"] = False frappe.get_doc(localdoc).save() - result = self.test_class.get_ldap_client_settings() self.assertFalse(result["enabled"]) # must match the edited doc @mock_ldap_connection - def test_update_user_fields(self): - + def test_update_user_fields(self: TestCase): test_user_data = { "username": "posix.user", "email": "posix.user1@unit.testing", @@ -391,11 +329,8 @@ class LDAP_TestCase: "phone": "08 1234 5678", "mobile_no": "0421 123 456", } - test_user = frappe.get_doc("User", test_user_data["email"]) - self.test_class.update_user_fields(test_user, test_user_data) - updated_user = frappe.get_doc("User", test_user_data["email"]) self.assertTrue(updated_user.middle_name == test_user_data["middle_name"]) @@ -404,8 +339,7 @@ class LDAP_TestCase: self.assertTrue(updated_user.mobile_no == test_user_data["mobile_no"]) @mock_ldap_connection - def test_sync_roles(self): - + def test_sync_roles(self: TestCase): if self.TEST_LDAP_SERVER.lower() == "openldap": test_user_data = { "posix.user1": [ @@ -457,9 +391,8 @@ class LDAP_TestCase: user.insert(ignore_permissions=True) for test_user in test_user_data: - - test_user_doc = frappe.get_doc("User", test_user + "@unit.testing") - test_user_roles = frappe.get_roles(test_user + "@unit.testing") + test_user_doc = frappe.get_doc("User", f"{test_user}@unit.testing") + test_user_roles = frappe.get_roles(f"{test_user}@unit.testing") self.assertTrue( len(test_user_roles) == 2, "User should only be a part of the All and Guest roles" @@ -467,28 +400,22 @@ class LDAP_TestCase: self.test_class.sync_roles(test_user_doc, test_user_data[test_user]) # update user roles - frappe.get_doc("User", test_user + "@unit.testing") - updated_user_roles = frappe.get_roles(test_user + "@unit.testing") + frappe.get_doc("User", f"{test_user}@unit.testing") + updated_user_roles = frappe.get_roles(f"{test_user}@unit.testing") self.assertTrue( len(updated_user_roles) == len(test_user_data[test_user]), - "syncing of the user roles failed. {} != {} for user {}".format( - len(updated_user_roles), len(test_user_data[test_user]), test_user - ), + f"syncing of the user roles failed. {len(updated_user_roles)} != {len(test_user_data[test_user])} for user {test_user}", ) for user_role in updated_user_roles: # match each users role mapped to ldap groups - self.assertTrue( role_to_group_map[user_role] in test_user_data[test_user], - "during sync_roles(), the user was given role {} which should not have occured".format( - user_role - ), + f"during sync_roles(), the user was given role {user_role} which should not have occured", ) @mock_ldap_connection - def test_create_or_update_user(self): - + def test_create_or_update_user(self: TestCase): test_user_data = { "posix.user1": [ "Users", @@ -498,28 +425,21 @@ class LDAP_TestCase: "frappe_default_guest", ], } - test_user = "posix.user1" - frappe.get_doc("User", test_user + "@unit.testing").delete() # remove user 1 + frappe.get_doc("User", f"{test_user}@unit.testing").delete() with self.assertRaises( frappe.exceptions.DoesNotExistError ): # ensure user deleted so function can be tested - frappe.get_doc("User", test_user + "@unit.testing") + frappe.get_doc("User", f"{test_user}@unit.testing") with mock.patch( "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.update_user_fields" ) as update_user_fields_method: - - update_user_fields_method.return_value = None - with mock.patch( "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.sync_roles" ) as sync_roles_method: - - sync_roles_method.return_value = None - # New user self.test_class.create_or_update_user(self.user1doc, test_user_data[test_user]) @@ -538,15 +458,12 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_get_ldap_attributes(self): - + def test_get_ldap_attributes(self: TestCase): method_return = self.test_class.get_ldap_attributes() - self.assertTrue(type(method_return) is list) @mock_ldap_connection - def test_fetch_ldap_groups(self): - + def test_fetch_ldap_groups(self: TestCase): if self.TEST_LDAP_SERVER.lower() == "openldap": test_users = {"posix.user": ["Users", "Administrators"], "posix.user2": ["Users", "Group3"]} elif self.TEST_LDAP_SERVER.lower() == "active directory": @@ -556,7 +473,6 @@ class LDAP_TestCase: } for test_user in test_users: - self.connection.search( search_base=self.ldap_user_path, search_filter=self.TEST_LDAP_SEARCH_STRING.format(test_user), @@ -569,18 +485,13 @@ class LDAP_TestCase: self.assertTrue(len(method_return) == len(test_users[test_user])) for returned_group in method_return: - self.assertTrue(returned_group in test_users[test_user]) @mock_ldap_connection - def test_authenticate(self): - + def test_authenticate(self: TestCase): with mock.patch( "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups" ) as fetch_ldap_groups_function: - - fetch_ldap_groups_function.return_value = None - self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password")) self.assertTrue( @@ -599,25 +510,19 @@ class LDAP_TestCase: ] # All invalid users should return 'invalid username or password' for username, password in enumerate(invalid_users): - with self.assertRaises(frappe.exceptions.ValidationError) as display_massage: - self.test_class.authenticate(username, password) self.assertTrue( str(display_massage.exception).lower() == "invalid username or password", - "invalid credentials passed authentication [user: {}, password: {}]".format( - username, password - ), + f"invalid credentials passed authentication [user: {username}, password: {password}]", ) @mock_ldap_connection - def test_complex_ldap_search_filter(self): - + def test_complex_ldap_search_filter(self: TestCase): ldap_search_filters = self.TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING for search_filter in ldap_search_filters: - self.test_class.ldap_search_string = search_filter if ( @@ -633,56 +538,45 @@ class LDAP_TestCase: else: self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password")) - def test_reset_password(self): - + def test_reset_password(self: TestCase): self.test_class = LDAPSettings(self.doc) # Create a clean doc localdoc = self.doc.copy() - localdoc["enabled"] = False frappe.get_doc(localdoc).save() with mock.patch( - "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap" + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap", + return_value=self.connection, ) as connect_to_ldap: - connect_to_ldap.return_value = self.connection - with self.assertRaises( frappe.exceptions.ValidationError ) as validation: # Fail if username string used self.test_class.reset_password("posix.user", "posix_user_password") - self.assertTrue(str(validation.exception) == "No LDAP User found for email: posix.user") - try: + with contextlib.suppress(Exception): self.test_class.reset_password( "posix.user1@unit.testing", "posix_user_password" ) # Change Password - - except Exception: # An exception from the tested class is ok, as long as the connection to LDAP was made writeable - pass - connect_to_ldap.assert_called_with(self.base_dn, self.base_password, read_only=False) @mock_ldap_connection - def test_convert_ldap_entry_to_dict(self): - + def test_convert_ldap_entry_to_dict(self: TestCase): self.connection.search( search_base=self.ldap_user_path, search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"), attributes=self.test_class.get_ldap_attributes(), ) - test_ldap_entry = self.connection.entries[0] - method_return = self.test_class.convert_ldap_entry_to_dict(test_ldap_entry) self.assertTrue(type(method_return) is dict) # must be dict self.assertTrue(len(method_return) == 6) # there are 6 fields in mock_ldap for use -class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase): +class Test_OpenLDAP(LDAP_TestCase, TestCase): TEST_LDAP_SERVER = "OpenLDAP" TEST_LDAP_SEARCH_STRING = "(uid={0})" DOCUMENT_GROUP_MAPPINGS = [ @@ -706,7 +600,7 @@ class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase): ] -class Test_ActiveDirectory(LDAP_TestCase, unittest.TestCase): +class Test_ActiveDirectory(LDAP_TestCase, TestCase): TEST_LDAP_SERVER = "Active Directory" TEST_LDAP_SEARCH_STRING = "(samaccountname={0})" DOCUMENT_GROUP_MAPPINGS = [ From 943334a90c43f536688cc6ef033e061108ee3019 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 18 Jul 2022 17:47:57 +0530 Subject: [PATCH 157/201] chore: fix docstrings --- frappe/integrations/google_oauth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index 1d5ed3723f..8bc54e0b1d 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -59,7 +59,6 @@ class GoogleOAuth: """Returns a dict with access and refresh token. :param oauth_code: code got back from google upon successful auhtorization - :param site_address: side address from which the request is being made """ data = { @@ -102,8 +101,7 @@ class GoogleOAuth: def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]: """Returns google authentication url. - :param site_address: side address from which the request is being made (for redirect back to site) - :param state: [optional] dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc) + :param state: dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc) """ state.update({"domain": self.domain}) From 26f4654b31a72b3b1233ae0ca83b3aa1b5a7d20b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 18 Jul 2022 18:22:12 +0530 Subject: [PATCH 158/201] test: Check user role & type updated via LDAP --- .../integrations/doctype/ldap_settings/test_ldap_settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index 8587c6c656..a158d42d61 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -121,6 +121,7 @@ class LDAP_TestCase: "default_role": "Newsletter Manager", "ldap_groups": cls.DOCUMENT_GROUP_MAPPINGS, "ldap_group_field": "", + "default_user_type": "System User", } cls.server = Server(host=cls.ldap_server, port=389, get_info=cls.LDAP_SCHEMA) @@ -338,6 +339,9 @@ class LDAP_TestCase: self.assertTrue(updated_user.phone == test_user_data["phone"]) self.assertTrue(updated_user.mobile_no == test_user_data["mobile_no"]) + self.assertEqual(updated_user.user_type, self.test_class.default_user_type) + self.assertIn(self.test_class.default_role, frappe.get_roles(updated_user.name)) + @mock_ldap_connection def test_sync_roles(self: TestCase): if self.TEST_LDAP_SERVER.lower() == "openldap": From 66c77f30dd0e03038dca559eeb3dd8fbb58117fb Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 15 Jul 2022 18:15:34 +0530 Subject: [PATCH 159/201] fix: chart wrapper padding This is close to the card boundary and looks ugly when numbers on Y axis start colliding with it. --- frappe/public/scss/desk/global.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 7466bdc874..1e68f374c4 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -589,6 +589,10 @@ details > summary:focus { background-color: var(--diff-removed); } +.chart-wrapper { + padding: 1em; +} + // REDESIGN TODO: Handling of broken images? // img.no-image:before { // .img-background(); From 3304e1c222a4a53d6e9cdcb7f7349221f3436978 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 18 Jul 2022 14:35:48 +0530 Subject: [PATCH 160/201] chore: bump frappe-charts to latest --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 1685dc9b25..c4ba042a89 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "driver.js": "^0.9.8", "editorjs-undo": "0.1.6", "fast-deep-equal": "^2.0.1", - "frappe-charts": "^2.0.0-rc13", + "frappe-charts": "2.0.0-rc22", "frappe-datatable": "^1.16.4", "frappe-gantt": "^0.6.0", "highlight.js": "^10.4.1", diff --git a/yarn.lock b/yarn.lock index b80d101883..57d5a47131 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1256,10 +1256,10 @@ fraction.js@^4.1.2: resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.2.0.tgz#448e5109a313a3527f5a3ab2119ec4cf0e0e2950" integrity sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA== -frappe-charts@^2.0.0-rc13: - version "2.0.0-rc13" - resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc13.tgz#fdb251d7ae311c41e38f90a3ae108070ec6b9072" - integrity sha512-Bv7IfllIrjRbKWHn5b769dOSenqdBixAr6m5kurf8ZUOJSLOgK4HOXItJ7BA8n9PvviH9/k5DaloisjLM2Bm1w== +frappe-charts@^2.0.0-rc22: + version "2.0.0-rc22" + resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc22.tgz#9a5a747febdc381a1d4d7af96e89cf519dfba8c0" + integrity sha512-N7f/8979wJCKjusOinaUYfMxB80YnfuVLrSkjpj4LtyqS0BGS6SuJxUnb7Jl4RWUFEIs7zEhideIKnyLeFZF4Q== frappe-datatable@^1.16.4: version "1.16.4" From 44630f5f62b75a58698f59c08686f7a1a2c17666 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 14 Jul 2022 20:11:58 +0530 Subject: [PATCH 161/201] feat: country specific number shortening in charts --- frappe/public/js/frappe/form/dashboard.js | 3 ++- frappe/public/js/frappe/utils/utils.js | 15 +++++++++++++-- .../js/frappe/views/reports/query_report.js | 4 ++-- .../js/frappe/views/reports/report_utils.js | 3 ++- .../public/js/frappe/views/reports/report_view.js | 3 ++- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index c057903a63..5c1b5d392f 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -554,7 +554,8 @@ frappe.ui.form.Dashboard = class FormDashboard { colors: ['green'], truncateLegends: 1, axisOptions: { - shortenYAxisNumbers: 1 + shortenYAxisNumbers: 1, + numberFormatter: frappe.utils.format_chart_axis_number, } }); this.show(); diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 877add95bf..78c9859b35 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1145,7 +1145,12 @@ Object.assign(frappe.utils, { { divisor: 1.0e+5, symbol: 'Lakh' - }], + }, + { + divisor: 1.0e+3, + symbol: 'K', + } + ], '': [{ divisor: 1.0e+12, @@ -1205,7 +1210,8 @@ Object.assign(frappe.utils, { axisOptions: { xIsSeries: 1, shortenYAxisNumbers: 1, - xAxisMode: 'tick' + xAxisMode: 'tick', + numberFormatter: frappe.utils.format_chart_axis_number, } }; @@ -1220,6 +1226,11 @@ Object.assign(frappe.utils, { return new frappe.Chart(wrapper, chart_args); }, + format_chart_axis_number(label, country) { + const default_country = frappe.sys_defaults.country; + return frappe.utils.shorten_number(label, country || default_country, 3); + }, + generate_route(item) { const type = item.type.toLowerCase(); if (type === "doctype") { diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index e15cc339ae..525bc5af4b 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -944,10 +944,10 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { }; } options.axisOptions = { - shortenYAxisNumbers: 1 + shortenYAxisNumbers: 1, + numberFormatter: frappe.utils.format_chart_axis_number, }; options.height = 280; - return options; } diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index f458a4daf6..35c8d132c8 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -30,7 +30,8 @@ frappe.report_utils = { colors: colors, axisOptions: { shortenYAxisNumbers: 1, - xAxisMode: 'tick' + xAxisMode: 'tick', + numberFormatter: frappe.utils.format_chart_axis_number, } }; diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 6880d472d3..2ea780c13d 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -529,7 +529,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { truncateLegends: 1, colors: ['#70E078', 'light-blue', 'orange', 'red'], axisOptions: { - shortenYAxisNumbers: 1 + shortenYAxisNumbers: 1, + numberFormatter: frappe.utils.format_chart_axis_number, }, tooltipOptions: { formatTooltipY: value => frappe.format(value, get_df(this.chart_args.y_axes[0]), { always_show_decimals: true, inline: true }, get_doc(value.doc)) From dcbf320b8b75496e56e4b72f889ddd169ade8693 Mon Sep 17 00:00:00 2001 From: Vladislav Date: Mon, 18 Jul 2022 16:14:35 +0300 Subject: [PATCH 162/201] fix: update ru translation (#17512) --- frappe/translations/ru.csv | 291 ++++++++++++++++++------------------- 1 file changed, 138 insertions(+), 153 deletions(-) diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 5e333c8e9d..bcd02baf93 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -107,7 +107,7 @@ Hourly,Почасовой, Hub Sync ID,Идентификатор синхронизации концентратора, IP Address,IP адрес, Image,Изображение, -Image View,Просмотр изображения, +Image View,Просмотр изображений, Import Data,Импорт данных, Import Log,Лог импорта, Inactive,Неактивный, @@ -246,7 +246,7 @@ Start Import,Начать импорт, State,Состояние, Stopped,Приостановлено, Subject,Тема, -Submit,Провести, +Submit,Подписать, Successful,Успешно, Summary,Резюме, Sunday,Воскресенье, @@ -445,7 +445,7 @@ Append To can be one of {0},Добавить к может быть одним Append To is mandatory for incoming mails,Добавить к является обязательным для входящих сообщений, "Append as communication against this DocType (must have fields, ""Status"", ""Subject"")","Добавить как коммуникацию для этого DocType (должен иметь поля, ""Статус"", ""Тема"")", Applicable Document Types,Применимые типы документов, -Apply,Подать заявление, +Apply,Применить, Apply Strict User Permissions,Применение строгих пользовательских разрешений, Apply To All Document Types,Применить ко всем типам документов, Apply this rule if the User is the Owner,"Применить это правило, если пользователь является владелец", @@ -679,8 +679,8 @@ Client Information,Информация о клиенте, Client Script,Скрипт клиента, Client URLs,URL-адреса клиентов, Client side script extensions in Javascript,Расширения клиентский сценарий в Javascript, -Collapsible,Складной, -Collapsible Depends On,Складные Зависит от, +Collapsible,Сворачиваемый, +Collapsible Depends On,Сворачиваемый - зависит от, Column,Колонка, Column {0} already exist.,Столбец {0} уже существует., Column Break,Разрыв столбца, @@ -706,7 +706,8 @@ Compiled Successfully,Успешно скомпилировано, Complete By,Завершить до, Complete Registration,Полная регистрация, Complete Setup,Завершение установки, -Completed By,Завершено, +Completed By,Завершил(а), +Completed On,Завершено, Compose Email,Написать письмо, Condition Detail,Детализация условий, Conditions,Условия, @@ -755,8 +756,6 @@ Created Custom Field {0} in {1},Дата создания настраиваем Created On,Дата создания, Criticism,Критика, Criticize,Критиковать, -Ctrl + Down,Ctrl + Down, -Ctrl + Up,Ctrl + Up, Ctrl+Enter to add comment,"Ctrl+Enter, чтобы добавить комментарий", Currency Name,Название валюты, Currency Precision,Точность валюты, @@ -879,7 +878,7 @@ Disable SMTP server authentication,Отключить аутентификаци Disable Sidebar Stats,Отключить статистику боковой панели, Disable Signup,Отключение Регистрация, Disable Standard Email Footer,Отключить стандартный нижний колонтитул электронной почты, -Discard,Отбросить, +Discard,Отменить, Display,Показать, Display Depends On,Показание зависит от, Do not allow user to change after set the first time,Не позволяйте пользователю изменять после установить в первый раз, @@ -1120,11 +1119,11 @@ First Transaction,Первая сделка, First data column must be blank.,Первая колонка данных должна быть пустой., First set the name and save the record.,Сначала задайте имя и сохраните запись., Flag,Флаг, -Float,Сплавы, -Float Precision,Float Precision, -Fold,Сложить, -Fold can not be at the end of the form,Fold не может быть в конце виде, -Fold must come before a Section Break,Сложите должны прийти до перерыва раздел, +Float,Дробное, +Float Precision,Плавающая точность, +Fold,Сворачиваемое, +Fold can not be at the end of the form,Сворачиваемое поле не может быть в конце формы, +Fold must come before a Section Break,Сворачиваемое должно идти до разрыва раздел, Folder,Папка, Folder name should not include '/' (slash),Имя папки не должно включать «/» (косая черта), Folder {0} is not empty,Папка {0} не пуста, @@ -1240,15 +1239,12 @@ Home Settings,Домашние настройки, Home/Test Folder 1,Главная/Тестовая Папка 1, Home/Test Folder 1/Test Folder 3,Главная/Тестовая Папка 1/Тестовая Папка 3, Home/Test Folder 2,Главная/Тестовая Папка 2, -Host,Host, -Hostname,Hostname, "How should this currency be formatted? If not set, will use system defaults","Как следует отображать числа в этой валюте? Если не указано, то будут использоваться системные значения", -I found these: ,Я нашел следующее:, +I found these: ,Я нашел это: , ID,ID, ID (name) of the entity whose property is to be set,"ID (имя) лица, имущество которого должно быть установлено", Icon will appear on the button,Иконка появится на кнопке, Identity Details,Сведения о личности, -Idx,Idx, "If Apply Strict User Permission is checked and User Permission is defined for a DocType for a User, then all the documents where value of the link is blank, will not be shown to that User","Если флажок Apply Strict User Permission установлен, а для пользователя DocType для пользователя задано разрешение пользователя, тогда все документы, где значение ссылки пустым, не будут показаны этому пользователю", If Checked workflow status will not override status in list view,"Если установлен флажок, статус процесса не будет отменять статус в журнале", If Owner,Если владелец, @@ -1303,7 +1299,7 @@ In Filter,В фильтрe, In Global Search,В глобальном поиске, In Grid View,В табличном виде, In Hours,В час, -In List View,В виде списка, +In List View,Отображать в списке, In Preview,В предварительном просмотре, In Reply To,В ответ на, In Standard Filter,В стандартный фильтр, @@ -1322,7 +1318,7 @@ Index,Индекс, Indicator,Индикатор, Info,Информация, Info:,Информация:, -Initial Sync Count,Первоначальная синхронизация Count, +Initial Sync Count,Первоначальная синхронизация, InnoDB,InnoDB, Insert Above,Вставить сверху, Insert After,Вставить после, @@ -1333,7 +1329,7 @@ Insert Column Before {0},Вставить столбец до {0}, Insert Style,Вставьте стиль, Insert new records,Вставить новые записи, Instructions Emailed,Инструкции отправлены по электронной почте, -Insufficient Permission for {0},Недостаточное разрешение для {0}, +Insufficient Permission for {0},Недостаточно прав для {0}, Int,Интервал, Integration Request,Интеграция заявки, Integration Request Service,Интеграция заявки на обслуживание, @@ -1374,33 +1370,30 @@ Invalid recipient address,Неверный адрес получателя, Invalid {0} condition,Недопустимое условие {0}, Inverse,Обратный, Is,Является, -Is Attachments Folder,Является папкой вложений, -Is Child Table,Является дочерней таблицей, +Is Attachments Folder,Это папка для вложений, +Is Child Table,Это дочерняя таблицей, Is Custom Field,Это нестандартное поле, Is First Startup,Первый запуск, -Is Folder,Папка, +Is Folder,Это папка, Is Global,Является глобальным, Is Globally Pinned,Глобально закреплено, Is Home Folder,Является корневой папкой, Is Mandatory Field,Является обязательным полем, Is Pinned,Прикреплено, -Is Primary Contact,Основной контакт, +Is Primary Contact,Это основной контакт, Is Private,Является приватным, -Is Published Field,Есть Опубликовано поле, -Is Published Field must be a valid fieldname,Опубликовано Поле должно быть действительным имя_полем, +Is Published Field,Это опубликованое поле, +Is Published Field must be a valid fieldname,Опубликованое роле должно быть допустимым именем поля, Is Single,Единственный, -Is Spam,Спам, -Is Standard,Стандартный отчёт, +Is Spam,Это спам, +Is Standard,Это стандартный отчёт, Is Submittable,Подлежит исполнению, Is Table,Является таблицей, Is Your Company Address,Является адресом вашей компании, It is risky to delete this file: {0}. Please contact your System Manager.,"Рискованно удалять этот файл: {0}. Пожалуйста, обратитесь к менеджеру системы.", Item cannot be added to its own descendents,Продукт не может быть добавлен к своим подпродуктам, -JS,JS, -JSON,JSON, JavaScript Format: frappe.query_reports['REPORTNAME'] = {},Формат JavaScript: frappe.query_reports ['REPORTNAME'] = {}, Javascript to append to the head section of the page.,Javascript для добавления к головной части страницы., -Jinja,Jinja, John Doe,Джон Доу, Kanban,Канбан, Kanban Board Column,Колонка канбан-доски, @@ -1480,7 +1473,7 @@ List,Список, List Filter,Фильтр списка, List View Setting,Настройка просмотра списка, List a document type,Перечислите тип документа, -"List as [{""label"": _(""Jobs""), ""route"":""jobs""}]","Список как [{""Ярлык"": _(""Работы""), ""маршруты"":""работы""}]", +"List as [{""label"": _(""Jobs""), ""route"":""jobs""}]","Список как [{""Метка"": _(""Работы""), ""маршруты"":""работы""}]", List of backups available for download,"Список резервных копий, доступных для загрузки", List of patches executed,Список выполненных патчей, List of themes for Website.,Список тем для сайта., @@ -1513,7 +1506,7 @@ Long Text,Длинный текст, Looks like something is wrong with this site's Paypal configuration.,"Похоже, что что-то не так с конфигурацией Paypal этого сайта.", Looks like something is wrong with this site's payment gateway configuration. No payment has been made.,"Похоже, что-то не так с конфигурацией платежного шлюза этого сайта. Платеж не был выполнен.", "Looks like something went wrong during the transaction. Since we haven't confirmed the payment, Paypal will automatically refund you this amount. If it doesn't, please send us an email and mention the Correlation ID: {0}.","Похоже, что-то пошло не так во время транзакции. Поскольку мы не подтвердили платеж, Paypal автоматически вернет вам эту сумму. Если это не так, отправьте нам электронное письмо и укажите идентификатор корреляции: {0}.", -Madam,Госпожа, +Madam,Мадам, Main Section,Основной раздел, "Make ""name"" searchable in Global Search","Индексировать ""name"" для глобального поиска", Make use of longer keyboard patterns,Используйте более длинных моделей клавиатуры, @@ -1540,7 +1533,7 @@ Max Value,Макс. значение, Max width for type Currency is 100px in row {0},Макс. ширина для типа валюты 100px в строке {0}, Maximum Attachment Limit for this record reached.,Достигнут предел вложений для этой записи., Maximum {0} rows allowed,Макс. {0} строк разрешено, -"Meaning of Submit, Cancel, Amend","Значение Провести, Отменить, Изменить", +"Meaning of Submit, Cancel, Amend","Значение Подписать, Отменить, Изменить", Mention transaction completion page URL,URL-ссылка на страницу-упоминание о завершении транзакции, Mentions,Упоминания, Menu,Меню, @@ -1606,14 +1599,14 @@ New Chat,Новый чат, New Comment on {0}: {1},Новый комментарий к {0}: {1}, New Connection,Новое соединение, New Custom Print Format,Новый пользовательский печатный бланк, -New Email,Новая электронная почта, +New Email,Новое письмо, New Email Account,Новый аккаунт электронной почты, New Event,Новое событие, New Folder,Новая папка, New Kanban Board,Новая панель канбан, New Message from Website Contact Page,Новое сообщение с формы обратной связи на сайте, New Name,Новое имя, -New Newsletter,Новый бюллетень, +New Newsletter,Новая новость, New Password,Новый пароль, New Password Required.,Требуется новый пароль., New Print Format Name,Название нового печатного бланка, @@ -1672,8 +1665,8 @@ No template found at path: {0},Нет шаблона по адресу: {0}, No {0} found,{0} не найдено, No {0} mail,Нет {0} почта, No {0} permission,Нет {0} разрешение, -None: End of Workflow,Ни один: Конец потока, -Not Allowed: Disabled User,Не разрешено: отключен пользователь, +None: End of Workflow,Нет: конец рабочего процесса, +Not Allowed: Disabled User,Не разрешено: пользователь отключен, Not Ancestors Of,Не предки, Not Descendants Of,Не потомки, Not Equals,Не равно, @@ -1688,7 +1681,7 @@ Not a valid Comma Separated Value (CSV File),"Не является допуст Not a valid User Image.,Недействительный изображение пользователя., Not a valid Workflow Action,Недоступное действие рабочего-процесса, Not a valid user,Не является действительным пользователем, -Not a zip file,Не zip файл, +Not a zip file,Не является zip файлом, Not allowed for {0}: {1},Не разрешено для {0}: {1}, Not allowed for {0}: {1} in Row {2}. Restricted field: {3},Недопустимо для {0}: {1} в строке {2}. Запрещенное поле: {3}, Not allowed for {0}: {1}. Restricted field: {2},Не допускается для {0}: {1}. Запрещенное поле: {2}, @@ -1732,11 +1725,11 @@ OTP Secret has been reset. Re-registration will be required on next login.,OTP S OTP secret can only be reset by the Administrator.,Секрет OTP может быть сброшен администратором., Office,Офис, Office 365,Офис 365, -Old Password,Старый Пароль, -Old Password Required.,Требуется старый пароль., +Old Password,Старый пароль, +Old Password Required.,Старый пароль обязателен., Older backups will be automatically deleted,Более старые резервные копии будут автоматически удалены, "On {0}, {1} wrote:","На {0}, {1} писал:", -"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.",После отправки поданные документы не могут быть изменены. Они могут быть только отменены и исправлены., +"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.",После отправки подписанные документы не могут быть изменены. Они могут быть только отменены и исправлены., "Once you have set this, the users will only be able access documents (eg. Blog Post) where the link exists (eg. Blogger).","После такой установки, пользователи получат доступ только к документам (например, сообщениям в блоге), связанным с этими разрешениями пользователя (например, блоггера).", One Last Step,Последний шаг, One Time Password (OTP) Registration Code from {},Одноразовый пароль (OTP) Регистрационный код от {}, @@ -1760,7 +1753,7 @@ Open a dialog with mandatory fields to create a new record quickly,"Открой Open a module or tool,Открыть модуль или инструмент, Open your authentication app on your mobile phone.,Откройте приложение для проверки подлинности на своем мобильном телефоне., Open {0},Открыть {0}, -Opened,Открыт, +Opened,Открыть, Operator must be one of {0},Оператор должен быть одним из {0}, Option 1,Опция 1, Option 2,Опция 2, @@ -1778,9 +1771,7 @@ Org History Heading,Org История Заголовок, Orientation,Ориентация, Original Value,Первоначальная стоимость, Outgoing email account not correct,Исходящая учетная запись электронной почты не верна, -Outlook.com,Outlook.com, Output,Вывод, -PDF,PDF, PDF Page Size,Размер PDF страницы, PDF Settings,Настройки PDF, PDF generation failed,Не удалось сгенерировать PDF-файл, @@ -1790,17 +1781,17 @@ Page HTML,Страница HTML, Page Length,Длина страницы, Page Name,Имя страницы, Page Settings,Настройки страницы, -Page has expired!,Страница просрочена!, +Page has expired!,Срок действия страницы истек!, Page not found,Страница не найдена, Page to show on the website\n,Страница для показа на сайте, Pages in Desk (place holders),Страницы-заглушки, Parent,Родитель, Parent Error Snapshot,Родитель снимка ошибки, Parent Label,Родительская метка, -Parent Table,Родитель Таблица, +Parent Table,Родительская таблица, Parent is required to get child table data,Родитель обязан получать данные дочерней таблицы, Parent is the name of the document to which the data will get added to.,"Родитель - это имя документа, к которому будут добавлены данные.", -Partial Success,Частичный успех, +Partial Success,Выполнено не полностью, Partially Successful,Частично успешный, Participants,Участники, Passive,Пассивный, @@ -1903,14 +1894,14 @@ Please specify which date field must be checked,"Просьба уточнить Please specify which value field must be checked,"Просьба уточнить, какие значения поля должны быть проверены", Please try again,"Пожалуйста, попробуйте еще раз", Please verify your Email Address,"Пожалуйста, подтвердите свой адрес электронной почты", -Point Allocation Periodicity,Периодичность распределения точек, +Point Allocation Periodicity,Периодичность распределения баллов, Points,Баллы, Points Given,Баллы засчитаны, Port,Порт, Portal Menu,Меню портала, -Portal Menu Item,Портал Пункт меню, +Portal Menu Item,Пункт меню портала, Post,Опубликовать, -Post Comment,Оставьте комментарий, +Post Comment,Оставить комментарий, Postal,Почтовый, Postal Code,Почтовый индекс, Postprocess Method,Метод постпроцесса, @@ -1966,15 +1957,6 @@ Provider Name,Имя поставщика, Public Key,Открытый ключ, Publishable Key,Ключ для публикации, Published On,Опубликовано на, -Pull,Тянуть, -Pull Failed,Не удалось выполнить Pull, -Pull Insert,Вставить вкладыш, -Pull Update,Pull Update, -Push,От себя, -Push Delete,Нажмите Удалить, -Push Failed,Ошибка отправки, -Push Insert,Push Insert, -Push Update,Push Update, Python Module,Модуль Python, Pyver,Pyver, QR Code,QR код, @@ -1986,7 +1968,7 @@ Query,Запрос, Query Report,Отчёт-выборка, Query must be a SELECT,Запрос должен быть ВЫБОР, Queue should be one of {0},Очередь должна быть одной из {0}, -Queued for backup. It may take a few minutes to an hour.,Queued для резервного копирования. Это может занять несколько минут до часа., +Queued for backup. It may take a few minutes to an hour.,В очереди для резервного копирования. Это может занять от несколько минут до часа., Queued for backup. You will receive an email with the download link,Очередь для резервного копирования. Вы получите электронное письмо с ссылкой для загрузки, Quick Help for Setting Permissions,Быстрая помощь при настройки прав доступа, Rating: ,Рейтинг: , @@ -2091,8 +2073,8 @@ Retake,пересдавать, Retry,Повторить, Return to the Verification screen and enter the code displayed by your authentication app,"Вернитесь на экран проверки и введите код, отображаемый приложением для аутентификации.", Reverse Icon Color,Обратный цвет значка, -Revert,Вернуть, -Revert Of,Вернуть из, +Revert,Возврат, +Revert Of,Возвращенно из, Reverted,Отменено, Review Level,Уровень обзора, Review Levels,Уровни обзора, @@ -2101,9 +2083,8 @@ Reviews,Отзывы, Revoke,Аннулировать, Revoked,Аннулировано, Rich Text,Форматированный текст, -Robots.txt,Robots.txt, -Role Name,Имя роли, -Role Permission for Page and Report,Роль Разрешение на страницу и отчет, +Role Name,Название роли, +Role Permission for Page and Report,Разрешение роли для страницы и отчета, Role Permissions,Разрешения роли, Role Profile,Профиль ролей, Role and Level,Роль и уровень, @@ -2198,7 +2179,7 @@ Select Print Format,Выберите бланк для печати, Select Print Format to Edit,Выберите печатный бланк для редактирования, Select Role,Выберите роль, Select Table Columns for {0},Выберите столбцы таблицы для {0}, -Select Your Region,Выберите регион, +Select Your Region,Выберите ваш регион, Select a Brand Image first.,Выберите бренд изображение в первую очередь., Select a DocType to make a new format,"Выберите DOCTYPE, чтобы сделать новый бланк", Select a chat to start messaging.,"Выберите чат, чтобы начать обмен сообщениями.", @@ -2234,7 +2215,6 @@ Send only if there is any data,"Отправить только если ест Send unsubscribe message in email,Отправить сообщение об отказе от подписки на электронную почту, Sender,Отправитель, Sender Email,Электронная почта отправителя, -Sendgrid,Sendgrid, Sent Read Receipt,Отправлять уведомление о прочтении, Sent or Received,Отправлено или получено, Sent/Received Email,Отправлено/Получено письмо, @@ -2279,8 +2259,8 @@ Setup Reports to be emailed at regular intervals,Настройка регуля Share,Поделиться, Share URL,Поделиться URL, Share With,Поделиться с, -Share this document with,Поделитесь этот документ с, -Share {0} with,Поделиться {0}, +Share this document with,Поделиться этим документом с, +Share {0} with,Поделиться {0} с, Shared,Общий, Shared With,Совместно с, Shared with everyone,Общий для всех, @@ -2325,8 +2305,6 @@ Single Post (article).,Один пост(статья)., Single Types have only one record no tables associated. Values are stored in tabSingles,"Холост Типы нет только одна запись не таблицы, связанные. Значения сохраняются в tabSingles", Skip Authorization,Пропустить авторизацию, Skip rows with errors,Пропустить строки с ошибками, -Skype,Skype, -Slack,Slack, Slack Channel,Slack канал, Slack Webhook Error,Slack Webhook ошибка, Slack Webhook URL,Неверный URL веб-хостинга, @@ -2350,14 +2328,13 @@ Something went wrong while generating dropbox access token. Please check error l Sorry! I could not find what you were looking for.,"Извините! Я не мог найти то, что вы ищете.", Sorry! Sharing with Website User is prohibited.,Извините! Поделиться с сайта Пользователю запрещается., Sorry! User should have complete access to their own record.,Извините! Пользователь должен иметь полный доступ к своей записи., -Sorry! You are not permitted to view this page.,Извините! Вам не разрешается для просмотра этой страницы., +Sorry! You are not permitted to view this page.,Извините! У вас нет разрешений для просмотра этой страницы., "Sorry, you're not authorized.","Извините, вы не авторизованы.", Sort Field,Сортировать поле, Sort Order,Порядок сортировки, Sort field {0} must be a valid fieldname,Сортировка поля {0} должен быть действительным имя_поля, Source Text,Исходный текст, Spam,Спам, -SparkPost,SparkPost, Special Characters are not allowed,Спецсимволы не допустимы, "Standard DocType cannot have default print format, use Customize Form","Стандартный DocType не может иметь формат печати по умолчанию, используйте Настроить форму", Standard Print Format cannot be updated,Стандартный печатный бланк не может быть обновлен, @@ -2371,12 +2348,11 @@ Start Date Field,Поле начальной даты, Start a conversation.,Начните разговор., Start entering data below this line,Начните вводить данные ниже этой линии, Start new Format,Начать новую Формат, -StartTLS,StartTLS, Started,Начал, Starting Frappe ...,Запуск Frappé ..., Starts on,Начало, -States,Статусы, -"States for workflow (e.g. Draft, Approved, Cancelled).","Статусы бизнес-процесса (например: черновик, утверждён, отменён).", +States,Состояния, +"States for workflow (e.g. Draft, Approved, Cancelled).","Состояния рабочего-процесса (например: черновик, утверждён, отменён).", Static Parameters,Статические параметры, Stats based on last month's performance (from {0} to {1}),Статистика на основе результатов прошлого месяца (от {0} до {1}), Stats based on last week's performance (from {0} to {1}),Статистика на основе результатов прошлой недели (от {0} до {1}), @@ -2396,7 +2372,7 @@ Subdomain,Субдомен, Subject Field,Поле темы, Submit after importing,Отправить после импорта, Submit an Issue,Отправить вопрос, -Submit this document to confirm,Провести этот документ для подтверждения, +Submit this document to confirm,Подписать этот документ для подтверждения, Submit {0} documents?,Отправить {0} документы?, Submitting {0},Помещение {0}, Submitted Document cannot be converted back to draft. Transition row {0},Проведенный Документ не может быть преобразован обратно в проект. Переходная строка {0}, @@ -2437,8 +2413,6 @@ Team Members,Члены команды, Team Members Heading,Члены команды Возглавлять, Temporarily Disabled,Временно отключен, Test Email Address,Проверить адрес электронной почты, -Test Runner,Тест Runner, -Test_Folder,Test_Folder, Text,Текст, Text Align,Выравнивание текста, Text Color,Цвет текста, @@ -2463,8 +2437,8 @@ The system provides many pre-defined roles. You can add new roles to set finer p The user from this field will be rewarded points,Пользователь из этого поля будет вознагражден баллами, Theme,Тема, Theme URL,URL темы, -There can be only one Fold in a form,Там может быть только один Fold в виде, -There is an error in your Address Template {0},Существует ошибка в адресной Шаблон {0}, +There can be only one Fold in a form,В форме может быть только один Fold, +There is an error in your Address Template {0},Ошибка в вашем шаблоне адреса {0}, There is no data to be exported,Нет данных для экспорта, There is some problem with the file url: {0},Существует некоторая проблема с файловой URL: {0}, There must be atleast one permission rule.,Там должно быть по крайней мере один правило разрешения., @@ -2502,7 +2476,7 @@ This is similar to a commonly used password.,Это похоже на обычн This is the template file generated with only the rows having some error. You should use this file for correction and import.,"Это файл шаблона, сгенерированный только строками с некоторой ошибкой. Вы должны использовать этот файл для исправления и импорта.", This link has already been activated for verification.,Эта ссылка уже была активирована для проверки., This link is invalid or expired. Please make sure you have pasted correctly.,"Эта ссылка является недействительным или истек. Пожалуйста, убедитесь, что вы вставили правильно.", -This may get printed on multiple pages,Это может быть напечатано на нескольких страницах, +This may get printed on multiple pages,Это будет напечатано на нескольких страницах, This month,Этот месяц, This query style is discontinued,Этот стиль запроса прекращен, This report was generated on {0},Этот отчет был создан в {0}, @@ -2511,7 +2485,6 @@ This request has not yet been approved by the user.,Этот запрос еще This role update User Permissions for a user,Эта роль обновляет разрешения пользователя для пользователя, This will log out {0} from all other devices,Это выведет {0} из всех других устройств, This will permanently remove your data.,Это навсегда удалит ваши данные., -Throttled,Throttled, Thumbnail URL,Миниатюра URL, Time Interval,Интервал времени, Time Series,Временные ряды, @@ -2529,7 +2502,7 @@ Timeseries,Временные ряды, Timestamp,Временная отметка, Title Case,Название дела, Title Field,Название поля, -Title Prefix,Название Приставка, +Title Prefix,Название приставки, Title field must be a valid fieldname,Название поля должно быть допустимым имя_поля, To Date Field,Поле даты, To Do,Список дел, @@ -2589,7 +2562,7 @@ Type:,Тип:, UID,UID, UIDNEXT,UIDNEXT, UIDVALIDITY,UIDVALIDITY, -UNSEEN,НЕПРОЧИТАННО, +UNSEEN,НЕПРОЧИТАННЫЕ, UPPER CASE,ВЕРХНИЙ РЕГИСТР, "URIs for receiving authorization code once the user allows access, as well as failure responses. Typically a REST endpoint exposed by the Client App.\n
e.g. http://hostname//api/method/frappe.www.login.login_via_facebook","Идентификаторы URI для получения кода авторизации, как только пользователь разрешает доступ, а также ответы недостаточность. Как правило, конечная точка REST подвергается Клиентом App.
например, HTTP: //hostname//api/method/frappe.www.login.login_via_facebook", URLs,URL-адрес, @@ -2691,7 +2664,6 @@ Value for {0} cannot be a list,Значение {0} не может быть с Value missing for,Нет значения для, Value too big,Слишком большое значение, Values Changed,Значения изменено, -Verdana,Verdana, Verfication Code,Код проверки, Verification Link,Ссылка для проверки, Verification code has been sent to your registered email address.,Код подтверждения отправлен на ваш адрес электронной почты регистрации., @@ -2712,7 +2684,7 @@ View document,Просмотр документа, View report in your browser,Просмотр отчета в вашем браузере, View this in your browser,Просмотреть это в вашем браузере, View {0},Просмотреть {0}, -Viewed By,просмотрены, +Viewed By,Просмотрено, Visit,Посетите нас по адресу, Visitor,Посетитель, We have received a request for deletion of {0} data associated with: {1},"Мы получили запрос на удаление {0} данных, связанных с: {1}", @@ -2721,7 +2693,7 @@ Web Form,Веб форма, Web Form Field,Поле веб формы, Web Form Fields,Поля веб формы, Web Page,Веб-страница, -Web Page Link Text,Web Текст Ссылка на страницу, +Web Page Link Text,Текст ссылки веб-страницы, Web Site,Веб-сайт, Web View,Web View, Webhook,Webhook, @@ -2765,9 +2737,7 @@ Workflow state represents the current state of a document.,Состояние р Write,Написать, Wrong fieldname {0} in add_fetch configuration of custom script,Неверное имя поля {0} в конфигурации add_fetch пользовательского скрипта, X Axis Field,Поле оси X, -XLSX,XLSX, Y Axis Fields,Поля оси Y, -Yahoo Mail,Yahoo Mail, Yandex.Mail,Яндекс.Почта, Yesterday,Вчера, You are connected to internet.,Вы подключены к Интернету., @@ -2783,14 +2753,14 @@ You are not permitted to view the newsletter.,У Вас нет прав для You are now following this document. You will receive daily updates via email. You can change this in User Settings.,Вы подписаны на обновления данного документа. Вы будете получать ежедневные обновления по электронной почте. Вы можете изменить это в настройках пользователя., You can add dynamic properties from the document by using Jinja templating.,Вы можете добавить динамические свойства из документа с помощью шаблонов Jinja., You can also copy-paste this ,Вы также можете скопировать и вставить это , -"You can change Submitted documents by cancelling them and then, amending them.","Вы можете изменить Проведенные (Submitted) документы, отменив их, а затем изменив их.", +"You can change Submitted documents by cancelling them and then, amending them.","Вы можете изменить утвержденные документы, отменив их, а затем отредактировав.", You can find things by asking 'find orange in customers',Можно искать что-либо написав «найти апельсин у клиентов», You can only upload upto 5000 records in one go. (may be less in some cases),"Вы можете загружать одновременно до 5000 записей. (Возможно меньше, в некоторых случаях)", You can use Customize Form to set levels on fields.,Вы можете использовать Настройку формы (Customize Form) для установки уровней для полей., You can use wildcard %,Вы можете использовать подстановочные %, You can't set 'Options' for field {0},Нельзя включить «Опции» для поля {0}, You can't set 'Translatable' for field {0},Вы не можете установить «Переводимый» для поля {0}, -You cannot give review points to yourself,Вы не можете давать оценки себе, +You cannot give review points to yourself,Вы не можете не можете начислять себе баллы обзора, You cannot unset 'Read Only' for field {0},Нельзя отменить «только чтение» для поля {0}, You do not have enough permissions to access this resource. Please contact your manager to get access.,"У вас нет достаточных прав для доступа к этому ресурсу. Пожалуйста, обратитесь к своему менеджеру, чтобы получить доступ.", You do not have enough permissions to complete the action,У Вас нет достаточных прав для завершения действия, @@ -2811,13 +2781,13 @@ You need to be in developer mode to edit a Standard Web Form,Вы должны You need to be logged in and have System Manager Role to be able to access backups.,"Вы должны войти в систему (и иметь роль менеджера системы), чтобы иметь доступ к резервным копиям.", You need to be logged in to access this {0}.,"Вы должны войти, чтобы получить доступ к {0}.", "You need to have ""Share"" permission","Вы должны иметь разрешение ""Поделиться""", -You need write permission to rename,"Вам нужно письменное разрешение, чтобы переименовать", +You need write permission to rename,"Вам нужно разрешение на запись, чтобы переименовать", You selected Draft or Cancelled documents,Вы выбрали черновик или отмененные документы, You unfollowed this document,Вы отписались от этого документа, Your Country,Ваша страна, Your Language,Ваш язык, Your Name,Ваше имя, -Your account has been locked and will resume after {0} seconds,Ваша учетная запись заблокирована и возобновится после {0} секунд, +Your account has been locked and will resume after {0} seconds,Ваша учетная запись заблокирована и будет доступна через {0} секунд, Your connection request to Google Calendar was successfully accepted,Ваш запрос на подключение к Календарю Google был успешно принят, Your information has been submitted,Ваша информация была представлена, Your login id is,Ваш ID для авторизации, @@ -2830,28 +2800,26 @@ Your payment was successfully accepted,Ваш платёж был успешно "Your session has expired, please login again to continue.","Ваша сессия истекла, пожалуйста, войдите снова, чтобы продолжить.", Zero,Ноль, Zero means send records updated at anytime,При нуле обновленные записи отправляются в любое время, -_doctype,_doctype, -_report,_report, adjust,настроить, after_insert,после_вставки, -align-center,выровнять-центр, -align-justify,выровнять-оправдать, -align-left,выровнять левый, -align-right,выровнять правый, +align-center,выровнять-по-центру, +align-justify,выровнять-по-ширине, +align-left,выровнять-по-левой-стороне, +align-right,выровнять-по-правой-стороне, ap-northeast-1,ар-северо-восток-1, ap-northeast-2,ар-северо-восток-2, ap-northeast-3,ар-северо-восток-3, ap-south-1,ар-юго-1, ap-southeast-1,ар-юго-восток-1, ap-southeast-2,ар-юго-восток-2, -arrow-down,Стрелка вниз, -arrow-left,стрелка налево, -arrow-right,стрелка направо, -arrow-up,Стрелка вверх, +arrow-down,стрелка-вниз, +arrow-left,стрелка-налево, +arrow-right,стрелка-направо, +arrow-up,стрелка-вверх, asterisk,звёздочка, backward,назад, -ban-circle,Запрет круга, -bell,Накладная, +ban-circle,бан-кружок, +bell,колокольчик, bookmark,закладка, briefcase,портфель, bullhorn,рупор, @@ -2909,11 +2877,10 @@ gave {0} points,дал {0} баллов, gift,подарок, glass,стекло, globe,глобус, -hand-down,ручной вниз, -hand-left,ручной левый, -hand-right,ручной право, -hand-up,грабитель, -hdd,жесткий диск, +hand-down,рука-вниз, +hand-left,рука-влево, +hand-right,рука-вправо, +hand-up,рука-вверх, headphones,наушники, heart,сердце, hub,хаб, @@ -3011,7 +2978,7 @@ zoom-in,приблизить, zoom-out,отдалить, {0} Calendar,{0} Календарь, {0} Chart,{0} Диаграмма, -{0} Dashboard,{0} Панель инструментов, +{0} Dashboard,{0} Показатели, {0} List,{0} Список, {0} Modules,{0} Модули, {0} Report,{0} Отчет, @@ -3028,7 +2995,7 @@ zoom-out,отдалить, {0} appreciated {1},{0} признателен {1}, {0} appreciation point for {1} {2},{0} благодарность за {1} {2}, {0} appreciation points for {1} {2},{0} баллы за {1} {2}, -{0} assigned {1}: {2},{0} назначено {1}: {2}, +{0} assigned {1}: {2},{0} назначил(а) {1}: {2}, {0} cannot be set for Single types,{0} не может быть установлена для отдельных видов, {0} comments,{0} комментариев, {0} created successfully,{0} создано успешно, @@ -3053,7 +3020,7 @@ zoom-out,отдалить, {0} is not a valid Workflow State. Please update your Workflow and try again.,{0} не является допустимым состоянием рабочего процесса. Обновите свой рабочий процесс и повторите попытку., {0} is now default print format for {1} doctype,{0} — теперь формат печати по умолчанию для {1} doctype, {0} is saved,{0} сохранено, -{0} items selected,{0} продуктов выбрано, +{0} items selected,{0} элементов выбрано, {0} logged in,{0} авторизирован, {0} logged out: {1},{0} вышел: {1}, {0} minutes ago,{0} минут назад, @@ -3097,7 +3064,7 @@ zoom-out,отдалить, {0}: Cannot set Assign Submit if not Submittable,"{0}: Не удается установить Назначить проведение, если не подлежит проведению", {0}: Cannot set Cancel without Submit,{0}: Не удается установить Отмена без отправки, {0}: Cannot set Import without Create,{0}: Не удается установить Импорт без Создать, -"{0}: Cannot set Submit, Cancel, Amend without Write","{0}: Не удается выполнить Провести, Отменить, Изменить без Записать", +"{0}: Cannot set Submit, Cancel, Amend without Write","{0}: Не удается выполнить Подписать, Отменить, Изменить без Записать", {0}: Cannot set import as {1} is not importable,{0}: Не удается установить импорт как {1} не является ввозу, {0}: No basic permissions set,{0}: Не установлен базовый набор разрешений, "{0}: Only one rule allowed with the same Role, Level and {1}","{0}: только одно правило допускается для той же роли, уровня и {1}", @@ -3146,7 +3113,7 @@ About {0} seconds remaining,Осталось {0} секунд, Access Log,Журнал доступа, Access not allowed from this IP Address,Доступ с этого IP-адреса запрещен, Action Type,Тип действия, -Activity Log by ,Активность Журнал по, +Activity Log by ,Журнал активности по , Add Fields,Добавить поля, Administration,Администрирование, After Cancel,После отмены, @@ -3162,7 +3129,7 @@ Allow Auto Repeat,Разрешить автоматическое повторе Allow Google Calendar Access,Разрешить доступ к календарю Google, Allow Google Contacts Access,Разрешить доступ к контактам Google, Allow Google Drive Access,Разрешить доступ Google Drive, -Allow Guest,Разрешить гость, +Allow Guest,Разрешить гостя, Allow Guests to Upload Files,Разрешить гостям загружать файлы, Also adding the status dependency field {0},Также добавляем поле зависимости статуса {0}, An error occurred while setting Session Defaults,Произошла ошибка при настройке параметров сеанса по умолчанию, @@ -3216,8 +3183,8 @@ Click on the link below to approve the request,"Нажмите на ссылку Click on the lock icon to toggle public/private,"Нажмите на значок замка, чтобы переключить публичный / приватный", Click on {0} to generate Refresh Token.,"Нажмите {0}, чтобы сгенерировать токен обновления.", Close Condition,Закрыть условие, -Column {0},Столбец {0}, -Columns / Fields,Колонны / Поля, +Column {0},Колонка {0}, +Columns / Fields,Колонки / Поля, "Configure notifications for mentions, assignments, energy points and more.","Настройте уведомления для упоминаний, назначений, энергетических очков и многое другое.", Contact Email,Эл.почта для связи, Contact Numbers,Контактные номера, @@ -3234,8 +3201,7 @@ Could not create razorpay order,Не удалось создать заказ н Create Log,Создать журнал, Create your first {0},Создайте свой первый {0}, Created {0} records successfully.,Создано {0} записей успешно., -Cron,Cron, -Cron Format,Крон Формат, +Cron Format,Cron формат, Daily Events should finish on the Same Day.,Ежедневные события должны заканчиваться в тот же день., Daily Long,Ежедневно, Default Role on Creation,Роль по умолчанию при создании, @@ -3257,10 +3223,10 @@ Document type is required to create a dashboard chart,Тип документа Documentation Link,Документация Ссылка, Don't Import,Не импортировать, Don't Send Emails,Не отправлять электронные письма, -"Drag and drop files, ","Перетащите файлы,", +"Drag and drop files, ","Перетащите файлы, ", Drop,Бросить, Drop Here,Бросить тут, -Drop files here,Перетащите файлы сюда, +Drop files here,Поместите файлы сюда, Dynamic Template,Динамический шаблон, ERPNext Role,ERPNext роль, Email / Notifications,Уведомления по электронной почте, @@ -3407,7 +3373,7 @@ Last Update,Последнее обновление, Last refreshed,Последнее обновление, Link Document Type,Тип документа ссылки, Link Fieldname,Имя поля ссылки, -Loading import file...,Загрузка файла импорта..., +Loading import file...,Загрузка импортируемого файла..., Local Document Type,Локальный тип документа, Log Data,Данные журнала, Main Section (HTML),Основной раздел (HTML), @@ -3415,7 +3381,7 @@ Main Section (Markdown),Основной раздел (Markdown), "Maintains a Log of all inserts, updates and deletions on Event Producer site for documents that have consumers.","Ведение журнала всех вставок, обновлений и удалений на сайте Event Producer для документов, имеющих потребителей.", Maintains a log of every event consumed along with the status of the sync and a Resync button in case sync fails.,"Ведение журнала всех использованных событий, а также состояния синхронизации и кнопки Resync в случае сбоя синхронизации.", Make all attachments private,Сделайте все вложения приватными, -Mandatory Depends On,Обязательно зависит от, +Mandatory Depends On,Обязателен - зависит от, Map Columns,Столбцы карты, Map columns from {0} to fields in {1},Сопоставить столбцы из {0} с полями в {1}, Mapping column {0} to field {1},Отображение столбца {0} в поле {1}, @@ -3426,7 +3392,7 @@ Me,Мне, Mention,Упоминание, Modules,Модули, Monthly Long,Ежемесячно долго, -Naming Series,Идентификация по Имени, +Naming Series,Именование серии, Navigate Home,Навигация Домой, Navigate list down,Переместиться вниз по списку, Navigate list up,Навигация по списку вверх, @@ -3483,7 +3449,7 @@ Press Alt Key to trigger additional shortcuts in Menu and Sidebar,"Нажмит Print Settings...,Настройки печати..., Producer Document Name,Название документа производителя, Producer URL,URL производителя, -Property Depends On,Недвижимость зависит от, +Property Depends On,Свойства зависят от, Pull from Google Calendar,Загрузить из календаря Google, Pull from Google Contacts,Загрузить из контактов Google, Pulled from Google Calendar,Загружено из календаря Google, @@ -3493,7 +3459,7 @@ Push to Google Contacts,Нажмите на контакты Google, Queue / Worker,Очередь / Рабочий, RAW Information Log,Необработанный информационный журнал, Raw Printing Settings...,Настройки необработанной печати..., -Read Only Depends On,Только чтение зависит от, +Read Only Depends On,Только для чтения - зависит от, Recent Activity,Недавняя активность, Reference document has been cancelled,Справочный документ был отменен, Reload File,Перезагрузить файл, @@ -3526,6 +3492,7 @@ Select Date Range,Выберите диапазон дат, Select Field,Выберите поле, Select Field...,Выберите поле..., Select Filters,Выберите фильтры, +Edit Filters,Изменить фильтры, Select Google Calendar to which event should be synced.,"Выберите календарь Google, к которому нужно синхронизировать событие.", Select Google Contacts to which contact should be synced.,"Выберите Google Контакты, с которыми контакт должен быть синхронизирован.", Select Group By...,Выбрать группу по..., @@ -3646,7 +3613,7 @@ Workflow Status,Состояние рабочего процесса, You are not allowed to export {} doctype,Вы не можете экспортировать {} doctype, You can try changing the filters of your report.,Вы можете попробовать изменить фильтры вашего отчета., You do not have permissions to cancel all linked documents.,У вас нет прав для отмены всех связанных документов., -You need to create these first: ,Вам нужно сначала создать это: , +You need to create these first: ,Вам сначала нужно создать: , You need to enable JavaScript for your app to work.,Вам нужно включить JavaScript для вашего приложения для работы., You need to install pycups to use this feature!,"Вам нужно установить pycups, чтобы использовать эту функцию!", Your Target,Ваша цель, @@ -3708,7 +3675,7 @@ Currency,Валюта, Customize,Настроить, Daily,Ежедневно, Date,Дата, -Dear,Уважаемый (ая), +Dear,Уважаемый(ая), Default,По умолчанию, Delete,Удалить, Description,Описание, @@ -3727,7 +3694,7 @@ Entity Type,Тип объекта, Error,Ошибка, Expired,Истек срок действия, Export,Экспорт, -Export not allowed. You need {0} role to export.,Экспорт не допускается. Вам нужно {0} роль для экспорта., +Export not allowed. You need {0} role to export.,Экспорт не допускается. Вам нужна роль {0} для экспорта., Fetching...,Получение..., Field,Поле, File Manager,Файловый менеджер, @@ -3788,7 +3755,6 @@ Set,Задать, Setup,Настройки, Setup Wizard,Мастер установки, Size,Размер, -Sr,Sr, Start,Начать, Start Time,Стартовое время, Status,Статус, @@ -3798,7 +3764,7 @@ Template,Шаблон, Thursday,Четверг, Title,Заголовок, Total,Общая сумма, -Totals,Всего:, +Totals,Всего, Tuesday,Вторник, Type,Тип, Update,Обновить, @@ -3809,10 +3775,10 @@ Welcome to {0},Добро пожаловать в {0}, Year,Год, Yearly,Ежегодно, You,Вы, -You can also copy-paste this link in your browser,Ещё можно скопировать эту ссылку в браузер, +You can also copy-paste this link in your browser,Вы также можете скопировать и вставить эту ссылку в свой браузер, and,и, {0} Name,{0} Имя, -{0} is required,{0} требуется, +{0} is required,{0} является обязательным, ALL,ВСЕ, Attach File,Прикрепить файл, Barcode,Штрих-код, @@ -3887,7 +3853,6 @@ Hidden,Скрытый, Javascript,Javascript, Ldap settings,Настройки Ldap, Mobile number,Мобильный номер, -Mx,Mx, No,Нет, Not found,Не обнаружена, Notes:,Примечания:, @@ -3994,7 +3959,7 @@ Desk Page,Рабочий стол, Desk Shortcut,Сочетание клавиш, Developer Mode Only,Только режим разработчика, Disable User Customization,Отключить настройку пользователя, -For example: {} Open,Например: {} Open, +For example: {} Open,Например: {} Открыто, Link Cards,Карты ссылок, Link To,Ссылка к, Onboarding,Вводный, @@ -4026,6 +3991,7 @@ Select Language,Выбрать язык, Confirm Translations,Подтвердить перевод, Contributed Translations,Добавленные переводы, Show Tags,Показать теги, +Hide Tags,Скрыть теги, Do not have permission to access {0} bucket.,У вас нет разрешения на доступ к сегменту {0}., Allow document creation via Email,Разрешить создание документов по электронной почте, Sender Field,Поле отправителя, @@ -4218,12 +4184,12 @@ since last year,с прошлого года, Show,Показать, New Number Card,Карточка с новым номером, Your Shortcuts,Ваши ярлыки, -You haven't added any Dashboard Charts or Number Cards yet.,Вы еще не добавили диаграммы или карточки с цифрами., -Click On Customize to add your first widget,"Нажмите "Настроить", чтобы добавить свой первый виджет.", +You haven't added any Dashboard Charts or Number Cards yet.,Вы еще не добавили диаграммы или карточки с показателями., +Click On Customize to add your first widget,"Нажмите Настроить, чтобы добавить свой первый виджет.", Are you sure you want to reset all customizations?,"Вы уверены, что хотите сбросить все настройки?", "Couldn't save, please check the data you have entered","Не удалось сохранить, проверьте данные, которые вы ввели", Validation Error,Ошибка проверки, -"You can only upload JPG, PNG, PDF, or Microsoft documents.","Вы можете загружать только документы в форматах JPG, PNG, PDF или Microsoft.", +"You can only upload JPG, PNG, PDF, or Microsoft documents.","Вы можете загружать только документы в форматах JPG, PNG, PDF или документы Microsoft.", Reverting length to {0} for '{1}' in '{2}'. Setting the length as {3} will cause truncation of data.,Возврат длины к {0} для '{1}' в '{2}'. Установка длины как {3} вызовет усечение данных., '{0}' not allowed for type {1} in row {2},'{0}' не разрешено для типа {1} в строке {2}, Option {0} for field {1} is not a child table,Вариант {0} для поля {1} не является дочерней таблицей, @@ -4437,7 +4403,7 @@ CTA,CTA, CTA Label,Метка CTA, CTA URL,CTA URL, Default Portal Home,Главная страница портала по умолчанию, -"Example: ""/desk""",Пример: "/ стол", +"Example: ""/desk""","Пример: ""/desk""", Social Link Settings,Настройки социальных ссылок, Social Link Type,Тип социальной ссылки, facebook,facebook, @@ -4543,11 +4509,11 @@ Too Many Requests,Слишком много запросов, {} is not a valid date string.,{} не является допустимой строкой даты., Invalid Date,Недействительная дата, Please select a valid date filter,"Пожалуйста, выберите действующий фильтр даты", -Value {0} must be in the valid duration format: d h m s,Значение {0} должно иметь допустимый формат продолжительности: dhms., +Value {0} must be in the valid duration format: d h m s,Значение {0} должно иметь допустимый формат продолжительности: д ч м с, Google Sheets URL is invalid or not publicly accessible.,URL-адрес Google Таблиц недействителен или не является общедоступным., "Google Sheets URL must end with ""gid={number}"". Copy and paste the URL from the browser address bar and try again.",URL-адрес Google Таблиц должен заканчиваться на "gid = {number}". Скопируйте и вставьте URL-адрес из адресной строки браузера и повторите попытку., Incorrect URL,Неверный URL, -"""{0}"" is not a valid Google Sheets URL","{0}" не является действительным URL-адресом Google Таблиц, +"""{0}"" is not a valid Google Sheets URL","""{0}"" не является действительным URL-адресом Google Таблиц", Duplicate Name,Повторяющееся имя, "Please check the value of ""Fetch From"" set for field {0}","Проверьте значение параметра "Получить из", установленное для поля {0}.", Wrong Fetch From value,Неверное значение Fetch From, @@ -4567,7 +4533,7 @@ Hourly comment limit reached for: {0},Достигнут лимит почасо Please add a valid comment.,"Пожалуйста, добавьте действительный комментарий.", Document {0} Already Restored,Документ {0} уже восстановлен, Restoring Deleted Document,Восстановление удаленного документа, -{function} of {fieldlabel},{функция} из {fieldlabel}, +{function} of {fieldlabel},{function} из {fieldlabel}, Invalid template file for import,Неверный файл шаблона для импорта, Invalid or corrupted content for import,Недействительный или поврежденный контент для импорта, Value {0} must in {1} format,Значение {0} должно быть в формате {1}, @@ -4575,7 +4541,7 @@ Value {0} must in {1} format,Значение {0} должно быть в фо Could not map column {0} to field {1},Не удалось сопоставить столбец {0} с полем {1}, Skipping Duplicate Column {0},Пропуск повторяющегося столбца {0}, The column {0} has {1} different date formats. Automatically setting {2} as the default format as it is the most common. Please change other values in this column to this format.,"Столбец {0} имеет {1} разные форматы даты. Автоматическая установка {2} в качестве формата по умолчанию, поскольку он является наиболее распространенным. Измените другие значения в этом столбце на этот формат.", -You have reached the hourly limit for generating password reset links. Please try again later.,"Вы достигли почасового лимита для создания ссылок для сброса пароля. Пожалуйста, попробуйте позже.", +You have reached the hourly limit for generating password reset links. Please try again later.,"Вы достигли лимита на создание ссылок для сброса пароля. Пожалуйста, попробуйте позже.", Please hide the standard navbar items instead of deleting them,Скройте стандартные элементы навигационной панели вместо их удаления, DocType's name should not start or end with whitespace,Имя DocType не должно начинаться или заканчиваться пробелом, File name cannot have {0},Имя файла не может содержать {0}, @@ -4600,14 +4566,14 @@ Delivery Failed,Доставка не удалась, Twilio WhatsApp Message Error,Ошибка сообщения Twilio WhatsApp, A featured post must have a cover image,В избранном посте должна быть обложка., Load More,Показать больше, -Published on,Опубликован в, +Published on,Опубликован, Enable developer mode to create a standard Web Template,"Включите режим разработчика, чтобы создать стандартный веб-шаблон", Was this article helpful?,Эта статья была полезной?, Thank you for your feedback!,Спасибо за ваш отзыв!, New Mention on {0},Новое упоминание о {0}, Assignment Update on {0},Обновление задания на {0}, New Document Shared {0},Новый документ опубликован {0}, -Energy Point Update on {0},Обновление Energy Point от {0}, +Energy Point Update on {0},Обновление баллов активности от {0}, You cannot create a dashboard chart from single DocTypes,Вы не можете создать диаграмму панели мониторинга из одного типа документов, Invalid json added in the custom options: {0},В настраиваемые параметры добавлен недопустимый json: {0}, Invalid JSON in card links for {0},Недействительный JSON в ссылках на карточки для {0}, @@ -4629,7 +4595,7 @@ Worflow States Don't Exist,Состояния Worflow не существуют, Save Anyway,Все равно сохранить, Energy Points:,Баллы активности:, Review Points:,Баллы обзора:, -Rank:,Ранг:, +Rank:,Рейтинг:, Monthly Rank:,Месячный рейтинг:, Invalid expression set in filter {0} ({1}),Недопустимое выражение в фильтре {0} ({1}), Invalid expression set in filter {0},В фильтре {0} задано недопустимое выражение, @@ -4688,12 +4654,12 @@ Open URL in a New Tab,Открыть URL в новой вкладке, Align Right,Выровнять по правому краю, Loading Filters...,Загрузка фильтров..., Count Customizations,Подсчет настроек, -For Example: {} Open,Например: {} Открыть, +For Example: {} Open,Например: {} Открыто, Choose Existing Card or create New Card,Выберите существующую карту или создайте новую карту, Number Cards,Числовые карты, Function Based On,Функция на основе, Add Filters,Добавить фильтры, -Skip,Пропускать, +Skip,Пропустить, Dismiss,Отклонить, Value cannot be negative for,Значение не может быть отрицательным для, Value cannot be negative for {0}: {1},Значение не может быть отрицательным для {0}: {1}, @@ -4702,9 +4668,28 @@ Authentication failed while receiving emails from Email Account: {0}.,Ошибк Message from server: {0},Сообщение с сервера: {0}, Documentation,Документация, User Forum,Форум пользователей, -Report an issue,Сообщить об ошибке, +Report an Issue,Сообщить об ошибке, +About,О системе, My Profile,Мой профиль, My Settings,Мои настройки, Toggle Full Width,Переключить ширину, Toggle Theme,Переключить тему, Modules,Модули, +You created this {0},Вы создали это {0}, +{0} created this {1},{0} создал(а) это {1}, +You edited this {0},Вы отредактировали это {0}, +{0} edited this {1},{0} отредактировал(а) это {1}, +You viewed this {0},Вы просмотрели это {0}, +{0} viewed this {1},{0} просмотрел(а) это {1}, +Apply Filters,Применить фильтры, ++ Add a Filter,+ Добавить фильтр, +Is Template,Является шаблоном, +Show Saved,Показать сохраненные, +Hide Saved,Скрыть сохраненные, +Add Tags,Добавить теги, +Page Size,Формат страницы, +Set all public,Сделать публичными, +Set all private,Сделать приватными, +Drag and drop files here or upload from,Перетащите файлы сюда или загрузите из, +My Device,Моё устройство, +Library,Библиотека, From 04f4fd8cfc4dc9cf9b4dd4696e23ea2ab0d0b5c3 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:00:54 +0530 Subject: [PATCH 163/201] fix: Redirect to comment in the doc comment section (#17538) --- frappe/templates/includes/comments/comments.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frappe/templates/includes/comments/comments.py b/frappe/templates/includes/comments/comments.py index c316db37fd..44963051ca 100644 --- a/frappe/templates/includes/comments/comments.py +++ b/frappe/templates/includes/comments/comments.py @@ -3,7 +3,7 @@ import re import frappe -from frappe import _ +from frappe import _, scrub from frappe.rate_limiter import rate_limit from frappe.utils.html_utils import clean_html from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit @@ -41,8 +41,13 @@ def add_comment(comment, comment_email, comment_by, reference_doctype, reference if route: clear_cache(route) - content = comment.content + "

{}

".format( - frappe.utils.get_request_site_address(), doc.route, comment.name, _("View Comment") + if doc.get("route"): + url = f"{frappe.utils.get_request_site_address()}/{doc.route}#{comment.name}" + else: + url = f"{frappe.utils.get_request_site_address()}/app/{scrub(doc.doctype)}/{doc.name}#comment-{comment.name}" + + content = comment.content + "

{}

".format( + url, _("View Comment") ) if doc.doctype == "Blog Post" and not doc.enable_email_notification: From a822092211533ff17ff9b92dd86f6f868ed63e2e Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 18 Jul 2022 22:16:25 +0530 Subject: [PATCH 164/201] fix: remove redundant condition --- frappe/patches/v13_0/encrypt_2fa_secrets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/patches/v13_0/encrypt_2fa_secrets.py b/frappe/patches/v13_0/encrypt_2fa_secrets.py index 1814ff50c5..3b220f485f 100644 --- a/frappe/patches/v13_0/encrypt_2fa_secrets.py +++ b/frappe/patches/v13_0/encrypt_2fa_secrets.py @@ -39,7 +39,6 @@ def execute(): .set(table.parent, PARENT_FOR_DEFAULTS) .set(table.defvalue, defvalue_cases) .where(table.parent == OLD_PARENT) - .where(table.defkey.like("%_otpsecret")) ).run() clear_defaults_cache() From 9a7f92ca1de2e91222764393a700db1191fb961c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 11 Jul 2022 22:16:34 +0530 Subject: [PATCH 165/201] fix!: allow system managers to toggle email queue --- frappe/client.py | 7 --- .../email/doctype/email_queue/email_queue.py | 17 +++++- .../doctype/email_queue/email_queue_list.js | 52 +++++++++++-------- frappe/email/queue.py | 2 +- frappe/patches.txt | 1 + .../patches/v14_0/set_hold_queue_default.py | 20 +++++++ frappe/public/js/frappe/defaults.js | 14 ----- 7 files changed, 68 insertions(+), 45 deletions(-) create mode 100644 frappe/patches/v14_0/set_hold_queue_default.py diff --git a/frappe/client.py b/frappe/client.py index 6ed40f8344..0b097909ca 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -274,13 +274,6 @@ def delete(doctype, name): frappe.delete_doc(doctype, name, ignore_missing=False) -@frappe.whitelist(methods=["POST", "PUT"]) -def set_default(key, value, parent=None): - """set a user default value""" - frappe.db.set_default(key, value, parent or frappe.session.user) - frappe.clear_cache(user=frappe.session.user) - - @frappe.whitelist(methods=["POST", "PUT"]) def bulk_update(docs): """Bulk update documents diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index eb07be0b38..9c783f675d 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -27,6 +27,7 @@ from frappe.utils import ( get_hook_method, get_string_between, nowdate, + sbool, split_emails, ) @@ -110,8 +111,11 @@ class EmailQueue(Document): return self.status in ["Not Sent", "Partially Sent"] def can_send_now(self): - hold_queue = cint(frappe.defaults.get_defaults().get("hold_queue")) == 1 - if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue: + if ( + frappe.are_emails_muted() + or not self.is_to_be_sent() + or cint(frappe.db.get_default("hold_queue")) == 1 + ): return False return True @@ -359,6 +363,8 @@ class SendMailContext: @frappe.whitelist() def retry_sending(name): doc = frappe.get_doc("Email Queue", name) + doc.check_permission() + if doc and (doc.status == "Error" or doc.status == "Partially Errored"): doc.status = "Not Sent" for d in doc.recipients: @@ -371,9 +377,16 @@ def retry_sending(name): def send_now(name): record = EmailQueue.find(name) if record: + record.check_permission() record.send() +@frappe.whitelist() +def toggle_sending(enable): + frappe.only_for("System Manager") + frappe.db.set_default("hold_queue", 0 if sbool(enable) else 1) + + def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" frappe.db.add_index( diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js index edc6250714..3cffc299af 100644 --- a/frappe/email/doctype/email_queue/email_queue_list.js +++ b/frappe/email/doctype/email_queue/email_queue_list.js @@ -3,27 +3,37 @@ frappe.listview_settings['Email Queue'] = { var colour = {'Sent': 'green', 'Sending': 'blue', 'Not Sent': 'grey', 'Error': 'red', 'Expired': 'orange'}; return [__(doc.status), colour[doc.status], "status,=," + doc.status]; }, - refresh: function(doclist){ - if (has_common(frappe.user_roles, ["Administrator", "System Manager"])){ - if (cint(frappe.defaults.get_default("hold_queue"))){ - doclist.page.clear_inner_toolbar() - doclist.page.add_inner_button(__("Resume Sending"), function() { - frappe.defaults.set_default("hold_queue", 0); - cur_list.refresh(); - }) - } else { - doclist.page.clear_inner_toolbar() - doclist.page.add_inner_button(__("Suspend Sending"), function() { - frappe.defaults.set_default("hold_queue", 1) - cur_list.refresh(); - }) - } - } - }, - - onload: function(listview) { + refresh: show_toggle_sending_button, + onload: function(list_view) { frappe.require("logtypes.bundle.js", () => { - frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + frappe.utils.logtypes.show_log_retention_message(list_view.doctype); }) } -} +}; + +function show_toggle_sending_button(list_view) { + if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) + return; + + const sending_disabled = cint(frappe.sys_defaults.hold_queue); + const label = sending_disabled ? __("Resume Sending") : __("Suspend Sending"); + + list_view.page.add_inner_button( + label, + async () => { + await frappe.xcall( + "frappe.email.doctype.email_queue.email_queue.toggle_sending", + + // enable if disabled + {enable: sending_disabled} + ); + + // set new value for hold_queue in sys_defaults + frappe.sys_defaults.hold_queue = sending_disabled ? 0 : 1; + + // clear the button and show one with the opposite label + list_view.page.remove_inner_button(label); + show_toggle_sending_button(list_view); + } + ); +} \ No newline at end of file diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 2c3e0ee011..9805ff7c3b 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -148,7 +148,7 @@ def flush(from_test=False): msgprint(_("Emails are muted")) from_test = True - if cint(frappe.defaults.get_defaults().get("hold_queue")) == 1: + if cint(frappe.db.get_default("hold_queue")) == 1: return for row in get_queue(): diff --git a/frappe/patches.txt b/frappe/patches.txt index d9b827931c..757f0169d2 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -205,3 +205,4 @@ frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.update_integration_request frappe.patches.v14_0.set_document_expiry_default frappe.patches.v14_0.delete_data_migration_tool +frappe.patches.v14_0.set_hold_queue_default \ No newline at end of file diff --git a/frappe/patches/v14_0/set_hold_queue_default.py b/frappe/patches/v14_0/set_hold_queue_default.py new file mode 100644 index 0000000000..442d3801bc --- /dev/null +++ b/frappe/patches/v14_0/set_hold_queue_default.py @@ -0,0 +1,20 @@ +import frappe +from frappe.cache_manager import clear_defaults_cache + + +def execute(): + frappe.db.set_default( + "hold_queue", + frappe.db.get_default("hold_queue", "Administrator") or 0, + parent="__default", + ) + + frappe.db.delete( + "DefaultValue", + { + "defkey": "hold_queue", + "parent": ("!=", "__default"), + }, + ) + + clear_defaults_cache() diff --git a/frappe/public/js/frappe/defaults.js b/frappe/public/js/frappe/defaults.js index 6115afb784..858880df01 100644 --- a/frappe/public/js/frappe/defaults.js +++ b/frappe/public/js/frappe/defaults.js @@ -47,20 +47,6 @@ frappe.defaults = { if(!$.isArray(d)) d = [d]; return d; }, - set_default: function(key, value, callback) { - if(typeof value!=="string") - value = JSON.stringify(value); - - frappe.boot.user.defaults[key] = value; - return frappe.call({ - method: "frappe.client.set_default", - args: { - key: key, - value: value - }, - callback: callback || function(r) {} - }); - }, set_user_default_local: function(key, value) { frappe.boot.user.defaults[key] = value; }, From 4b0a9da400d87578a39a8d2b81f0efe893515dac Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Mon, 18 Jul 2022 22:32:12 +0530 Subject: [PATCH 166/201] refactor: rename `hold_queue` to `suspend_email_queue` --- frappe/email/doctype/email_queue/email_queue.py | 4 ++-- frappe/email/doctype/email_queue/email_queue_list.js | 6 +++--- frappe/email/queue.py | 2 +- frappe/patches.txt | 2 +- ..._default.py => set_suspend_email_queue_default.py} | 11 ++--------- 5 files changed, 9 insertions(+), 16 deletions(-) rename frappe/patches/v14_0/{set_hold_queue_default.py => set_suspend_email_queue_default.py} (63%) diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 9c783f675d..3c020eea39 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -114,7 +114,7 @@ class EmailQueue(Document): if ( frappe.are_emails_muted() or not self.is_to_be_sent() - or cint(frappe.db.get_default("hold_queue")) == 1 + or cint(frappe.db.get_default("suspend_email_queue")) == 1 ): return False @@ -384,7 +384,7 @@ def send_now(name): @frappe.whitelist() def toggle_sending(enable): frappe.only_for("System Manager") - frappe.db.set_default("hold_queue", 0 if sbool(enable) else 1) + frappe.db.set_default("suspend_email_queue", 0 if sbool(enable) else 1) def on_doctype_update(): diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js index 3cffc299af..ab2a1b9a45 100644 --- a/frappe/email/doctype/email_queue/email_queue_list.js +++ b/frappe/email/doctype/email_queue/email_queue_list.js @@ -15,7 +15,7 @@ function show_toggle_sending_button(list_view) { if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; - const sending_disabled = cint(frappe.sys_defaults.hold_queue); + const sending_disabled = cint(frappe.sys_defaults.suspend_email_queue); const label = sending_disabled ? __("Resume Sending") : __("Suspend Sending"); list_view.page.add_inner_button( @@ -28,8 +28,8 @@ function show_toggle_sending_button(list_view) { {enable: sending_disabled} ); - // set new value for hold_queue in sys_defaults - frappe.sys_defaults.hold_queue = sending_disabled ? 0 : 1; + // set new value for suspend_email_queue in sys_defaults + frappe.sys_defaults.suspend_email_queue = sending_disabled ? 0 : 1; // clear the button and show one with the opposite label list_view.page.remove_inner_button(label); diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 9805ff7c3b..bc02c6be32 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -148,7 +148,7 @@ def flush(from_test=False): msgprint(_("Emails are muted")) from_test = True - if cint(frappe.db.get_default("hold_queue")) == 1: + if cint(frappe.db.get_default("suspend_email_queue")) == 1: return for row in get_queue(): diff --git a/frappe/patches.txt b/frappe/patches.txt index 757f0169d2..7abe7893c5 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -205,4 +205,4 @@ frappe.patches.v14_0.update_auto_account_deletion_duration frappe.patches.v14_0.update_integration_request frappe.patches.v14_0.set_document_expiry_default frappe.patches.v14_0.delete_data_migration_tool -frappe.patches.v14_0.set_hold_queue_default \ No newline at end of file +frappe.patches.v14_0.set_suspend_email_queue_default \ No newline at end of file diff --git a/frappe/patches/v14_0/set_hold_queue_default.py b/frappe/patches/v14_0/set_suspend_email_queue_default.py similarity index 63% rename from frappe/patches/v14_0/set_hold_queue_default.py rename to frappe/patches/v14_0/set_suspend_email_queue_default.py index 442d3801bc..8cdb05a177 100644 --- a/frappe/patches/v14_0/set_hold_queue_default.py +++ b/frappe/patches/v14_0/set_suspend_email_queue_default.py @@ -4,17 +4,10 @@ from frappe.cache_manager import clear_defaults_cache def execute(): frappe.db.set_default( - "hold_queue", + "suspend_email_queue", frappe.db.get_default("hold_queue", "Administrator") or 0, parent="__default", ) - frappe.db.delete( - "DefaultValue", - { - "defkey": "hold_queue", - "parent": ("!=", "__default"), - }, - ) - + frappe.db.delete("DefaultValue", {"defkey": "hold_queue"}) clear_defaults_cache() From 8ac7e3221497951a1d6cd1666850dd8c115b2627 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 19 Jul 2022 11:50:48 +0530 Subject: [PATCH 167/201] ci: check commit messages with commitlint (#17541) --- .github/workflows/semantic-commits.yml | 30 ++++++++++++++++++++++++++ package.json | 5 +++++ 2 files changed, 35 insertions(+) create mode 100644 .github/workflows/semantic-commits.yml diff --git a/.github/workflows/semantic-commits.yml b/.github/workflows/semantic-commits.yml new file mode 100644 index 0000000000..4bfb273ffa --- /dev/null +++ b/.github/workflows/semantic-commits.yml @@ -0,0 +1,30 @@ +name: Semantic Commits + +on: + pull_request: {} + +permissions: + contents: read + +concurrency: + group: commitcheck-frappe-${{ github.event.number }} + cancel-in-progress: true + +jobs: + commitlint: + name: Check Commit Titles + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v3 + with: + node-version: 14 + check-latest: true + + - name: Check commit titles + run: | + npm install @commitlint/cli @commitlint/config-conventional + npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} diff --git a/package.json b/package.json index c4ba042a89..49617caf7e 100644 --- a/package.json +++ b/package.json @@ -82,5 +82,10 @@ "snyk": true, "nyc": { "report-dir": ".cypress-coverage" + }, + "commitlint": { + "extends": [ + "@commitlint/config-conventional" + ] } } From 4eb1fe74a5f85b6d3008b7dfd5c543d82502fb7d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 19 Jul 2022 12:03:18 +0530 Subject: [PATCH 168/201] chore: dont fetch full repo In most cases we need 10-50 previous commits. Checking out full repo is time consuming and not required. --- .github/workflows/semantic-commits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semantic-commits.yml b/.github/workflows/semantic-commits.yml index 4bfb273ffa..a3536d5019 100644 --- a/.github/workflows/semantic-commits.yml +++ b/.github/workflows/semantic-commits.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 200 - uses: actions/setup-node@v3 with: From 21a7291d00cc1c1fa809012f0c46d2e1eef189ef Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 19 Jul 2022 14:32:34 +0530 Subject: [PATCH 169/201] ci: Stripped down config for Semantic checks (#17542) * ci: Stripped down config for Semantic checks Conventional commit enforces too many pointless rules, all we care about is type and subject. * ci: auto merge on commit pass --- .github/semantic.yml | 30 ------------------------------ .mergify.yml | 1 + commitlint.config.js | 25 +++++++++++++++++++++++++ package.json | 5 ----- 4 files changed, 26 insertions(+), 35 deletions(-) delete mode 100644 .github/semantic.yml create mode 100644 commitlint.config.js diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index fa15046b4a..0000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Always validate the PR title AND all the commits -titleAndCommits: true - -# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowMergeCommits: true - -# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowRevertCommits: true - -# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# Tool Reference: https://github.com/zeke/semantic-pull-requests - -# By default types specified in commitizen/conventional-commit-types is used. -# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# You can override the valid types -types: - - BREAKING CHANGE - - feat - - fix - - docs - - style - - refactor - - perf - - test - - build - - ci - - chore - - revert diff --git a/.mergify.yml b/.mergify.yml index d9896df921..a863ee67dd 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -21,6 +21,7 @@ pull_request_rules: - name: Automatic merge on CI success and review conditions: - status-success=Sider + - status-success=Check Commit Titles - status-success=Python Unit Tests (MariaDB) (1) - status-success=Python Unit Tests (MariaDB) (2) - status-success=Python Unit Tests (Postgres) (1) diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000000..8847564e53 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,25 @@ +module.exports = { + parserPreset: 'conventional-changelog-conventionalcommits', + rules: { + 'subject-empty': [2, 'never'], + 'type-case': [2, 'always', 'lower-case'], + 'type-empty': [2, 'never'], + 'type-enum': [ + 2, + 'always', + [ + 'build', + 'chore', + 'ci', + 'docs', + 'feat', + 'fix', + 'perf', + 'refactor', + 'revert', + 'style', + 'test', + ], + ], + }, +}; diff --git a/package.json b/package.json index 49617caf7e..c4ba042a89 100644 --- a/package.json +++ b/package.json @@ -82,10 +82,5 @@ "snyk": true, "nyc": { "report-dir": ".cypress-coverage" - }, - "commitlint": { - "extends": [ - "@commitlint/config-conventional" - ] } } From c7726d6394d26f7e72a5abb3266b576ef9c4c87c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 19 Jul 2022 15:00:23 +0530 Subject: [PATCH 170/201] fix: Pick default_role for Sytem User type only --- .../doctype/ldap_settings/ldap_settings.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index f9d083a12f..735b96968c 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -145,7 +145,10 @@ class LDAPSettings(Document): def sync_roles(self, user: "User", additional_groups: list = None): current_roles = {d.role for d in user.get("roles")} - needed_roles = {self.default_role} + if self.default_user_type == "System User": + needed_roles = {self.default_role} + else: + needed_roles = set() lower_groups = [g.lower() for g in additional_groups or []] all_mapped_roles = {r.erpnext_role for r in self.ldap_groups} @@ -170,15 +173,12 @@ class LDAPSettings(Document): user = frappe.get_doc("User", user_data["email"]) LDAPSettings.update_user_fields(user=user, user_data=user_data) else: - doc = user_data - doc.update( - { - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": self.default_user_type, - } - ) + doc = user_data | { + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": self.default_user_type, + } user = frappe.get_doc(doc) user.insert(ignore_permissions=True) From c55bb98482bb32589b46380373de8edd62ab980a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 19 Jul 2022 15:01:16 +0530 Subject: [PATCH 171/201] test: LDAP test for website user creation --- .../ldap_settings/test_ldap_settings.py | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index a158d42d61..9080e0c82a 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -23,6 +23,11 @@ class LDAP_TestCase: LDAP_LDIF_JSON = None TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = None + # for adding type hints during development ^_^ + assertTrue = TestCase.assertTrue + assertEqual = TestCase.assertEqual + assertIn = TestCase.assertIn + def mock_ldap_connection(f): @functools.wraps(f) def wrapped(self, *args, **kwargs): @@ -51,6 +56,8 @@ class LDAP_TestCase: frappe.get_doc("User", "posix.user1@unit.testing").delete() with contextlib.suppress(Exception): frappe.get_doc("User", "posix.user2@unit.testing").delete() + with contextlib.suppress(Exception): + frappe.get_doc("User", "website_ldap_user@test.com").delete() @classmethod def setUpClass(cls): @@ -153,7 +160,7 @@ class LDAP_TestCase: cls.connection = None @mock_ldap_connection - def test_mandatory_fields(self: TestCase): + def test_mandatory_fields(self): mandatory_fields = [ "ldap_server_url", "ldap_directory_server", @@ -190,7 +197,7 @@ class LDAP_TestCase: self.fail(f"Document LDAP Settings field [{non_mandatory_field}] should not be mandatory") @mock_ldap_connection - def test_validation_ldap_search_string(self: TestCase): + def test_validation_ldap_search_string(self): invalid_ldap_search_strings = [ "", "uid={0}", @@ -209,7 +216,7 @@ class LDAP_TestCase: frappe.get_doc(localdoc).save() self.fail(f"LDAP search string [{invalid_search_string}] should not validate") - def test_connect_to_ldap(self: TestCase): + def test_connect_to_ldap(self): # prevent these parameters for security or lack of the und user from being able to configure prevent_connection_parameters = { "mode": { @@ -306,7 +313,7 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_get_ldap_client_settings(self: TestCase): + def test_get_ldap_client_settings(self): result = self.test_class.get_ldap_client_settings() self.assertIsInstance(result, dict) @@ -320,7 +327,7 @@ class LDAP_TestCase: self.assertFalse(result["enabled"]) # must match the edited doc @mock_ldap_connection - def test_update_user_fields(self: TestCase): + def test_update_user_fields(self): test_user_data = { "username": "posix.user", "email": "posix.user1@unit.testing", @@ -343,7 +350,19 @@ class LDAP_TestCase: self.assertIn(self.test_class.default_role, frappe.get_roles(updated_user.name)) @mock_ldap_connection - def test_sync_roles(self: TestCase): + def test_create_website_user(self): + new_test_user_data = { + "username": "website_ldap_user.test", + "email": "website_ldap_user@test.com", + "first_name": "Website User - LDAP Test", + } + self.test_class.default_user_type = "Website User" + self.test_class.create_or_update_user(user_data=new_test_user_data, groups=[]) + new_user = frappe.get_doc("User", new_test_user_data["email"]) + self.assertEqual(new_user.user_type, "Website User") + + @mock_ldap_connection + def test_sync_roles(self): if self.TEST_LDAP_SERVER.lower() == "openldap": test_user_data = { "posix.user1": [ @@ -419,7 +438,7 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_create_or_update_user(self: TestCase): + def test_create_or_update_user(self): test_user_data = { "posix.user1": [ "Users", @@ -462,12 +481,12 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_get_ldap_attributes(self: TestCase): + def test_get_ldap_attributes(self): method_return = self.test_class.get_ldap_attributes() self.assertTrue(type(method_return) is list) @mock_ldap_connection - def test_fetch_ldap_groups(self: TestCase): + def test_fetch_ldap_groups(self): if self.TEST_LDAP_SERVER.lower() == "openldap": test_users = {"posix.user": ["Users", "Administrators"], "posix.user2": ["Users", "Group3"]} elif self.TEST_LDAP_SERVER.lower() == "active directory": @@ -492,7 +511,7 @@ class LDAP_TestCase: self.assertTrue(returned_group in test_users[test_user]) @mock_ldap_connection - def test_authenticate(self: TestCase): + def test_authenticate(self): with mock.patch( "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups" ) as fetch_ldap_groups_function: @@ -523,7 +542,7 @@ class LDAP_TestCase: ) @mock_ldap_connection - def test_complex_ldap_search_filter(self: TestCase): + def test_complex_ldap_search_filter(self): ldap_search_filters = self.TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING for search_filter in ldap_search_filters: @@ -542,7 +561,7 @@ class LDAP_TestCase: else: self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password")) - def test_reset_password(self: TestCase): + def test_reset_password(self): self.test_class = LDAPSettings(self.doc) # Create a clean doc @@ -567,7 +586,7 @@ class LDAP_TestCase: connect_to_ldap.assert_called_with(self.base_dn, self.base_password, read_only=False) @mock_ldap_connection - def test_convert_ldap_entry_to_dict(self: TestCase): + def test_convert_ldap_entry_to_dict(self): self.connection.search( search_base=self.ldap_user_path, search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"), From 31ef1c7355fa393f275c6fd4c00e86e2e8f7a08c Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Tue, 19 Jul 2022 15:23:04 +0530 Subject: [PATCH 172/201] perf: option to skip realtime notify update after insert (#17543) While doing optimization for the period closing voucher, found that `notify_update` takes a significant amount of time to execute (200 seconds in this case), even though it was not required at all in this specific case (insert of GL Entry). That's why made the function optional by using a flag. Related PR: https://github.com/frappe/erpnext/pull/31626 --- frappe/model/document.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 9b781b1999..864f2d50b4 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1092,7 +1092,9 @@ class Document(BaseDocument): self.run_method("on_update_after_submit") self.clear_cache() - self.notify_update() + + if not hasattr(self.flags, "notify_update") or self.flags.notify_update: + self.notify_update() update_global_search(self) From 2bf14bb29f1c748ced9c0592806b40724e2a9ff2 Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:08:17 +0200 Subject: [PATCH 173/201] fix: translate page numbers (#17492) --- frappe/templates/print_format/print_format.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/print_format/print_format.css b/frappe/templates/print_format/print_format.css index baaf5b087d..a10b218b94 100644 --- a/frappe/templates/print_format/print_format.css +++ b/frappe/templates/print_format/print_format.css @@ -18,7 +18,7 @@ /* page number */ {% set page_number_position = print_format.page_number.lower().replace(' ', '_') %} {% if page_number_position in ['top_left', 'top_center', 'top_right', 'bottom_left', 'bottom_center', 'bottom_right'] %} - {{ render_margin_text(page_number_position, 'counter(page) " of " counter(pages)') }} + {{ render_margin_text(page_number_position, _('{0} of {1}').format('counter(page) "', '" counter(pages)')) }} {% endif %} } From a50e0ffa085e5164bb9d04f03192f9de94d2ee40 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:52:15 +0530 Subject: [PATCH 174/201] refactor: Webform (#17232) --- cypress/integration/web_form.js | 237 +++++++++++- cypress/support/commands.js | 11 +- frappe/__init__.py | 8 + .../web_form/edit_profile/edit_profile.json | 6 +- frappe/patches.txt | 1 + frappe/patches/v14_0/update_webforms.py | 14 + frappe/public/js/frappe-web.bundle.js | 1 + .../js/frappe/form/controls/datetime.js | 1 + frappe/public/js/frappe/form/formatters.js | 2 +- frappe/public/js/frappe/web_form/web_form.js | 111 +++--- .../js/frappe/web_form/web_form_list.js | 360 ++++++++++-------- .../js/frappe/web_form/webform_script.js | 151 +++----- frappe/public/scss/desk/global.scss | 4 + frappe/public/scss/website/web_form.scss | 288 +++++++++----- frappe/templates/base.html | 7 +- frappe/tests/test_webform.py | 12 +- frappe/utils/data.py | 9 + .../doctype/web_form/templates/web_form.html | 238 ++++++------ .../web_form/templates/web_form_row.html | 4 - .../doctype/web_form/templates/web_list.html | 45 +++ .../website/doctype/web_form/test_web_form.py | 8 +- frappe/website/doctype/web_form/web_form.js | 239 ++++++++---- frappe/website/doctype/web_form/web_form.json | 197 ++++++---- frappe/website/doctype/web_form/web_form.py | 219 +++++++---- .../website/doctype/web_form/web_form_list.js | 10 + .../web_form_field/web_form_field.json | 11 +- .../doctype/web_form_list_column/__init__.py | 0 .../web_form_list_column.json | 48 +++ .../web_form_list_column.py | 9 + frappe/website/doctype/web_page/web_page.py | 4 + .../website_settings/website_settings.py | 3 + frappe/website/page_renderers/web_form.py | 12 +- frappe/website/router.py | 26 ++ frappe/website/serve.py | 3 + frappe/website/utils.py | 18 +- .../web_form/request_data/request_data.json | 8 +- .../request_to_delete_data.json | 8 +- 37 files changed, 1527 insertions(+), 806 deletions(-) create mode 100644 frappe/patches/v14_0/update_webforms.py delete mode 100644 frappe/website/doctype/web_form/templates/web_form_row.html create mode 100644 frappe/website/doctype/web_form/templates/web_list.html create mode 100644 frappe/website/doctype/web_form/web_form_list.js create mode 100644 frappe/website/doctype/web_form_list_column/__init__.py create mode 100644 frappe/website/doctype/web_form_list_column/web_form_list_column.json create mode 100644 frappe/website/doctype/web_form_list_column/web_form_list_column.py diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js index bd1c7e147e..74edee0eb9 100644 --- a/cypress/integration/web_form.js +++ b/cypress/integration/web_form.js @@ -3,24 +3,253 @@ context('Web Form', () => { cy.login(); }); + it('Create Web Form', () => { + cy.visit('/app/web-form/new'); + + cy.intercept('POST', '/api/method/frappe.desk.form.save.savedocs').as('save_form'); + + cy.fill_field('title', 'Note'); + cy.fill_field('doc_type', 'Note', 'Link'); + cy.fill_field('module', 'Website', 'Link'); + cy.click_custom_action_button('Get Fields'); + cy.click_custom_action_button('Publish'); + + cy.wait('@save_form'); + + cy.get_field('route').should('have.value', 'note'); + cy.get('.title-area .indicator-pill').contains('Published'); + }); + + it('Open Web Form (Logged in User)', () => { + cy.visit('/note'); + + cy.fill_field('title', 'Note 1'); + cy.get('.web-form-actions button').contains('Save').click(); + + cy.url().should('include', '/note/Note%201'); + + cy.visit('/note'); + cy.url().should('include', '/note/Note%201'); + }); + + it('Open Web Form (Guest)', () => { + cy.request('/api/method/logout'); + cy.visit('/note'); + + cy.url().should('include', '/note/new'); + + cy.fill_field('title', 'Guest Note 1'); + cy.get('.web-form-actions button').contains('Save').click(); + + cy.url().should('include', '/note/new'); + + cy.visit('/note'); + cy.url().should('include', '/note/new'); + }); + + it('Login Required', () => { + cy.login(); + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "Form Settings"}).click(); + cy.get('input[data-fieldname="login_required"]').check({force: true}); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/Note%201'); + + cy.call('logout'); + + cy.visit('/note'); + cy.get_open_dialog() + .get('.modal-message') + .contains('You are not permitted to access this page without login.'); + }); + + it('Show List', () => { + cy.login(); + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "List Settings"}).click(); + cy.get('input[data-fieldname="show_list"]').check(); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/list'); + cy.get('.web-list-table').should('be.visible'); + }); + + it('Show Custom List Title', () => { + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "List Settings"}).click(); + cy.fill_field('list_title', 'Note List'); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/list'); + cy.get('.web-list-header h1').should('contain.text', 'Note List'); + }); + + it('Show Custom List Columns', () => { + cy.visit('/note'); + cy.url().should('include', '/note/list'); + + cy.get('.web-list-table thead th').contains('Name'); + cy.get('.web-list-table thead th').contains('Title'); + + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "List Settings"}).click(); + + cy.get('[data-fieldname="list_columns"] .grid-footer button').contains('Add Row').as('add-row'); + + cy.get('@add-row').click(); + cy.get('[data-fieldname="list_columns"] .grid-body .rows').as('grid-rows'); + cy.get('@grid-rows').find('.grid-row:first [data-fieldname="fieldname"]').click(); + cy.get('@grid-rows').find('.grid-row:first select[data-fieldname="fieldname"]').select('Title (Data)'); + + cy.get('@add-row').click(); + cy.get('@grid-rows').find('.grid-row[data-idx="2"] [data-fieldname="fieldname"]').click(); + cy.get('@grid-rows').find('.grid-row[data-idx="2"] select[data-fieldname="fieldname"]').select('Public (Check)'); + + cy.get('@add-row').click(); + cy.get('@grid-rows').find('.grid-row:last [data-fieldname="fieldname"]').click(); + cy.get('@grid-rows').find('.grid-row:last select[data-fieldname="fieldname"]').select('Content (Text Editor)'); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/list'); + cy.get('.web-list-table thead th').contains('Title'); + cy.get('.web-list-table thead th').contains('Public'); + cy.get('.web-list-table thead th').contains('Content'); + }); + + it('Breadcrumbs', () => { + cy.visit('/note/Note 1'); + cy.get('.breadcrumb-container .breadcrumb .breadcrumb-item:first a') + .should('contain.text', 'Note').click(); + cy.url().should('include', '/note/list'); + }); + + it('Custom Breadcrumbs', () => { + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "Form Settings"}).click(); + cy.get('.form-section .section-head').contains('Customization').click(); + cy.fill_field('breadcrumbs', '[{"label": _("Notes"), "route":"note"}]', 'Code'); + cy.get('.form-section .section-head').contains('Customization').click(); + cy.save(); + + cy.visit('/note/Note 1'); + cy.get('.breadcrumb-container .breadcrumb .breadcrumb-item:first a') + .should('contain.text', 'Notes'); + }); + + it('Read Only', () => { + cy.login(); + cy.visit('/note'); + cy.url().should('include', '/note/list'); + + // Read Only Field + cy.get('.web-list-table tbody tr[id="Note 1"]').click(); + cy.get('.frappe-control[data-fieldname="title"] .control-input') + .should('have.css', 'display', 'none'); + }); + + it('Edit Mode', () => { + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "Form Settings"}).click(); + cy.get('input[data-fieldname="allow_edit"]').check(); + + cy.save(); + + cy.visit('/note/Note 1'); + cy.url().should('include', '/note/Note%201'); + + cy.get('.web-form-actions a').contains('Edit').click(); + cy.url().should('include', '/note/Note%201/edit'); + + // Editable Field + cy.get_field('title').should('have.value', 'Note 1'); + + cy.fill_field('title', ' Edited'); + cy.get('.web-form-actions button').contains('Save').click(); + cy.get_field('title').should('have.value', 'Note 1 Edited'); + }); + + it('Allow Multiple Response', () => { + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "Form Settings"}).click(); + cy.get('input[data-fieldname="allow_multiple"]').check(); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/list'); + + cy.get('.web-list-actions a:visible').contains('New').click(); + cy.url().should('include', '/note/new'); + + cy.fill_field('title', 'Note 2'); + cy.get('.web-form-actions button').contains('Save').click(); + }); + + it('Allow Delete', () => { + cy.visit('/app/web-form/note'); + + cy.findByRole("tab", {name: "Form Settings"}).click(); + cy.get('input[data-fieldname="allow_delete"]').check(); + + cy.save(); + + cy.visit('/note'); + cy.url().should('include', '/note/list'); + + cy.get('.web-list-table tbody tr[id="Note 1"] .list-col-checkbox').click(); + cy.get('.web-list-table tbody tr[id="Note 2"] .list-col-checkbox').click(); + cy.get('.web-list-actions button:visible').contains('Delete').click({force: true}); + + cy.get('.web-list-actions button').contains('Delete').should('not.be.visible'); + + cy.visit('/note'); + cy.get('.web-list-table tbody tr[id="Note 1"]').should('not.exist'); + cy.get('.web-list-table tbody tr[id="Note 2"]').should('not.exist'); + cy.get('.web-list-table tbody tr[id="Guest Note 1"]').should('exist'); + }); + it('Navigate and Submit a WebForm', () => { cy.visit('/update-profile'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + + cy.get('.web-form-actions a').contains('Edit').click(); + + cy.fill_field('last_name', '_Test User'); + cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); cy.url().should('include', '/me'); }); it('Navigate and Submit a MultiStep WebForm', () => { cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(() => { cy.visit('/update-profile-duplicate'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); + + cy.get('.web-form-actions a').contains('Edit').click(); + + cy.fill_field('last_name', '_Test User'); + cy.get('.btn-next').should('be.visible'); cy.get('.btn-next').click(); + cy.get('.btn-previous').should('be.visible'); cy.get('.btn-next').should('not.be.visible'); + cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); cy.url().should('include', '/me'); }); }); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 5ee26348e2..5424e8c6e4 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -162,7 +162,12 @@ Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => { if (fieldtype === 'Select') { cy.get('@input').select(value); } else { - cy.get('@input').type(value, {waitForAnimations: false, force: true, delay: 100}); + cy.get('@input').type(value, { + waitForAnimations: false, + parseSpecialCharSequences: false, + force: true, + delay: 100 + }); } return cy.get('@input'); }); @@ -358,6 +363,10 @@ Cypress.Commands.add('open_list_filter', () => { cy.get('.filter-popover').should('exist'); }); +Cypress.Commands.add('click_custom_action_button', (name) => { + cy.get(`.custom-actions [data-label="${encodeURIComponent(name)}"]`).click(); +}); + Cypress.Commands.add('click_action_button', (name) => { cy.findByRole('button', {name: 'Actions'}).click(); cy.get(`.actions-btn-group [data-label="${encodeURIComponent(name)}"]`).click(); diff --git a/frappe/__init__.py b/frappe/__init__.py index 3057eacd3b..cd1bfc5583 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1796,6 +1796,14 @@ def respond_as_web_page( local.response["context"] = context +def redirect(url): + """Raise a 301 redirect to url""" + from frappe.exceptions import Redirect + + flags.redirect_location = url + raise Redirect + + def redirect_to_message(title, html, http_status_code=None, context=None, indicator_color=None): """Redirects to /message?id=random Similar to respond_as_web_page, but used to 'redirect' and show message pages like success, failure, etc. with a detailed message diff --git a/frappe/core/web_form/edit_profile/edit_profile.json b/frappe/core/web_form/edit_profile/edit_profile.json index c04e705820..cedef71c0e 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.json +++ b/frappe/core/web_form/edit_profile/edit_profile.json @@ -18,9 +18,10 @@ "introduction_text": "", "is_multi_step_form": 0, "is_standard": 1, + "list_columns": [], "login_required": 1, "max_attachment_size": 0, - "modified": "2022-03-22 15:00:43.456738", + "modified": "2022-07-18 16:51:19.796411", "modified_by": "Administrator", "module": "Core", "name": "edit-profile", @@ -29,9 +30,8 @@ "route": "update-profile", "route_to_success_link": 0, "show_attachments": 0, - "show_in_grid": 0, + "show_list": 0, "show_sidebar": 0, - "sidebar_items": [], "success_message": "Profile updated successfully.", "success_url": "/me", "title": "Update Profile", diff --git a/frappe/patches.txt b/frappe/patches.txt index 437648bf9e..f79cadae87 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -194,6 +194,7 @@ frappe.patches.v14_0.remove_is_first_startup frappe.patches.v14_0.clear_long_pending_stale_logs frappe.patches.v14_0.log_settings_migration frappe.patches.v14_0.setup_likes_from_feedback +frappe.patches.v14_0.update_webforms [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v14_0/update_webforms.py b/frappe/patches/v14_0/update_webforms.py new file mode 100644 index 0000000000..46918f216e --- /dev/null +++ b/frappe/patches/v14_0/update_webforms.py @@ -0,0 +1,14 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + + +import frappe + + +def execute(): + frappe.reload_doc("website", "doctype", "web_form_list_column") + frappe.reload_doctype("Web Form") + + for web_form in frappe.db.get_all("Web Form", fields=["*"]): + if web_form.allow_multiple and not web_form.show_list: + frappe.db.set_value("Web Form", web_form.name, "show_list", True) diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js index a3bac55e23..21703f83b8 100644 --- a/frappe/public/js/frappe-web.bundle.js +++ b/frappe/public/js/frappe-web.bundle.js @@ -3,6 +3,7 @@ import "./frappe/class.js"; import "./frappe/polyfill.js"; import "./lib/moment.js"; import "./frappe/provide.js"; +import "./frappe/form/formatters.js"; import "./frappe/format.js"; import "./frappe/utils/number_format.js"; import "./frappe/utils/utils.js"; diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index a086b1b879..c266a928e6 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -14,6 +14,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co } get_start_date() { + this.value = this.value == null ? undefined : this.value; let value = frappe.datetime.convert_to_user_tz(this.value); return frappe.datetime.str_to_obj(value); } diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 5cf5a2f4f3..5a15b4fd45 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -196,7 +196,7 @@ frappe.form.formatters = { Datetime: function(value) { if(value) { return moment(frappe.datetime.convert_to_user_tz(value)) - .format(frappe.boot.sysdefaults.date_format.toUpperCase() + ' ' + frappe.boot.sysdefaults.time_format || 'HH:mm:ss'); + .format(frappe.boot.sysdefaults.date_format.toUpperCase() + ' ' + (frappe.boot.sysdefaults.time_format || 'HH:mm:ss')); } else { return ""; } diff --git a/frappe/public/js/frappe/web_form/web_form.js b/frappe/public/js/frappe/web_form/web_form.js index 11e0b782ae..21d88eac49 100644 --- a/frappe/public/js/frappe/web_form/web_form.js +++ b/frappe/public/js/frappe/web_form/web_form.js @@ -23,13 +23,14 @@ export default class WebForm extends frappe.ui.FieldGroup { this.set_sections(); this.set_field_values(); this.setup_listeners(); - if (this.introduction_text) this.set_form_description(this.introduction_text); - if (this.allow_print && !this.is_new) this.setup_print_button(); - if (this.is_new) this.setup_cancel_button(); - this.setup_primary_action(); + + if (this.is_new || this.is_form_editable) { + this.setup_primary_action(); + } + + this.setup_footer_actions(); this.setup_previous_next_button(); this.toggle_section(); - $(".link-btn").remove(); // webform client script frappe.init_client_script && frappe.init_client_script(); @@ -70,6 +71,14 @@ export default class WebForm extends frappe.ui.FieldGroup { this.sections = $(`.form-section`); } + setup_footer_actions() { + if (this.is_multi_step_form) return; + + if ($('.web-form-container').height() > 600) { + $(".web-form-footer").removeClass("hide"); + } + } + setup_previous_next_button() { let me = this; @@ -87,7 +96,7 @@ export default class WebForm extends frappe.ui.FieldGroup { $('.btn-previous').on('click', function () { let is_validated = me.validate_section(); - if (!is_validated) return; + if (!is_validated) return false; /** The eslint utility cannot figure out if this is an infinite loop in backwards and @@ -107,12 +116,13 @@ export default class WebForm extends frappe.ui.FieldGroup { } /* eslint-enable for-direction */ me.toggle_section(); + return false; }); $('.btn-next').on('click', function () { let is_validated = me.validate_section(); - if (!is_validated) return; + if (!is_validated) return false; for (let idx = me.current_section; idx < me.sections.length; idx++) { let is_empty = me.is_next_section_empty(idx); @@ -123,6 +133,7 @@ export default class WebForm extends frappe.ui.FieldGroup { } } me.toggle_section(); + return false; }); } @@ -132,56 +143,20 @@ export default class WebForm extends frappe.ui.FieldGroup { } set_default_values() { + let defaults = {}; + for (let df of this.fields) { + if (df.default) { + defaults[df.fieldname] = df.default; + } + } let values = frappe.utils.get_query_params(); delete values.new; + Object.assign(defaults, values); this.set_values(values); } - set_form_description(intro) { - let intro_wrapper = document.getElementById('introduction'); - intro_wrapper.innerHTML = intro; - intro_wrapper.classList.remove('hidden'); - } - - add_button(name, type, action, wrapper_class=".web-form-actions") { - const button = document.createElement("button"); - button.classList.add("btn", "btn-" + type, "btn-sm", "ml-2"); - button.innerHTML = name; - button.onclick = action; - document.querySelector(wrapper_class).appendChild(button); - } - - add_button_to_footer(name, type, action) { - this.add_button(name, type, action, '.web-form-footer'); - } - - add_button_to_header(name, type, action) { - this.add_button(name, type, action, '.web-form-actions'); - } - setup_primary_action() { - this.add_button_to_header(this.button_label || __("Save", null, "Button in web form"), "primary", () => - this.save() - ); - - if (!this.is_multi_step_form && $('.frappe-card').height() > 600) { - // add button on footer if page is long - this.add_button_to_footer(this.button_label || __("Save", null, "Button in web form"), "primary", () => - this.save() - ); - } - } - - setup_cancel_button() { - this.add_button_to_header(__("Cancel", null, "Button in web form"), "light", () => this.cancel()); - } - - setup_print_button() { - this.add_button_to_header( - frappe.utils.icon('print'), - "light", - () => this.print() - ); + $(".web-form-container").on("submit", () => this.save()); } validate_section() { @@ -349,18 +324,21 @@ export default class WebForm extends frappe.ui.FieldGroup { window.saving = false; } }); - return true; + return false; } - print() { - window.open(`/printview? - doctype=${this.doc_type} - &name=${this.doc.name} - &format=${this.print_format || "Standard"}`, '_blank'); + edit() { + window.location.href = window.location.pathname + "/edit"; } cancel() { - window.location.href = window.location.pathname; + let path = window.location.pathname; + if (this.is_new) { + path = path.replace('/new', ''); + } else { + path = path.replace('/edit', ''); + } + window.location.href = path; } handle_success(data) { @@ -375,12 +353,19 @@ export default class WebForm extends frappe.ui.FieldGroup { // redirect setTimeout(() => { + let path = window.location.pathname; + if (this.success_url) { - window.location.href = this.success_url; - } else if(this.login_required) { - window.location.href = - window.location.pathname + "?name=" + data.name; + path = this.success_url; + } else if (this.login_required) { + if (this.is_new && data.name) { + path = path.replace("/new", ""); + path = path + "/" + data.name; + } else if (this.is_form_editable) { + path = path.replace("/edit", ""); + } } - }, 2000); + window.location.href = path; + }, 1000); } } diff --git a/frappe/public/js/frappe/web_form/web_form_list.js b/frappe/public/js/frappe/web_form/web_form_list.js index 27e1695788..a4e7480f94 100644 --- a/frappe/public/js/frappe/web_form/web_form_list.js +++ b/frappe/public/js/frappe/web_form/web_form_list.js @@ -6,63 +6,74 @@ export default class WebFormList { constructor(opts) { Object.assign(this, opts); frappe.web_form_list = this; - this.wrapper = document.getElementById("list-table"); + this.wrapper = $(".web-list-table"); this.make_actions(); this.make_filters(); - $('.link-btn').remove(); } refresh() { - if (this.table) { - Array.from(this.table.tBodies).forEach(tbody => tbody.remove()); - let check = document.getElementById('select-all'); - if (check) - check.checked = false; - } this.rows = []; - this.page_length = 20; this.web_list_start = 0; + this.page_length = 10; frappe.run_serially([ () => this.get_list_view_fields(), () => this.get_data(), + () => this.remove_more(), () => this.make_table(), () => this.create_more() ]); } + remove_more() { + $('.more').remove(); + } + make_filters() { this.filters = {}; this.filter_input = []; - const filter_area = document.getElementById('list-filters'); + let filter_area = $('.web-list-filters'); frappe.call('frappe.website.doctype.web_form.web_form.get_web_form_filters', { web_form_name: this.web_form_name }).then(response => { let fields = response.message; + fields.length && filter_area.removeClass('hide'); fields.forEach(field => { - let col = document.createElement('div.col-sm-4'); - col.classList.add('col', 'col-sm-3'); - filter_area.appendChild(col); - if (field.default) this.add_filter(field.fieldname, field.default, field.fieldtype); + if (["Text Editor", "Text", "Small Text"].includes(field.fieldtype)) { + field.fieldtype = "Data"; + } + + if (["Table", "Signature"].includes(field.fieldtype)) { + return; + } let input = frappe.ui.form.make_control({ df: { fieldtype: field.fieldtype, fieldname: field.fieldname, options: field.options, + input_class: 'input-xs', only_select: true, label: __(field.label), onchange: (event) => { - $('#more').remove(); this.add_filter(field.fieldname, input.value, field.fieldtype); this.refresh(); } }, - parent: col, - value: field.default, + parent: filter_area, render_input: 1, + only_input: field.fieldtype == "Check" ? false : true, }); + + $(input.wrapper) + .addClass('col-md-2') + .attr("title", __(field.label)).tooltip({ + delay: { "show": 600, "hide": 100}, + trigger: "hover" + }); + + input.$input.attr("placeholder", __(field.label)); this.filter_input.push(input); }); this.refresh(); @@ -73,37 +84,65 @@ export default class WebFormList { if (!value) { delete this.filters[field]; } else { - if (fieldtype === 'Data') value = ['like', value + '%']; + if (["Data", "Currency", "Float", "Int"].includes(fieldtype)) { + value = ['like', '%' + value + '%']; + } Object.assign(this.filters, Object.fromEntries([[field, value]])); } } get_list_view_fields() { - return frappe - .call({ - method: - "frappe.website.doctype.web_form.web_form.get_in_list_view_fields", - args: { doctype: this.doctype } - }) - .then(response => (this.fields_list = response.message)); + if (this.columns) return this.columns; + + if (this.list_columns) { + this.columns = this.list_columns.map(df => { + return { + label: df.label, + fieldname: df.fieldname, + fieldtype: df.fieldtype + }; + }); + } } fetch_data() { - return frappe.call({ + let args = { method: "frappe.www.list.get_list_data", args: { doctype: this.doctype, - fields: this.fields_list.map(df => df.fieldname), limit_start: this.web_list_start, + limit: this.page_length, web_form_name: this.web_form_name, ...this.filters } - }); + }; + + if (this.no_change(args)) { + // console.log('throttled'); + return Promise.resolve(); + } + + return frappe.call(args); + } + + no_change(args) { + // returns true if arguments are same for the last 3 seconds + // this helps in throttling if called from various sources + if (this.last_args && JSON.stringify(args) === this.last_args) { + return true; + } + this.last_args = JSON.stringify(args); + setTimeout(() => { + this.last_args = null; + }, 3000); + return false; } async get_data() { let response = await this.fetch_data(); - this.data = await response.message; + if (response) { + this.data = await response.message; + } } more() { @@ -118,159 +157,145 @@ export default class WebFormList { } make_table() { - this.columns = this.fields_list.map(df => { - return { - label: df.label, - fieldname: df.fieldname, - fieldtype: df.fieldtype - }; + this.table = $(`
{{ frappe.meta.get_label(doc.ref_doctype, table_info[0]) }} {{ table_info[1] }} {{ item[0] }}{{ item[1] }}{{ item[2] }}{{ item[1] }}{{ item[2] }}
`); + + this.make_table_head(); + this.make_table_body(); + } + + make_table_head() { + let $thead = $(` + + + + + + ${__("Sr")}. + + + `); + + this.check_all = $thead.find('input.select-all'); + this.check_all.on("click", event => { + this.toggle_select_all(event.target.checked); }); - if (!this.table) { - this.table = document.createElement("table"); - this.table.classList.add("table"); - this.make_table_head(); - } + this.columns.forEach(col => { + let $tr = $thead.find("tr"); + let $th = $(`${__(col.label)}`); + $th.appendTo($tr); + }); + $thead.appendTo(this.table); + } + + make_table_body() { if (this.data.length) { + this.wrapper.empty(); + + if (this.table) { + this.table.find('tbody').remove(); + + if (this.check_all.length) { + this.check_all.prop("checked", false); + } + } + this.append_rows(this.data); - this.wrapper.appendChild(this.table); + this.table.appendTo(this.wrapper); } else { - let new_button = ""; - let empty_state = document.createElement("div"); - empty_state.classList.add("no-result", "text-muted", "flex", "justify-center", "align-center"); + if (this.wrapper.find('.no-result').length) return; + this.wrapper.empty(); frappe.has_permission(this.doctype, "", "create", () => { - new_button = ` - - `; - - empty_state.innerHTML = ` -
-
- Generic Empty State -
-

${__("No {0} found", [__(this.doctype)])}

- ${new_button} -
- `; - - this.wrapper.appendChild(empty_state); + this.setup_empty_state(); }); } } - make_table_head() { - // Create Heading - let thead = this.table.createTHead(); - let row = thead.insertRow(); + setup_empty_state() { + let new_button = ` + + `; - let th = document.createElement("th"); + let empty_state = $(` +
+
+
+ Generic Empty State +
+

${__("No {0} found", [__(this.doctype)])}

+ ${new_button} +
+
+ `); - let checkbox = document.createElement("input"); - checkbox.type = "checkbox"; - checkbox.id = "select-all"; - checkbox.onclick = event => - this.toggle_select_all(event.target.checked); - - th.appendChild(checkbox); - row.appendChild(th); - - add_heading(row, __("Sr")); - this.columns.forEach(col => { - add_heading(row, __(col.label)); - }); - - function add_heading(row, label) { - let th = document.createElement("th"); - th.innerText = label; - row.appendChild(th); - } + empty_state.appendTo(this.wrapper); } append_rows(row_data) { - const tbody = this.table.childNodes[1] || this.table.createTBody(); + let $tbody = this.table.find('tbody'); + + if (!$tbody.length) { + $tbody = $(``); + $tbody.appendTo(this.table); + } + row_data.forEach((data_item) => { - let row_element = tbody.insertRow(); - row_element.setAttribute("id", data_item.name); + let $row_element = $(``); let row = new frappe.ui.WebFormListRow({ - row: row_element, + row: $row_element, doc: data_item, columns: this.columns, serial_number: this.rows.length + 1, events: { - onEdit: () => this.open_form(data_item.name), - onSelect: () => this.toggle_delete() + on_edit: () => this.open_form(data_item.name), + on_select: () => { + this.toggle_new(); + this.toggle_delete(); + } } }); this.rows.push(row); + $row_element.appendTo($tbody); }); } make_actions() { - const actions = document.querySelector(".list-view-actions"); + const actions = $(".web-list-actions"); frappe.has_permission(this.doctype, "", "delete", () => { - this.addButton(actions, "delete-rows", "danger", true, "Delete", () => - this.delete_rows() - ); + this.add_button(actions, "delete-rows", "danger", true, "Delete", () => this.delete_rows()); }); - - this.addButton( - actions, - "new", - "primary", - false, - "New", - () => (window.location.href = window.location.pathname + "?new=1") - ); } - addButton(wrapper, id, type, hidden, name, action) { - if (document.getElementById(id)) return; - const button = document.createElement("button"); - if (type == "secondary") { - button.classList.add( - "btn", - "btn-secondary", - "btn-sm", - "ml-2" - ); - } - else if (type == "danger") { - button.classList.add( - "btn", - "btn-danger", - "button-delete", - "btn-sm", - "ml-2" - ); - } - else { - button.classList.add("btn", "btn-primary", "btn-sm", "ml-2"); - } + add_button(wrapper, name, type, hidden, text, action) { + if ($(`.${name}`).length) return; - button.id = id; - button.innerText = name; - button.hidden = hidden; + hidden = hidden ? "hide" : ""; + type = type == "danger" ? "danger button-delete" : type; - button.onclick = action; - wrapper.appendChild(button); + let button = $(` + + `); + + button.on("click", () => action()); + button.appendTo(wrapper); } create_more() { if (this.rows.length >= this.page_length) { - const footer = document.querySelector(".list-view-footer"); - this.addButton(footer, "more", "secondary", false, "More", () => this.more()); + const footer = $(".web-list-footer"); + this.add_button(footer, "more", "secondary", false, "Load More", () => this.more()); } } @@ -279,7 +304,12 @@ export default class WebFormList { } open_form(name) { - window.location.href = window.location.pathname + "?name=" + name; + let path = window.location.pathname; + if (path.includes('/list')) { + path = path.replace('/list', ''); + } + + window.location.href = path + "/" + name; } get_selected() { @@ -287,9 +317,15 @@ export default class WebFormList { } toggle_delete() { - if (!this.settings.allow_delete) return - let btn = document.getElementById("delete-rows"); - btn.hidden = !this.get_selected().length; + if (!this.settings.allow_delete) return; + let btn = $(".delete-rows"); + !this.get_selected().length ? btn.addClass('hide') : btn.removeClass('hide'); + } + + toggle_new() { + if (!this.settings.allow_delete) return; + let btn = $(".button-new"); + this.get_selected().length ? btn.addClass('hide') : btn.removeClass('hide'); } delete_rows() { @@ -305,8 +341,9 @@ export default class WebFormList { } }) .then(() => { - this.refresh() - this.toggle_delete() + this.refresh(); + this.toggle_delete(); + this.toggle_new(); }); } }; @@ -319,40 +356,37 @@ frappe.ui.WebFormListRow = class WebFormListRow { make_row() { // Add Checkboxes - let cell = this.row.insertCell(); - cell.classList.add('list-col-checkbox'); + let $cell = $(``); - this.checkbox = document.createElement("input"); - this.checkbox.type = "checkbox"; - this.checkbox.onclick = event => { + this.checkbox = $(``); + this.checkbox.on("click", event => { this.toggle_select(event.target.checked); event.stopImmediatePropagation(); - } - - cell.appendChild(this.checkbox); + }); + this.checkbox.appendTo($cell); + $cell.appendTo(this.row); // Add Serial Number - let serialNo = this.row.insertCell(); - serialNo.classList.add('list-col-serial'); - serialNo.innerText = this.serial_number; + let serialNo = $(`${__(this.serial_number)}`); + serialNo.appendTo(this.row); this.columns.forEach(field => { - let cell = this.row.insertCell(); let formatter = frappe.form.get_formatter(field.fieldtype); - cell.innerHTML = this.doc[field.fieldname] && + let value = this.doc[field.fieldname] && __(formatter(this.doc[field.fieldname], field, {only_value: 1}, this.doc)) || ""; + let cell = $(`${value}`); + cell.appendTo(this.row); }); - this.row.onclick = () => this.events.onEdit(); - this.row.style.cursor = "pointer"; + this.row.on("click", () => this.events.on_edit()); } toggle_select(checked) { - this.checkbox.checked = checked; - this.events.onSelect(checked); + this.checkbox.prop("checked", checked); + this.events.on_select(checked); } is_selected() { - return this.checkbox.checked; + return this.checkbox.prop("checked"); } }; diff --git a/frappe/public/js/frappe/web_form/webform_script.js b/frappe/public/js/frappe/web_form/webform_script.js index 30ff03cb5d..31fecc778c 100644 --- a/frappe/public/js/frappe/web_form/webform_script.js +++ b/frappe/public/js/frappe/web_form/webform_script.js @@ -2,23 +2,15 @@ import WebFormList from './web_form_list' import WebForm from './web_form' frappe.ready(function() { - let query_params = frappe.utils.get_query_params(); - let wrapper = $(".web-form-wrapper"); - let is_list = parseInt(wrapper.data('is-list')) || query_params.is_list; - let webform_doctype = wrapper.data('web-form-doctype'); - let webform_name = wrapper.data('web-form'); - let login_required = parseInt(wrapper.data('login-required')); - let allow_delete = parseInt(wrapper.data('allow-delete')); - let doc_name = query_params.name || ''; - let is_new = query_params.new; + let web_form_doc = frappe.web_form_doc; + let reference_doc = frappe.reference_doc; - if (login_required) show_login_prompt(); - else if (is_list) show_grid(); - else show_form(webform_doctype, webform_name, is_new); + show_login_prompt(); - document.querySelector("body").style.display = "block"; + web_form_doc.is_list ? show_list() : show_form(); function show_login_prompt() { + if (frappe.session.user != "Guest" || !web_form_doc.login_required) return; const login_required = new frappe.ui.Dialog({ title: __("Not Permitted"), primary_action_label: __("Login"), @@ -30,102 +22,79 @@ frappe.ready(function() { login_required.set_message(__("You are not permitted to access this page without login.")); } - function show_grid() { + function show_list() { new WebFormList({ - parent: wrapper, - doctype: webform_doctype, - web_form_name: webform_name, + doctype: web_form_doc.doc_type, + web_form_name: web_form_doc.name, + list_columns: web_form_doc.list_columns, settings: { - allow_delete + allow_delete: web_form_doc.allow_delete } }); } function show_form() { let web_form = new WebForm({ - parent: wrapper, - is_new, - web_form_name: webform_name, + parent: $(".web-form-wrapper"), + is_new: web_form_doc.is_new, + is_form_editable: web_form_doc.is_form_editable, + web_form_name: web_form_doc.name, }); + let doc = reference_doc || {}; + setup_fields(web_form_doc, doc); - get_data().then(r => { - const data = setup_fields(r.message); - let web_form_doc = data.web_form; + web_form.prepare(web_form_doc, doc); + web_form.make(); - // if (web_form_doc.name && web_form_doc.allow_edit === 0) { - // if (!window.location.href.includes("?new=1")) { - // window.location.replace(window.location.pathname + "?new=1"); - // } - // } - let doc = r.message.doc || build_doc(r.message); - web_form.prepare(web_form_doc, r.message.doc && web_form_doc.allow_edit === 1 ? r.message.doc : {}); - web_form.make(); + if (web_form_doc.is_new) { web_form.set_default_values(); - }) - - function build_doc(form_data) { - let doc = {}; - form_data.web_form.web_form_fields.forEach(df => { - if (df.default) return doc[df.fieldname] = df.default; - }); - return doc; } - function get_data() { - return frappe.call({ - method: "frappe.website.doctype.web_form.web_form.get_form_data", - args: { - doctype: webform_doctype, - docname: doc_name, - web_form_name: webform_name - }, - freeze: true - }); - } + $(".file-size").each(function () { + $(this).text(frappe.form.formatters.FileSize($(this).text())); + }); + } - function setup_fields(form_data) { - form_data.web_form.web_form_fields.map(df => { - df.is_web_form = true; - if (df.fieldtype === "Table") { - df.get_data = () => { - let data = []; - if (form_data.doc) { - data = form_data.doc[df.fieldname]; - } - return data; - }; - - df.fields = form_data[df.fieldname]; - $.each(df.fields || [], function(_i, field) { - if (field.fieldtype === "Link") { - field.only_select = true; - } - field.is_web_form = true; - }); - - if (df.fieldtype === "Attach") { - df.is_private = true; + function setup_fields(web_form_doc, doc_data) { + web_form_doc.web_form_fields.forEach(df => { + df.is_web_form = true; + df.read_only = !web_form_doc.is_new && !web_form_doc.is_form_editable; + if (df.fieldtype === "Table") { + df.get_data = () => { + let data = []; + if (doc_data && doc_data[df.fieldname]) { + return doc_data[df.fieldname]; } + return data; + }; - delete df.parent; - delete df.parentfield; - delete df.parenttype; - delete df.doctype; - - return df; - } - if (df.fieldtype === "Link") { - df.only_select = true; - } - if (["Attach", "Attach Image"].includes(df.fieldtype)) { - if (typeof df.options !== "object") { - df.options = {}; + $.each(df.fields || [], function(_i, field) { + if (field.fieldtype === "Link") { + field.only_select = true; } - df.options.disable_file_browser = true; - } - }); + field.is_web_form = true; + }); - return form_data; - } + if (df.fieldtype === "Attach") { + df.is_private = true; + } + + delete df.parent; + delete df.parentfield; + delete df.parenttype; + delete df.doctype; + + return df; + } + if (df.fieldtype === "Link") { + df.only_select = true; + } + if (["Attach", "Attach Image"].includes(df.fieldtype)) { + if (typeof df.options !== "object") { + df.options = {}; + } + df.options.disable_file_browser = true; + } + }); } }); diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 1e68f374c4..0d7ca9ac06 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -76,6 +76,10 @@ a.badge-hover { text-decoration: underline; } +.pointer { + cursor: pointer; +} + .inline-block { display: inline-block; } diff --git a/frappe/public/scss/website/web_form.scss b/frappe/public/scss/website/web_form.scss index 010182e1e5..77c3d21880 100644 --- a/frappe/public/scss/website/web_form.scss +++ b/frappe/public/scss/website/web_form.scss @@ -5,37 +5,212 @@ max-width: 800px; margin: auto; - .frappe-card { - padding: 1rem; + h1 { + font-size: 1.9rem; + margin-top: 0; + margin-bottom: 0; + } - h1 { - font-size: 1.9rem; - margin-top: 0; - margin-bottom: 0; - } + .web-form-container { + border: 1px solid var(--dark-border-color); + border-radius: var(--border-radius-md); + padding: 2rem; - .web-form-head { - margin: 0 -1rem; - padding: 0 1rem 1rem 1rem; - margin-bottom: 1rem; + .web-form-header { + display: flex; + justify-content: space-between; + margin: 0 -2rem 1rem; + padding: 0 2rem 1rem; border-bottom: 1px solid var(--border-color); + + .web-form-actions { + align-self: center; + } } - #introduction { - margin-bottom: 2rem; - } - - #introduction p { + .web-form-introduction { color: var(--text-muted); + margin-bottom: 2rem; + + p { + color: var(--text-muted); + } } - .web-form-actions button { - margin-top: 0.1rem; + .web-form-wrapper { + .form-control { + color: var(--text-color); + background-color: var(--control-bg); + } + + .form-section { + .section-head { + font-weight: bold; + font-size: var(--text-xl); + padding: var(--padding-md) 0; + } + } + + .form-column { + padding: 0 var(--padding-md); + + &:first-child { + padding-left: 0; + } + + &:last-child { + padding-right: 0; + } + + @include media-breakpoint-down(sm) { + padding: 0; + } + } + } + + .web-form-footer { + text-align: right; + } + + .attachments { + margin: 1rem -2rem 0; + padding: 1rem 2rem 0; + border-top: 1px solid var(--border-color); + + .attachment { + display: flex; + justify-content: space-between; + gap: 6px; + max-width: 300px; + color: var(--text-muted); + font-size: var(--text-md); + + &:hover { + text-decoration: none; + .file-name span { + text-decoration: underline; + } + } + } } } - .frappe-card.list-card { - min-height: 400px; + .web-list-container { + min-height: 470px; + border: 1px solid var(--dark-border-color); + border-radius: var(--border-radius-md); + padding: 2rem; + + .web-list-header { + display: flex; + justify-content: space-between; + + .web-list-actions { + align-self: center; + } + } + + .web-list-filters { + display: flex; + flex-wrap: wrap; + margin: 1rem -2rem 0; + padding: 1rem 2rem 0; + border-top: 1px solid var(--border-color); + gap: 10px; + + .form-group.frappe-control { + min-width: 145px; + padding: 0px; + margin: 0px; + align-self: center; + + .checkbox { + .input-xs { + height: var(--checkbox-size); + } + + .help-box { + display: none; + } + } + + .input-xs { + height: 28px; + line-height: 1.2; + } + } + } + + .web-list-table { + overflow: auto; + margin: 1rem -2rem 0; + + .table { + border-bottom: 1px solid var(--border-color); + border-top: 1px solid var(--border-color); + + thead tr { + th { + border: 0; + font-size: 13px; + font-weight: normal; + color: var(--text-muted); + + &:first-child { + padding-left: 1.5rem; + } + + &:last-child { + padding-right: 1.5rem; + } + + input[type="checkbox"] { + margin-bottom: -2px; + } + } + } + + tbody tr { + color: var(--text-color); + cursor: pointer; + + td { + font-size: 13px; + border-top: 1px solid var(--border-color); + + &:first-child { + padding-left: 1.5rem; + } + + &:last-child { + padding-right: 1.5rem; + } + } + } + + input[type="checkbox"] { + margin-left: 0.5rem; + margin-top: 2px; + } + + .list-col-checkbox { + width: 1rem; + } + + .list-col-serial { + width: 1.5rem; + } + } + + .no-result { + min-height: 330px; + border-top: 1px solid var(--border-color); + } + } + + .web-list-footer { + text-align: right; + } } .breadcrumb-container.container { @@ -45,76 +220,3 @@ } } } - -.web-form-wrapper { - .form-control { - color: var(--text-color); - background-color: var(--control-bg); - } - - .form-section { - .section-head { - font-weight: bold; - font-size: var(--text-xl); - padding: var(--padding-md) 0; - } - } - - .form-column { - padding: 0 var(--padding-md); - - &:first-child { - padding-left: 0; - } - - &:last-child { - padding-right: 0; - } - - @include media-breakpoint-down(sm) { - padding: 0; - } - } -} - -.list-table { - margin-left: -1rem; - margin-right: -1rem; - - .table { - thead { - th { - border: 0; - font-size: 13px; - font-weight: normal; - color: var(--text-muted); - - input[type="checkbox"] { - margin-bottom: -2px; - } - } - } - - tr { - color: var(--text-color); - - td { - font-size: 13px; - border-top: 1px solid var(--border-color); - } - } - - input[type="checkbox"] { - margin-left: 0.5rem; - margin-top: 2px; - } - - .list-col-checkbox { - width: 1rem; - } - - .list-col-serial { - width: 1.5rem; - } - } -} diff --git a/frappe/templates/base.html b/frappe/templates/base.html index b11b775179..e3bfea559e 100644 --- a/frappe/templates/base.html +++ b/frappe/templates/base.html @@ -96,12 +96,7 @@ {% block base_scripts %} diff --git a/frappe/tests/test_webform.py b/frappe/tests/test_webform.py index d95e4a7498..51868dfdb1 100644 --- a/frappe/tests/test_webform.py +++ b/frappe/tests/test_webform.py @@ -8,17 +8,17 @@ from frappe.www.list import get_list_context class TestWebform(unittest.TestCase): def test_webform_publish_functionality(self): - edit_profile = frappe.get_doc("Web Form", "edit-profile") + request_data = frappe.get_doc("Web Form", "request-data") # publish webform - edit_profile.published = True - edit_profile.save() - set_request(method="GET", path="update-profile") + request_data.published = True + request_data.save() + set_request(method="GET", path="request-data/new") response = get_response() self.assertEqual(response.status_code, 200) # un-publish webform - edit_profile.published = False - edit_profile.save() + request_data.published = False + request_data.save() response = get_response() self.assertEqual(response.status_code, 404) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 8a970e57cc..262cc3fc7d 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1951,6 +1951,15 @@ def generate_hash(*args, **kwargs) -> str: return frappe.generate_hash(*args, **kwargs) +def dict_with_keys(dict, keys): + """Returns a new dict with a subset of keys""" + out = {} + for key in dict: + if key in keys: + out[key] = dict[key] + return out + + def guess_date_format(date_string: str) -> str: DATE_FORMATS = [ r"%d/%b/%y", diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html index 96072a19ea..5b35e6b5b4 100644 --- a/frappe/website/doctype/web_form/templates/web_form.html +++ b/frappe/website/doctype/web_form/templates/web_form.html @@ -1,149 +1,129 @@ {% extends "templates/web.html" %} -{% block title %}{{ _(title) }}{% endblock %} - {% block breadcrumbs %}{% endblock %} -{% macro container_attributes() %} -data-web-form="{{ name }}" data-web-form-doctype="{{ doc_type }}" data-login-required="{{ frappe.utils.cint(login_required and frappe.session.user=='Guest') }}" data-is-list="{{ frappe.utils.cint(is_list) }}" data-allow-delete="{{ allow_delete }}" +{% macro action_buttons() %} + {% if allow_print and not is_new %} + {% set print_format_url = "/printview?doctype=" + doc_type + "&name=" + doc_name + "&format=" + print_format %} + + + + + {% endif %} + + {% if allow_edit and doc_name and not is_form_editable %} + + {{ _("Edit", null, "Button in web form") }} + {% endif %} + + {% if is_new or is_form_editable %} + + {{ _("Cancel", null, "Button in web form") }} + + + {% endif %} {% endmacro %} {% block page_content %} -{% if has_header and login_required and allow_multiple %} - -{% include "templates/includes/breadcrumbs.html" %} -{% else %} -
-{% endif %} - - -
- {% if is_list %} - -
-

{{ _(title) }}

-
-
- -
-
-
-
- {% else %} - -
-

{{ _(title) }}

-
-
-
- -
-
-
- - {% if show_attachments and not frappe.form_dict.new and attachments %} -
-
{{ _("Attachments") }}
- {% for attachment in attachments %} -
- -
-
- {{ attachment.file_name }} -
-
- {{ attachment.file_size }} -
-
- -
- {% endfor %} -
- {% endif %} {# attachments #} - + + {% if has_header and login_required and show_list %} + {% include "templates/includes/breadcrumbs.html" %} + {% else %} +
{% endif %} -
-{% if allow_comments and not frappe.form_dict.new and not is_list -%} - -
- {% include 'templates/includes/comments/comments.html' %} -
-{%- else -%} -
-{%- endif %} {# comments #} + +
+
+

{{ _(title) }}

+
+ {{ action_buttons() }} +
+
+
+ {% if introduction_text %} +
{{ introduction_text }}
+ {% endif %} +
+
+ {{ action_buttons() }} +
+
+ + + {% if show_attachments and not is_new and attachments %} +
+
{{ _("Attachments") }}
+ {% for attachment in attachments %} + +
+ + {{ attachment.file_name }} +
+
{{ attachment.file_size }}
+ + {% endfor %} +
+ {% endif %} {# attachments #} +
+ + + {% if allow_comments and not is_new and not is_list -%} +
+

{{ _("Comments") }}

+ {% include 'templates/includes/comments/comments.html' %} +
+ {%- else -%} +
+ {%- endif %} {# comments #} {% endblock page_content %} {% block script %} - -{{ include_script("controls.bundle.js") }} -{% if is_list %} -{{ include_script("dialog.bundle.js") }} -{{ include_script("web_form.bundle.js") }} -{{ include_script("bootstrap-4-web.bundle.js") }} -{% else %} -{{ include_script("dialog.bundle.js") }} - - -{{ include_script("web_form.bundle.js") }} -{{ include_script("bootstrap-4-web.bundle.js") }} - -{% if client_script %} -frappe.init_client_script = () => { - try { - {{ client_script }} - } catch(e) { - console.error('Error in web form client script'); - console.error(e); - } -} -{% endif %} + + -{% if script is defined %} - {{ script }} -{% endif %} - -{% endif %} + {{ include_script("controls.bundle.js") }} + {{ include_script("dialog.bundle.js") }} + {{ include_script("web_form.bundle.js") }} + {{ include_script("bootstrap-4-web.bundle.js") }} + + {% endblock script %} {% block style %} -{% if not is_list %} -{{ include_style('web_form.bundle.css') }} -{% endif %} - - + {% endblock %} diff --git a/frappe/website/doctype/web_form/templates/web_form_row.html b/frappe/website/doctype/web_form/templates/web_form_row.html deleted file mode 100644 index 2b999819cb..0000000000 --- a/frappe/website/doctype/web_form/templates/web_form_row.html +++ /dev/null @@ -1,4 +0,0 @@ - - \ No newline at end of file diff --git a/frappe/website/doctype/web_form/templates/web_list.html b/frappe/website/doctype/web_form/templates/web_list.html new file mode 100644 index 0000000000..2ec6edaf1c --- /dev/null +++ b/frappe/website/doctype/web_form/templates/web_list.html @@ -0,0 +1,45 @@ +{% extends "templates/web.html" %} + +{% block breadcrumbs %}{% endblock %} + +{% block page_content %} + +
+ +
+

{{ _(list_title or title) }}

+
+ {%- if allow_multiple -%} + New + {%- endif -%} +
+
+
+
+
+
+{% endblock page_content %} + +{% block script %} + + + {{ include_script("controls.bundle.js") }} + {{ include_script("dialog.bundle.js") }} + {{ include_script("web_form.bundle.js") }} + {{ include_script("bootstrap-4-web.bundle.js") }} +{% endblock script %} + +{% block style %} + +{% endblock %} \ No newline at end of file diff --git a/frappe/website/doctype/web_form/test_web_form.py b/frappe/website/doctype/web_form/test_web_form.py index 5689bdbeef..13c73d1f14 100644 --- a/frappe/website/doctype/web_form/test_web_form.py +++ b/frappe/website/doctype/web_form/test_web_form.py @@ -4,6 +4,7 @@ import json import unittest import frappe +from frappe.utils import set_request from frappe.website.doctype.web_form.web_form import accept from frappe.website.serve import get_response_content @@ -68,8 +69,9 @@ class TestWebForm(unittest.TestCase): ) def test_webform_render(self): - content = get_response_content("request-data") - self.assertIn("

Request Data

", content) + set_request(method="GET", path="manage-events/new") + content = get_response_content("manage-events/new") + self.assertIn("

New Manage Events

", content) self.assertIn('data-doctype="Web Form"', content) - self.assertIn('data-path="request-data"', content) + self.assertIn('data-path="manage-events/new"', content) self.assertIn('source-type="Generator"', content) diff --git a/frappe/website/doctype/web_form/web_form.js b/frappe/website/doctype/web_form/web_form.js index 1f27b350be..63b71d35b4 100644 --- a/frappe/website/doctype/web_form/web_form.js +++ b/frappe/website/doctype/web_form/web_form.js @@ -1,89 +1,149 @@ -frappe.web_form = { - set_fieldname_select: function(frm) { - return new Promise(resolve => { - var me = this, - doc = frm.doc; - if (doc.doc_type) { - frappe.model.with_doctype(doc.doc_type, function() { - var fields = $.map(frappe.get_doc("DocType", frm.doc.doc_type).fields, function(d) { - if (frappe.model.no_value_type.indexOf(d.fieldtype) === -1 || - d.fieldtype === 'Table') { - return { label: d.label + ' (' + d.fieldtype + ')', value: d.fieldname }; - } else { - return null; - } - }); - var currency_fields = $.map(frappe.get_doc("DocType", frm.doc.doc_type).fields, function(d) { - if (d.fieldtype === 'Currency' || d.fieldtype === 'Float') { - return { label: d.label, value: d.fieldname }; - } else { - return null; - } - }); - - frm.fields_dict.web_form_fields.grid.update_docfield_property( - 'fieldname', 'options', fields - ); - frappe.meta.get_docfield("Web Form", "amount_field", frm.doc.name).options = [""].concat(currency_fields); - frm.refresh_field("amount_field"); - resolve(); - }); - } - }); - } -}; - frappe.ui.form.on("Web Form", { refresh: function(frm) { // show is-standard only if developer mode frm.get_field("is_standard").toggle(frappe.boot.developer_mode); - frappe.web_form.set_fieldname_select(frm); - if (frm.doc.is_standard && !frappe.boot.developer_mode) { frm.set_read_only(); frm.disable_save(); } + render_list_settings_message(frm); - frm.add_custom_button(__('Get Fields'), () => { - let webform_fieldtypes = frappe.meta.get_field('Web Form Field', 'fieldtype').options.split('\n'); - let fieldnames = (frm.doc.web_form_fields || []).map(d => d.fieldname); - frappe.model.with_doctype(frm.doc.doc_type, () => { - let meta = frappe.get_meta(frm.doc.doc_type); - for (let field of meta.fields) { - if (webform_fieldtypes.includes(field.fieldtype) - && !fieldnames.includes(field.fieldname)) { - frm.add_child('web_form_fields', { - fieldname: field.fieldname, - label: field.label, - fieldtype: field.fieldtype, - options: field.options, - reqd: field.reqd, - default: field.default, - read_only: field.read_only || field.is_virtual, - depends_on: field.depends_on, - mandatory_depends_on: field.mandatory_depends_on, - read_only_depends_on: field.read_only_depends_on, - hidden: field.hidden, - description: field.description + frm.trigger('set_fields'); + frm.trigger('add_get_fields_button'); + frm.trigger('add_publish_button'); + }, + + login_required: function(frm) { + render_list_settings_message(frm); + }, + + validate: function(frm) { + if (!frm.doc.login_required) { + frm.set_value("allow_multiple", 0); + frm.set_value("allow_edit", 0); + frm.set_value("show_list", 0); + } + + !frm.doc.allow_multiple && frm.set_value("allow_delete", 0); + frm.doc.allow_multiple && frm.set_value("show_list", 1); + + if (!frm.doc.web_form_fields) { + frm.scroll_to_field('web_form_fields'); + frappe.throw(__("Atleast one field is required in Web Form Fields Table")); + } + }, + + add_publish_button(frm) { + frm.add_custom_button(frm.doc.published ? __("Unpublish") : __("Publish"), () => { + frm.set_value("published", !frm.doc.published); + frm.save(); + }); + }, + + add_get_fields_button(frm) { + frm.add_custom_button(__("Get Fields"), () => { + let webform_fieldtypes = frappe.meta + .get_field("Web Form Field", "fieldtype") + .options.split("\n"); + + let added_fields = (frm.doc.fields || []).map(d => d.fieldname); + + get_fields_for_doctype(frm.doc.doc_type).then(fields => { + for (let df of fields) { + if ( + webform_fieldtypes.includes(df.fieldtype) && + !added_fields.includes(df.fieldname) && + !df.hidden + ) { + frm.add_child("web_form_fields", { + fieldname: df.fieldname, + label: df.label, + fieldtype: df.fieldtype, + options: df.options, + reqd: df.reqd, + default: df.default, + read_only: df.read_only, + depends_on: df.depends_on, + mandatory_depends_on: df.mandatory_depends_on, + read_only_depends_on: df.read_only_depends_on, }); } } - frm.refresh(); + frm.refresh_field('web_form_fields'); + frm.scroll_to_field('web_form_fields'); }); }); }, + set_fields(frm) { + let doc = frm.doc; + + let update_options = options => { + [ + frm.fields_dict.web_form_fields.grid, + frm.fields_dict.list_columns.grid + ].forEach(obj => { + obj.update_docfield_property("fieldname", "options", options); + }); + }; + + if (!doc.doc_type) { + update_options([]); + frm.set_df_property("amount_field", "options", []); + return; + } + + update_options([`Fetching fields from ${doc.doc_type}...`]); + + get_fields_for_doctype(doc.doc_type).then(fields => { + let as_select_option = df => ({ + label: df.label + " (" + df.fieldtype + ")", + value: df.fieldname + }); + update_options(fields.map(as_select_option)); + + let currency_fields = fields + .filter(df => ["Currency", "Float"].includes(df.fieldtype)) + .map(as_select_option); + if (!currency_fields.length) { + currency_fields = [ + { + label: `No currency fields in ${doc.doc_type}`, + value: "", + disabled: true + } + ]; + } + frm.set_df_property("amount_field", "options", currency_fields); + }); + }, + title: function(frm) { if (frm.doc.__islocal) { var page_name = frm.doc.title.toLowerCase().replace(/ /g, "-"); frm.set_value("route", page_name); - frm.set_value("success_url", "/" + page_name); } }, doc_type: function(frm) { - frappe.web_form.set_fieldname_select(frm); + frm.trigger('set_fields'); + }, + + allow_multiple: function(frm) { + frm.doc.allow_multiple && frm.set_value("show_list", 1); + } +}); + + +frappe.ui.form.on("Web Form List Column", { + fieldname: function(frm, doctype, name) { + let doc = frappe.get_doc(doctype, name); + let df = frappe.meta.get_docfield(frm.doc.doc_type, doc.fieldname); + if (!df) return; + doc.fieldtype = df.fieldtype; + doc.label = df.label; + frm.refresh_field("list_columns"); } }); @@ -93,22 +153,61 @@ frappe.ui.form.on("Web Form Field", { var doc = frappe.get_doc(doctype, name); if (['Section Break', 'Column Break', 'Page Break'].includes(doc.fieldtype)) { doc.fieldname = ''; + doc.options = ""; frm.refresh_field("web_form_fields"); } }, fieldname: function(frm, doctype, name) { - var doc = frappe.get_doc(doctype, name); - var df = $.map(frappe.get_doc("DocType", frm.doc.doc_type).fields, function(d) { - return doc.fieldname == d.fieldname ? d : null; - })[0]; + let doc = frappe.get_doc(doctype, name); + let df = frappe.meta.get_docfield(frm.doc.doc_type, doc.fieldname); + if (!df) return; doc.label = df.label; - doc.reqd = df.reqd; + doc.fieldtype = df.fieldtype; doc.options = df.options; - doc.fieldtype = frappe.meta.get_docfield("Web Form Field", "fieldtype") - .options.split("\n").indexOf(df.fieldtype) === -1 ? "Data" : df.fieldtype; - doc.description = df.description; - doc["default"] = df["default"]; + doc.reqd = df.reqd; + doc.default = df.default; + doc.read_only = df.read_only; + doc.depends_on = df.depends_on; + doc.mandatory_depends_on = df.mandatory_depends_on; + doc.read_only_depends_on = df.read_only_depends_on; + frm.refresh_field("web_form_fields"); } }); + + +function get_fields_for_doctype(doctype) { + return new Promise(resolve => + frappe.model.with_doctype(doctype, resolve) + ).then(() => { + return frappe.meta.get_docfields(doctype).filter(df => { + return ( + (frappe.model.is_value_type(df.fieldtype) && + !["lft", "rgt"].includes(df.fieldname)) || + ["Table", "Table Multiselect"].includes(df.fieldtype) + ); + }); + }); +} + +function render_list_settings_message(frm) { + // render list setting message + if (frm.fields_dict['list_setting_message'] && !frm.doc.login_required) { + const switch_to_form_settings_tab = ` + + ${__("Form Settings Tab")} + + `; + $(frm.fields_dict['list_setting_message'].wrapper) + .html($( + `
+ ${__("Login is required to see web form list view. Enable login_required from {0} to see list settings", [switch_to_form_settings_tab])} +
` + )) + .find('span') + .click(() => frm.scroll_to_field('login_required')); + } else { + $(frm.fields_dict['list_setting_message'].wrapper).empty(); + } +} diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index 08b2854059..0872c1d654 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -5,43 +5,51 @@ "document_type": "Document", "engine": "InnoDB", "field_order": [ + "title_and_route_tab", "title", "route", + "published", + "column_break_4", "doc_type", "module", - "column_break_4", "is_standard", - "is_multi_step_form", - "published", + "introduction", + "introduction_text", + "form_settings_tab", "login_required", - "route_to_success_link", - "allow_edit", + "is_multi_step_form", "allow_multiple", - "apply_document_permissions", - "show_in_grid", + "allow_edit", "allow_delete", + "column_break_18", + "apply_document_permissions", "allow_print", "print_format", "allow_comments", "show_attachments", "allow_incomplete", - "introduction", - "introduction_text", - "fields", + "form_fields", "web_form_fields", "max_attachment_size", - "client_script_section", - "client_script", - "custom_css_section", - "custom_css", "actions", + "breadcrumbs", "button_label", + "column_break_29", "success_message", + "route_to_success_link", "success_url", - "sidebar_settings", + "list_settings_tab", + "list_setting_message", + "show_list", + "list_title", + "list_columns", + "sidebar_settings_tab", "show_sidebar", - "sidebar_items", - "payments", + "website_sidebar", + "scripting_style_tab", + "client_script", + "custom_css", + "payments_tab", "accept_payment", "payment_gateway", "payment_button_label", @@ -50,10 +58,7 @@ "amount_based_on_field", "amount_field", "amount", - "currency", - "advanced", - "web_page_link_text", - "breadcrumbs" + "currency" ], "fields": [ { @@ -118,25 +123,18 @@ "depends_on": "login_required", "fieldname": "allow_edit", "fieldtype": "Check", - "label": "Allow Edit" + "label": "Allow Editing After Submit" }, { "default": "0", "depends_on": "login_required", "fieldname": "allow_multiple", "fieldtype": "Check", - "label": "Allow Multiple" + "label": "Allow Multiple Responses" }, { "default": "0", - "depends_on": "allow_multiple", - "fieldname": "show_in_grid", - "fieldtype": "Check", - "label": "Show as Grid" - }, - { - "default": "0", - "depends_on": "allow_multiple", + "depends_on": "eval: doc.allow_multiple && doc.login_required", "fieldname": "allow_delete", "fieldtype": "Check", "label": "Allow Delete" @@ -187,11 +185,6 @@ "ignore_xss_filter": 1, "label": "Introduction" }, - { - "fieldname": "fields", - "fieldtype": "Section Break", - "label": "Fields" - }, { "fieldname": "web_form_fields", "fieldtype": "Table", @@ -203,13 +196,6 @@ "fieldtype": "Int", "label": "Max Attachment Size (in MB)" }, - { - "collapsible": 1, - "collapsible_depends_on": "client_script", - "fieldname": "client_script_section", - "fieldtype": "Section Break", - "label": "Client Script" - }, { "description": "For help see Client Script API and Examples", "fieldname": "client_script", @@ -220,13 +206,13 @@ "collapsible": 1, "fieldname": "actions", "fieldtype": "Section Break", - "label": "Actions" + "label": "Customization" }, { "default": "Save", "fieldname": "button_label", "fieldtype": "Data", - "label": "Button Label" + "label": "Submit Button Label" }, { "description": "Message to be displayed on successful completion (only for Guest users)", @@ -235,36 +221,18 @@ "label": "Success Message" }, { + "depends_on": "route_to_success_link", "description": "Go to this URL after completing the form", "fieldname": "success_url", "fieldtype": "Data", "label": "Success URL" }, - { - "collapsible": 1, - "fieldname": "sidebar_settings", - "fieldtype": "Section Break", - "label": "Sidebar Settings" - }, { "default": "0", "fieldname": "show_sidebar", "fieldtype": "Check", "label": "Show Sidebar" }, - { - "fieldname": "sidebar_items", - "fieldtype": "Table", - "label": "Sidebar Items", - "options": "Portal Menu Item" - }, - { - "collapsible": 1, - "collapsible_depends_on": "accept_payment", - "fieldname": "payments", - "fieldtype": "Section Break", - "label": "Payments" - }, { "default": "0", "fieldname": "accept_payment", @@ -321,18 +289,6 @@ "label": "Currency", "options": "Currency" }, - { - "collapsible": 1, - "fieldname": "advanced", - "fieldtype": "Section Break", - "label": "Advanced" - }, - { - "description": "Text to be displayed for Link to Web Page if this form has a web page. Link route will be automatically generated based on `page_name` and `parent_website_route`", - "fieldname": "web_page_link_text", - "fieldtype": "Data", - "label": "Web Page Link Text" - }, { "description": "List as [{\"label\": _(\"Jobs\"), \"route\":\"jobs\"}]", "fieldname": "breadcrumbs", @@ -345,13 +301,6 @@ "label": "Custom CSS", "options": "CSS" }, - { - "collapsible": 1, - "collapsible_depends_on": "custom_css", - "fieldname": "custom_css_section", - "fieldtype": "Section Break", - "label": "Custom CSS" - }, { "default": "0", "fieldname": "apply_document_permissions", @@ -363,13 +312,93 @@ "fieldname": "is_multi_step_form", "fieldtype": "Check", "label": "Is Multi Step Form" + }, + { + "default": "0", + "depends_on": "login_required", + "fieldname": "show_list", + "fieldtype": "Check", + "label": "Show List" + }, + { + "depends_on": "eval: doc.login_required && doc.show_list", + "fieldname": "list_title", + "fieldtype": "Data", + "label": "Title" + }, + { + "depends_on": "eval: doc.login_required && doc.show_list", + "fieldname": "list_columns", + "fieldtype": "Table", + "label": "List Columns", + "options": "Web Form List Column" + }, + { + "fieldname": "title_and_route_tab", + "fieldtype": "Tab Break", + "label": "Title & Route" + }, + { + "collapsible": 1, + "fieldname": "form_fields", + "fieldtype": "Section Break", + "label": "Form Fields" + }, + { + "fieldname": "column_break_18", + "fieldtype": "Column Break" + }, + { + "fieldname": "website_sidebar", + "fieldtype": "Link", + "label": "Website Sidebar", + "options": "Website Sidebar" + }, + { + "fieldname": "column_break_29", + "fieldtype": "Column Break" + }, + { + "fieldname": "list_setting_message", + "fieldtype": "HTML", + "label": "List Setting Message" + }, + { + "fieldname": "form_settings_tab", + "fieldtype": "Tab Break", + "label": "Form Settings" + }, + { + "collapsible": 1, + "collapsible_depends_on": "show_list", + "fieldname": "list_settings_tab", + "fieldtype": "Tab Break", + "label": "List Settings" + }, + { + "collapsible": 1, + "fieldname": "sidebar_settings_tab", + "fieldtype": "Tab Break", + "label": "Sidebar Settings" + }, + { + "fieldname": "scripting_style_tab", + "fieldtype": "Tab Break", + "label": "Scripting / Style" + }, + { + "collapsible": 1, + "collapsible_depends_on": "accept_payment", + "fieldname": "payments_tab", + "fieldtype": "Tab Break", + "label": "Payments" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2022-03-23 15:44:41.385001", + "modified": "2022-07-18 15:51:15.288860", "modified_by": "Administrator", "module": "Website", "name": "Web Form", diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index ee8861a7aa..e1c9e798e5 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -13,8 +13,8 @@ from frappe.desk.form.meta import get_code_files_via_hooks from frappe.integrations.utils import get_payment_gateway_controller from frappe.modules.utils import export_module_json, get_doc_module from frappe.rate_limiter import rate_limit -from frappe.utils import cstr -from frappe.website.utils import get_comment_list +from frappe.utils import cstr, dict_with_keys, strip_html +from frappe.website.utils import get_boot_data, get_comment_list, get_sidebar_items from frappe.website.website_generator import WebsiteGenerator @@ -32,17 +32,20 @@ class WebForm(WebsiteGenerator): if not self.module: self.module = frappe.db.get_value("DocType", self.doc_type, "module") - if ( - not ( - frappe.flags.in_install - or frappe.flags.in_patch - or frappe.flags.in_test - or frappe.flags.in_fixtures - ) - and self.is_standard - and not frappe.conf.developer_mode - ): - frappe.throw(_("You need to be in developer mode to edit a Standard Web Form")) + in_user_env = not ( + frappe.flags.in_install + or frappe.flags.in_patch + or frappe.flags.in_test + or frappe.flags.in_fixtures + ) + if in_user_env and self.is_standard and not frappe.conf.developer_mode: + # only published can be changed for standard web forms + if self.has_value_changed("published"): + published_value = self.published + self.reload() + self.published = published_value + else: + frappe.throw(_("You need to be in developer mode to edit a Standard Web Form")) if not frappe.flags.in_import: self.validate_fields() @@ -131,60 +134,131 @@ def get_context(context): def get_context(self, context): """Build context to render the `web_form.html` template""" + context.is_form_editable = False self.set_web_form_module() - doc, delimeter = make_route_string(frappe.form_dict) - context.doc = doc - context.delimeter = delimeter + if frappe.form_dict.is_list: + context.template = "website/doctype/web_form/templates/web_list.html" + else: + context.template = "website/doctype/web_form/templates/web_form.html" # check permissions - if frappe.session.user == "Guest" and frappe.form_dict.name: - frappe.throw( - _("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError - ) + if frappe.form_dict.name: + if frappe.session.user == "Guest": + frappe.throw( + _("You need to be logged in to access this {0}.").format(self.doc_type), + frappe.PermissionError, + ) - if frappe.form_dict.name and not self.has_web_form_permission( - self.doc_type, frappe.form_dict.name + if not frappe.db.exists(self.doc_type, frappe.form_dict.name): + raise frappe.PageDoesNotExistError() + + if not self.has_web_form_permission(self.doc_type, frappe.form_dict.name): + frappe.throw( + _("You don't have the permissions to access this document"), frappe.PermissionError + ) + + if frappe.local.path == self.route: + path = f"/{self.route}/list" if self.show_list else f"/{self.route}/new" + frappe.redirect(path) + + if frappe.form_dict.is_list and not self.show_list: + frappe.redirect(f"/{self.route}/new") + + if frappe.form_dict.is_edit and not self.allow_edit: + frappe.redirect(f"/{self.route}/{frappe.form_dict.name}") + + if frappe.form_dict.is_edit: + context.is_form_editable = True + + if ( + not frappe.form_dict.is_edit + and not frappe.form_dict.is_read + and self.allow_edit + and frappe.form_dict.name ): - frappe.throw( - _("You don't have the permissions to access this document"), frappe.PermissionError - ) + context.is_form_editable = True + frappe.redirect(f"/{frappe.local.path}/edit") + + if ( + frappe.session.user != "Guest" + and not self.allow_multiple + and not frappe.form_dict.name + and not frappe.form_dict.is_list + ): + name = frappe.db.get_value(self.doc_type, {"owner": frappe.session.user}, "name") + if name: + frappe.redirect(f"/{self.route}/{name}") + + # Show new form when + # - User is Guest + # - Login not required + route_to_new = frappe.session.user == "Guest" and not self.login_required + if not frappe.form_dict.is_new and route_to_new: + frappe.redirect(f"/{self.route}/new") self.reset_field_parent() if self.is_standard: self.use_meta_fields() - if not frappe.session.user == "Guest": - if self.allow_edit: - if self.allow_multiple: - if not frappe.form_dict.name and not frappe.form_dict.new: - # list data is queried via JS - context.is_list = True - else: - if frappe.session.user != "Guest" and not frappe.form_dict.name: - frappe.form_dict.name = frappe.db.get_value( - self.doc_type, {"owner": frappe.session.user}, "name" - ) + # add keys from form_dict to context + context.update(dict_with_keys(frappe.form_dict, ["is_list", "is_new", "is_edit", "is_read"])) - if not frappe.form_dict.name: - # only a single doc allowed and no existing doc, hence new - frappe.form_dict.new = 1 + for df in self.web_form_fields: + if df.fieldtype == "Column Break": + context.has_column_break = True + break + + # load web form doc + context.web_form_doc = self.as_dict(no_nulls=True) + context.web_form_doc.update(dict_with_keys(context, ["is_list", "is_new", "is_form_editable"])) + + if self.show_sidebar and self.website_sidebar: + context.sidebar_items = get_sidebar_items(self.website_sidebar) if frappe.form_dict.is_list: - context.is_list = True + self.load_list_data(context) + else: + self.load_form_data(context) - # always render new form if login is not required or doesn't allow editing existing ones - if not self.login_required or not self.allow_edit: - frappe.form_dict.new = 1 + self.add_custom_context_and_script(context) + self.load_translations(context) + + context.boot = get_boot_data() + context.boot["link_title_doctypes"] = frappe.boot.get_link_title_doctypes() + + def load_translations(self, context): + translated_messages = frappe.translate.get_dict("doctype", self.doc_type) + # Sr is not added by default, had to be added manually + translated_messages["Sr"] = _("Sr") + context.translated_messages = frappe.as_json(translated_messages) + + def load_list_data(self, context): + if not self.list_columns: + self.list_columns = get_in_list_view_fields(self.doc_type) + context.web_form_doc.list_columns = self.list_columns + + def load_form_data(self, context): + """Load document `doc` and `layout` properties for template""" + context.parents = [] + if self.show_list: + context.parents.append( + { + "label": _(self.title), + "route": f"{self.route}/list", + } + ) - self.load_document(context) context.parents = self.get_parents(context) if self.breadcrumbs: context.parents = frappe.safe_eval(self.breadcrumbs, {"_": _}) - context.has_header = (frappe.form_dict.name or frappe.form_dict.new) and ( + if frappe.form_dict.is_new: + context.title = _("New {0}").format(context.title) + + context.has_header = (frappe.form_dict.name or frappe.form_dict.is_new) and ( frappe.session.user != "Guest" or not self.login_required ) @@ -193,33 +267,40 @@ def get_context(context): "'" ) - self.add_custom_context_and_script(context) if not context.max_attachment_size: context.max_attachment_size = get_max_file_size() / 1024 / 1024 - context.show_in_grid = self.show_in_grid - self.load_translations(context) - context.link_title_doctypes = frappe.boot.get_link_title_doctypes() + # For Table fields, server-side processing for meta + for field in context.web_form_doc.web_form_fields: + if field.fieldtype == "Table": + field.fields = get_in_list_view_fields(field.options) - def load_translations(self, context): - translated_messages = frappe.translate.get_dict("doctype", self.doc_type) - # Sr is not added by default, had to be added manually - translated_messages["Sr"] = _("Sr") - context.translated_messages = frappe.as_json(translated_messages) + if field.fieldtype == "Link": + field.fieldtype = "Autocomplete" + field.options = get_link_options( + self.name, field.options, field.allow_read_on_all_link_options + ) - def load_document(self, context): - """Load document `doc` and `layout` properties for template""" - if frappe.form_dict.name or frappe.form_dict.new: - context.layout = self.get_layout() - context.parents = [{"route": self.route, "label": _(self.title)}] + context.reference_doc = {} + # load reference doc if frappe.form_dict.name: - context.doc = frappe.get_doc(self.doc_type, frappe.form_dict.name) - context.title = context.doc.get(context.doc.meta.get_title_field()) - context.doc.add_seen() - - context.reference_doctype = context.doc.doctype - context.reference_name = context.doc.name + context.doc_name = frappe.form_dict.name + context.reference_doc = frappe.get_doc(self.doc_type, context.doc_name) + context.title = strip_html( + context.reference_doc.get(context.reference_doc.meta.get_title_field()) + ) + if context.is_form_editable: + context.parents.append( + { + "label": _(context.title), + "route": f"{self.route}/{context.doc_name}", + } + ) + context.title = _("Edit") + context.reference_doc.add_seen() + context.reference_doctype = context.reference_doc.doctype + context.reference_name = context.reference_doc.name if self.show_attachments: context.attachments = frappe.get_all( @@ -233,7 +314,11 @@ def get_context(context): ) if self.allow_comments: - context.comment_list = get_comment_list(context.doc.doctype, context.doc.name) + context.comment_list = get_comment_list( + context.reference_doc.doctype, context.reference_doc.name + ) + + context.reference_doc = json.loads(context.reference_doc.as_json()) def get_payment_gateway_url(self, doc): if self.accept_payment: @@ -594,7 +679,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): # For Table fields, server-side processing for meta for field in out.web_form.web_form_fields: if field.fieldtype == "Table": - field.fields = frappe.get_meta(field.options).fields + field.fields = get_in_list_view_fields(field.options) out.update({field.fieldname: field.fields}) if field.fieldtype == "Link": diff --git a/frappe/website/doctype/web_form/web_form_list.js b/frappe/website/doctype/web_form/web_form_list.js new file mode 100644 index 0000000000..f426fd9899 --- /dev/null +++ b/frappe/website/doctype/web_form/web_form_list.js @@ -0,0 +1,10 @@ +frappe.listview_settings['Web Form'] = { + add_fields: ["title", "published"], + get_indicator: function(doc) { + if (doc.published) { + return [__("Published"), "green", "published,=,1"]; + } else { + return [__("Not Published"), "gray", "published,=,0"]; + } + } +}; \ No newline at end of file diff --git a/frappe/website/doctype/web_form_field/web_form_field.json b/frappe/website/doctype/web_form_field/web_form_field.json index 36b1ca2c15..4e0d58d42d 100644 --- a/frappe/website/doctype/web_form_field/web_form_field.json +++ b/frappe/website/doctype/web_form_field/web_form_field.json @@ -10,7 +10,6 @@ "label", "allow_read_on_all_link_options", "reqd", - "depends_on", "read_only", "show_in_filter", "hidden", @@ -19,6 +18,7 @@ "max_length", "max_value", "property_depends_on_section", + "depends_on", "mandatory_depends_on", "column_break_16", "read_only_depends_on", @@ -63,7 +63,7 @@ { "fieldname": "depends_on", "fieldtype": "Code", - "label": "Depends On" + "label": "Display Depends On" }, { "default": "0", @@ -146,12 +146,13 @@ ], "istable": 1, "links": [], - "modified": "2022-01-28 10:41:25.422345", + "modified": "2022-06-06 16:00:55.627950", "modified_by": "Administrator", "module": "Website", "name": "Web Form Field", "owner": "Administrator", "permissions": [], "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/website/doctype/web_form_list_column/__init__.py b/frappe/website/doctype/web_form_list_column/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/doctype/web_form_list_column/web_form_list_column.json b/frappe/website/doctype/web_form_list_column/web_form_list_column.json new file mode 100644 index 0000000000..e55aeadca6 --- /dev/null +++ b/frappe/website/doctype/web_form_list_column/web_form_list_column.json @@ -0,0 +1,48 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2022-06-20 20:02:12.132569", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "fieldname", + "fieldtype", + "label" + ], + "fields": [ + { + "fieldname": "fieldname", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Fieldname", + "reqd": 1 + }, + { + "fieldname": "label", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Label" + }, + { + "fieldname": "fieldtype", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Fieldtype", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2022-06-21 17:22:14.978947", + "modified_by": "Administrator", + "module": "Website", + "name": "Web Form List Column", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/website/doctype/web_form_list_column/web_form_list_column.py b/frappe/website/doctype/web_form_list_column/web_form_list_column.py new file mode 100644 index 0000000000..9aff5f1ecc --- /dev/null +++ b/frappe/website/doctype/web_form_list_column/web_form_list_column.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class WebFormListColumn(Document): + pass diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index a94838baed..b523eb2e83 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -16,6 +16,7 @@ from frappe.website.utils import ( find_first_image, get_comment_list, get_html_content_based_on_type, + get_sidebar_items, ) from frappe.website.website_generator import WebsiteGenerator @@ -70,6 +71,9 @@ class WebPage(WebsiteGenerator): if not self.show_title: context["no_header"] = 1 + if self.show_sidebar: + context.sidebar_items = get_sidebar_items(self.website_sidebar) + self.set_metatags(context) self.set_breadcrumbs(context) self.set_title_and_header(context) diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index bd634b4f32..fffbd94684 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -7,6 +7,7 @@ from frappe import _ from frappe.integrations.google_oauth import GoogleOAuth from frappe.model.document import Document from frappe.utils import encode, get_request_site_address +from frappe.website.utils import get_boot_data class WebsiteSettings(Document): @@ -190,6 +191,8 @@ def get_website_settings(context=None): if settings.splash_image: context["splash_image"] = settings.splash_image + context.boot = get_boot_data() + return context diff --git a/frappe/website/page_renderers/web_form.py b/frappe/website/page_renderers/web_form.py index 1953118790..74996e4a78 100644 --- a/frappe/website/page_renderers/web_form.py +++ b/frappe/website/page_renderers/web_form.py @@ -1,11 +1,13 @@ -import frappe from frappe.website.page_renderers.document_page import DocumentPage +from frappe.website.router import get_page_info_from_web_form class WebFormPage(DocumentPage): def can_render(self): - webform_name = frappe.db.exists("Web Form", {"route": self.path, "published": 1}, cache=True) - if webform_name: + web_form = get_page_info_from_web_form(self.path) + if web_form: self.doctype = "Web Form" - self.docname = webform_name - return bool(webform_name) + self.docname = web_form.name + return True + else: + return False diff --git a/frappe/website/router.py b/frappe/website/router.py index 24a085224b..aa1e15d4c9 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -30,6 +30,32 @@ def get_page_info_from_web_page_with_dynamic_routes(path): return page_info[end_point] +def get_page_info_from_web_form(path): + """Query published web forms and evaluate if the route matches""" + rules, page_info = [], {} + web_forms = frappe.db.get_all("Web Form", ["name", "route", "modified"], {"published": 1}) + for d in web_forms: + rules.append(Rule(f"/{d.route}", endpoint=d.name)) + rules.append(Rule(f"/{d.route}/list", endpoint=d.name)) + rules.append(Rule(f"/{d.route}/new", endpoint=d.name)) + rules.append(Rule(f"/{d.route}/", endpoint=d.name)) + rules.append(Rule(f"/{d.route}//edit", endpoint=d.name)) + d.doctype = "Web Form" + page_info[d.name] = d + + end_point = evaluate_dynamic_routes(rules, path) + if end_point: + if path.endswith("/list"): + frappe.form_dict.is_list = True + elif path.endswith("/new"): + frappe.form_dict.is_new = True + elif path.endswith("/edit"): + frappe.form_dict.is_edit = True + else: + frappe.form_dict.is_read = True + return page_info[end_point] + + def evaluate_dynamic_routes(rules, path): """ Use Werkzeug routing to evaluate dynamic routes like /project/ diff --git a/frappe/website/serve.py b/frappe/website/serve.py index 2c33b5df51..7eb8b017f1 100644 --- a/frappe/website/serve.py +++ b/frappe/website/serve.py @@ -1,5 +1,6 @@ import frappe from frappe.website.page_renderers.error_page import ErrorPage +from frappe.website.page_renderers.not_found_page import NotFoundPage from frappe.website.page_renderers.not_permitted_page import NotPermittedPage from frappe.website.page_renderers.redirect_page import RedirectPage from frappe.website.path_resolver import PathResolver @@ -19,6 +20,8 @@ def get_response(path=None, http_status_code=200): return RedirectPage(endpoint or path, http_status_code).render() except frappe.PermissionError as e: response = NotPermittedPage(endpoint, http_status_code, exception=e).render() + except frappe.PageDoesNotExistError: + response = NotFoundPage(endpoint, http_status_code).render() except Exception as e: frappe.log_error(f"{path} failed") response = ErrorPage(exception=e).render() diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 8a0cfbab7c..508026f064 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -12,7 +12,7 @@ from werkzeug.wrappers import Response import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import md_to_html +from frappe.utils import cint, get_time_zone, md_to_html FRONTMATTER_PATTERN = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M) H1_TAG_PATTERN = re.compile("

([^<]*)") @@ -158,6 +158,20 @@ def get_home_page_via_hooks(): return home_page +def get_boot_data(): + return { + "sysdefaults": { + "float_precision": cint(frappe.get_system_settings("float_precision")) or 3, + "date_format": frappe.get_system_settings("date_format") or "yyyy-mm-dd", + "time_format": frappe.get_system_settings("time_format") or "HH:mm:ss", + }, + "time_zone": { + "system": get_time_zone(), + "user": frappe.db.get_value("User", frappe.session.user, "time_zone") or get_time_zone(), + }, + } + + def is_signup_disabled(): return frappe.db.get_single_value("Website Settings", "disable_signup", True) @@ -393,7 +407,7 @@ def get_frontmatter(string): } -def get_sidebar_items(parent_sidebar, basepath): +def get_sidebar_items(parent_sidebar, basepath=None): import frappe.www.list sidebar_items = [] diff --git a/frappe/website/web_form/request_data/request_data.json b/frappe/website/web_form/request_data/request_data.json index 591ef4a031..c52a2f6203 100644 --- a/frappe/website/web_form/request_data/request_data.json +++ b/frappe/website/web_form/request_data/request_data.json @@ -11,6 +11,7 @@ "apply_document_permissions": 0, "breadcrumbs": "", "button_label": "Request Data", + "client_script": "", "creation": "2019-01-24 16:19:26.886096", "currency": "INR", "doc_type": "Personal Data Download Request", @@ -18,10 +19,12 @@ "doctype": "Web Form", "idx": 0, "introduction_text": "

Request a file containing your personally identifiable information (PII) that is saved on our system. The file will be in JSON format and is sent to you by email. If you would like to have your PII deleted from our system, please make a request to delete data.

", + "is_multi_step_form": 0, "is_standard": 1, + "list_columns": [], "login_required": 0, "max_attachment_size": 0, - "modified": "2021-03-25 10:52:13.149538", + "modified": "2022-07-18 16:51:07.281527", "modified_by": "Administrator", "module": "Website", "name": "request-data", @@ -31,9 +34,8 @@ "route": "request-data", "route_to_success_link": 1, "show_attachments": 0, - "show_in_grid": 0, + "show_list": 0, "show_sidebar": 0, - "sidebar_items": [], "success_message": "A download link with your data will be sent to the email address associated with your account.", "success_url": "/desk", "title": "Request Data", diff --git a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json index 1113297df6..ce11666a34 100644 --- a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json +++ b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json @@ -9,6 +9,7 @@ "amount": 0.0, "amount_based_on_field": 0, "apply_document_permissions": 0, + "breadcrumbs": "", "button_label": "Submit", "client_script": "", "creation": "2019-01-25 14:24:12.588810", @@ -19,10 +20,12 @@ "doctype": "Web Form", "idx": 0, "introduction_text": "

Send a request to delete your account and personally identifiable information (PII) that is stored on our system. You will receive an email to verify your request. Once the request is verified we will take care of deleting your PII. If you just want to check what PII we have stored, you can request your data.

", + "is_multi_step_form": 0, "is_standard": 1, + "list_columns": [], "login_required": 0, "max_attachment_size": 0, - "modified": "2021-11-30 17:56:03.099870", + "modified": "2022-07-18 16:51:30.949738", "modified_by": "Administrator", "module": "Website", "name": "request-to-delete-data", @@ -32,9 +35,8 @@ "route": "request-for-account-deletion", "route_to_success_link": 0, "show_attachments": 0, - "show_in_grid": 0, + "show_list": 0, "show_sidebar": 0, - "sidebar_items": [], "success_message": "An email to verify your request has been sent to your email address. Please verify your request to complete the process.", "success_url": "/", "title": "Request for Account Deletion", From eea2616aac12cdc3cefa15987d9958088cc7011e Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 19 Jul 2022 22:28:14 +0530 Subject: [PATCH 175/201] style: use `middleware` decorator to keep function definition intact --- frappe/app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index b9db59cdb1..298d94b06c 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -25,7 +25,7 @@ from frappe.utils import get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot from frappe.website.serve import get_response -local_manager = LocalManager([frappe.local]) +local_manager = LocalManager(frappe.local) _site = None _sites_path = os.environ.get("SITES_PATH", ".") @@ -44,6 +44,7 @@ class RequestContext: frappe.destroy() +@local_manager.middleware @Request.application def application(request): response = None @@ -313,9 +314,6 @@ def after_request(rollback): return rollback -application = local_manager.make_middleware(application) - - def serve( port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path="." ): From 8f48c4d9432aeee6a403c8cad336f1e8f21cf227 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 20 Jul 2022 08:24:49 +0530 Subject: [PATCH 176/201] fix: No need to check for permssion again while attaching a file --- frappe/core/doctype/file/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index e59ec2aede..d99e5cff48 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -327,7 +327,7 @@ def attach_files_to_document(doc: "File", event) -> None: folder="Home/Attachments", ) try: - file.insert() + file.insert(ignore_permissions=True) except Exception: doc.log_error("Error Attaching File") From d5bf9b60e098bb42fa4aa00d1c26d11c913702df Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 20 Jul 2022 10:36:40 +0530 Subject: [PATCH 177/201] fix: Query for unseen notification --- frappe/boot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/boot.py b/frappe/boot.py index ad729746fe..c8c98e506f 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -14,7 +14,7 @@ from frappe.email.inbox import get_email_accounts from frappe.model.base_document import get_controller from frappe.query_builder import DocType from frappe.query_builder.functions import Count -from frappe.query_builder.terms import SubQuery +from frappe.query_builder.terms import ParameterizedValueWrapper, SubQuery from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, @@ -331,8 +331,8 @@ def get_unseen_notes(): (note.notify_on_every_login == 1) & (note.expire_notification_on > frappe.utils.now()) & ( - SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( - [frappe.session.user] + ParameterizedValueWrapper(frappe.session.user).notin( + SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)) ) ) ) From 99c69907611eddf7ee35ea12469e7462f68d15db Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 20 Jul 2022 11:30:57 +0530 Subject: [PATCH 178/201] fix: Allow `fields` to have "*" without array --- frappe/desk/reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index caba0212b9..ac0d04c737 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -225,7 +225,7 @@ def parse_json(data): if isinstance(data.get("or_filters"), str): data["or_filters"] = json.loads(data["or_filters"]) if isinstance(data.get("fields"), str): - data["fields"] = json.loads(data["fields"]) + data["fields"] = ["*"] if data["fields"] == "*" else json.loads(data["fields"]) if isinstance(data.get("docstatus"), str): data["docstatus"] = json.loads(data["docstatus"]) if isinstance(data.get("save_user_settings"), str): From 6bbd8fcb32e8ede5b65d69f28b0944ab2daff366 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 20 Jul 2022 13:16:51 +0530 Subject: [PATCH 179/201] fix: Filter notes with `notify_on_login` and not `notify_on_every_login` --- frappe/boot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/boot.py b/frappe/boot.py index c8c98e506f..e43446f352 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -328,7 +328,7 @@ def get_unseen_notes(): frappe.qb.from_(note) .select(note.name, note.title, note.content, note.notify_on_every_login) .where( - (note.notify_on_every_login == 1) + (note.notify_on_login == 1) & (note.expire_notification_on > frappe.utils.now()) & ( ParameterizedValueWrapper(frappe.session.user).notin( From 67350e9ac3f2dc0fceb1899c8692adcd9cdd4213 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 20 Jul 2022 13:23:49 +0530 Subject: [PATCH 180/201] test: Add a test case to validate `get_unseen_notes` --- frappe/tests/test_boot.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 frappe/tests/test_boot.py diff --git a/frappe/tests/test_boot.py b/frappe/tests/test_boot.py new file mode 100644 index 0000000000..8f26a078b0 --- /dev/null +++ b/frappe/tests/test_boot.py @@ -0,0 +1,29 @@ +import unittest + +import frappe +from frappe.boot import get_unseen_notes +from frappe.desk.doctype.note.note import mark_as_seen + + +class TestBootData(unittest.TestCase): + def test_get_unseen_notes(self): + frappe.db.delete("Note") + frappe.db.delete("Note Seen By") + note = frappe.get_doc( + { + "doctype": "Note", + "title": "Test Note", + "notify_on_login": 1, + "content": "Test Note 1", + "public": 1, + } + ) + note.insert() + + frappe.set_user("test@example.com") + unseen_notes = [d.title for d in get_unseen_notes()] + self.assertListEqual(unseen_notes, ["Test Note"]) + + mark_as_seen(note.name) + unseen_notes = [d.title for d in get_unseen_notes()] + self.assertListEqual(unseen_notes, []) From 8a8f0a1c798363ff57d16bfcb67ff55b2f0fd311 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Jul 2022 17:51:45 +0530 Subject: [PATCH 181/201] fix: replace incorrect validation on doctype links (#17561) --- frappe/core/doctype/doctype/doctype.py | 27 ++++++++++++++++----- frappe/core/doctype/doctype/test_doctype.py | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 2b28373384..9a8a976b9f 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -17,7 +17,7 @@ from frappe.cache_manager import clear_controller_cache, clear_user_cache from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.database.schema import validate_column_length, validate_column_name -from frappe.desk.notifications import delete_notification_count_for +from frappe.desk.notifications import delete_notification_count_for, get_filters_for from frappe.desk.utils import validate_route_conflict from frappe.model import ( child_table_fields, @@ -982,11 +982,7 @@ def validate_links_table_fieldnames(meta): fieldnames = tuple(field.fieldname for field in meta.fields) for index, link in enumerate(meta.links, 1): - if not frappe.get_meta(link.link_doctype).has_field(link.link_fieldname): - message = _("Document Links Row #{0}: Could not find field {1} in {2} DocType").format( - index, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype) - ) - frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname")) + _test_connection_query(doctype=link.link_doctype, field=link.link_fieldname, idx=index) if not link.is_child_table: continue @@ -1015,6 +1011,25 @@ def validate_links_table_fieldnames(meta): frappe.throw(message, frappe.ValidationError, _("Invalid Table Fieldname")) +def _test_connection_query(doctype, field, idx): + """Make sure that connection can be queried. + + This function executes query similar to one that would be executed for + finding count on dashboard and hence validates if fieldname/doctype are + correct. + """ + filters = get_filters_for(doctype) or {} + filters[field] = "" + + try: + frappe.get_all(doctype, filters=filters, limit=1, distinct=True, ignore_ifnull=True) + except Exception as e: + frappe.clear_last_message() + msg = _("Document Links Row #{0}: Invalid doctype or fieldname.").format(idx) + msg += "
" + str(e) + frappe.throw(msg, InvalidFieldNameError) + + def validate_fields_for_doctype(doctype): meta = frappe.get_meta(doctype, cached=False) validate_links_table_fieldnames(meta) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 3a5ca4329f..a083939c94 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -543,7 +543,7 @@ class TestDocType(unittest.TestCase): # check invalid doctype doc.append("links", {"link_doctype": "User2", "link_fieldname": "first_name"}) - self.assertRaises(frappe.DoesNotExistError, validate_links_table_fieldnames, doc) + self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc) doc.links = [] # reset links table # check invalid fieldname From e7082d611f6b60e55281aab767d60146611d4a6b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 20 Jul 2022 21:05:23 +0530 Subject: [PATCH 182/201] fix: broken realtime doc change updates (#17567) --- frappe/model/document.py | 4 ++-- frappe/tests/test_document.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 864f2d50b4..3bddaa9aae 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1093,7 +1093,7 @@ class Document(BaseDocument): self.clear_cache() - if not hasattr(self.flags, "notify_update") or self.flags.notify_update: + if self.flags.get("notify_update", True): self.notify_update() update_global_search(self) @@ -1147,7 +1147,7 @@ class Document(BaseDocument): :param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary :param value: value of the property to be updated :param update_modified: default True. updates the `modified` and `modified_by` properties - :param notify: default False. run doc.notify_updated() to send updates via socketio + :param notify: default False. run doc.notify_update() to send updates via socketio :param commit: default False. run frappe.db.commit() """ if isinstance(fieldname, dict): diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 454ca76983..2cd37366fd 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -3,7 +3,7 @@ import unittest from contextlib import contextmanager from datetime import timedelta -from unittest.mock import patch +from unittest.mock import Mock, patch import frappe from frappe.app import make_form_dict @@ -373,6 +373,19 @@ class TestDocument(unittest.TestCase): except Exception as e: self.fail(f"Invalid doc hook: {doctype}:{hook}\n{e}") + def test_realtime_notify(self): + todo = frappe.new_doc("ToDo") + todo.description = "this will trigger realtime update" + todo.notify_update = Mock() + todo.insert() + self.assertEqual(todo.notify_update.call_count, 1) + + todo.reload() + todo.flags.notify_update = False + todo.description = "this won't trigger realtime update" + todo.save() + self.assertEqual(todo.notify_update.call_count, 1) + class TestDocumentWebView(unittest.TestCase): def get(self, path, user="Guest"): From 836ce67d8580edff5d98d123dc2495c2923d951a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 22 Jun 2022 13:27:00 +0530 Subject: [PATCH 183/201] feat: undo/redo on form view This commit implements basic undo/redo on form view fields --- .../js/frappe/form/controls/base_control.js | 2 ++ frappe/public/js/frappe/form/form.js | 22 ++++++++++++ frappe/public/js/frappe/form/undo_manager.js | 36 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 frappe/public/js/frappe/form/undo_manager.js diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 73831d493b..478ecc0174 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -187,6 +187,8 @@ frappe.ui.form.Control = class BaseControl { return Promise.resolve(); } + const old_value = this.get_model_value(); + this.frm?.undo_manager.record_change({fieldname: me.df.fieldname, old_value, new_value: value}); this.inside_change_event = true; function set(value) { me.inside_change_event = false; diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 148ec7ca86..c2363757c7 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -13,6 +13,7 @@ import './script_helpers'; import './sidebar/form_sidebar'; import './footer/footer'; import './form_tour'; +import {UndoManager } from './undo_manager'; frappe.ui.form.Controller = class FormController { constructor(opts) { @@ -38,6 +39,7 @@ frappe.ui.form.Form = class FrappeForm { this.fetch_dict = {}; this.parent = parent; this.doctype_layout = frappe.get_doc('DocType Layout', doctype_layout_name); + this.undo_manager = new UndoManager({frm: this}); this.setup_meta(doctype); this.beforeUnloadListener = (event) => { @@ -143,6 +145,26 @@ frappe.ui.form.Form = class FrappeForm { condition: () => !this.is_new() }); + // Undo and redo + frappe.ui.keys.add_shortcut({ + shortcut: 'ctrl+z', + action: () => this.undo_manager.undo(), + page: this.page, + description: __('Undo last action'), + }); + frappe.ui.keys.add_shortcut({ + shortcut: 'shift+ctrl+z', + action: () => this.undo_manager.redo(), + page: this.page, + description: __('Redo last action'), + }); + frappe.ui.keys.add_shortcut({ + shortcut: 'ctrl+y', + action: () => this.undo_manager.redo(), + page: this.page, + description: __('Redo last action'), + }); + let grid_shortcut_keys = [ { 'shortcut': 'Up Arrow', diff --git a/frappe/public/js/frappe/form/undo_manager.js b/frappe/public/js/frappe/form/undo_manager.js new file mode 100644 index 0000000000..6046ce15eb --- /dev/null +++ b/frappe/public/js/frappe/form/undo_manager.js @@ -0,0 +1,36 @@ +export class UndoManager { + constructor({ frm }) { + this.frm = frm; + this.undo_stack = []; + this.redo_stack = []; + } + record_change({ fieldname, old_value, new_value }) { + this.undo_stack.push({ fieldname, old_value, new_value }); + } + + undo() { + const change = this.undo_stack.pop(); + if (change) { + this.frm.set_value(change.fieldname, change.old_value); + this.redo_stack.push(change); + } else { + this.show_alert(__("Nothing left to undo")); + } + } + + redo() { + const change = this.redo_stack.pop(); + if (change) { + this.frm.set_value(change.fieldname, change.new_value); + this.undo_stack.push(change); + } else { + this.show_alert(__("Nothing left to redo")); + } + } + + show_alert(msg) { + // reduce duration + // keyboard interactions shouldn't have long running + frappe.show_alert(msg, 3); + } +} From 715299fc083991b3e086b3b8ecdf51a47741be5f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 21:07:44 +0530 Subject: [PATCH 184/201] fix: erase undo/redo history on doc change/refresh --- frappe/public/js/frappe/form/form.js | 2 ++ frappe/public/js/frappe/form/undo_manager.js | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index c2363757c7..e52b213ba6 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -379,6 +379,8 @@ frappe.ui.form.Form = class FrappeForm { cur_frm = this; + this.undo_manager.erase_history(); + if(this.docname) { // document to show this.save_disabled = false; // set the doc diff --git a/frappe/public/js/frappe/form/undo_manager.js b/frappe/public/js/frappe/form/undo_manager.js index 6046ce15eb..2f1ab54a07 100644 --- a/frappe/public/js/frappe/form/undo_manager.js +++ b/frappe/public/js/frappe/form/undo_manager.js @@ -33,4 +33,9 @@ export class UndoManager { // keyboard interactions shouldn't have long running frappe.show_alert(msg, 3); } + + erase_history() { + this.undo_stack = []; + this.redo_stack = []; + } } From a1ca1e2cc6228853e8a0ba27e6f323303b3ec59b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 21:50:34 +0530 Subject: [PATCH 185/201] feat: undo/redo on child table fields --- .../js/frappe/form/controls/base_control.js | 9 +++- frappe/public/js/frappe/form/undo_manager.js | 50 ++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 478ecc0174..f8ceec4c12 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -188,7 +188,14 @@ frappe.ui.form.Control = class BaseControl { } const old_value = this.get_model_value(); - this.frm?.undo_manager.record_change({fieldname: me.df.fieldname, old_value, new_value: value}); + this.frm?.undo_manager.record_change({ + fieldname: me.df.fieldname, + old_value, + new_value: value, + doctype: this.doctype, + docname: this.docname, + is_child: Boolean(this.doc?.parenttype) + }); this.inside_change_event = true; function set(value) { me.inside_change_event = false; diff --git a/frappe/public/js/frappe/form/undo_manager.js b/frappe/public/js/frappe/form/undo_manager.js index 2f1ab54a07..5dd94b82d2 100644 --- a/frappe/public/js/frappe/form/undo_manager.js +++ b/frappe/public/js/frappe/form/undo_manager.js @@ -4,33 +4,69 @@ export class UndoManager { this.undo_stack = []; this.redo_stack = []; } - record_change({ fieldname, old_value, new_value }) { - this.undo_stack.push({ fieldname, old_value, new_value }); + record_change({ + fieldname, + old_value, + new_value, + doctype, + docname, + is_child, + }) { + this.undo_stack.push({ + fieldname, + old_value, + new_value, + doctype, + docname, + is_child, + }); + console.log(this.undo_stack[this.undo_stack.length - 1]); } undo() { const change = this.undo_stack.pop(); if (change) { - this.frm.set_value(change.fieldname, change.old_value); - this.redo_stack.push(change); + this.apply_change(change); + this.push_reverse_entry(change, this.redo_stack); } else { this.show_alert(__("Nothing left to undo")); } } + push_reverse_entry(change, stack) { + stack.push({ + ...change, + new_value: change.old_value, + old_value: change.new_value, + }); + } + redo() { const change = this.redo_stack.pop(); if (change) { - this.frm.set_value(change.fieldname, change.new_value); - this.undo_stack.push(change); + this.apply_change(change); + this.push_reverse_entry(change, this.undo_stack); } else { this.show_alert(__("Nothing left to redo")); } } + apply_change(change) { + if (change.is_child) { + frappe.model.set_value( + change.doctype, + change.docname, + change.fieldname, + change.old_value + ); + } else { + this.frm.set_value(change.fieldname, change.old_value); + } + } + show_alert(msg) { // reduce duration - // keyboard interactions shouldn't have long running + // keyboard interactions shouldn't have long running annoying toasts frappe.show_alert(msg, 3); } From 129152c1c599f53e696de0011ec82e29302df2db Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 22:25:55 +0530 Subject: [PATCH 186/201] feat: highlight changed fields when doing undo/redo --- .../js/frappe/form/controls/base_control.js | 2 +- frappe/public/js/frappe/form/form.js | 8 ++-- frappe/public/js/frappe/form/undo_manager.js | 44 +++++++++---------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index f8ceec4c12..b066e5141a 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -188,7 +188,7 @@ frappe.ui.form.Control = class BaseControl { } const old_value = this.get_model_value(); - this.frm?.undo_manager.record_change({ + this.frm?.undo_manager?.record_change({ fieldname: me.df.fieldname, old_value, new_value: value, diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index e52b213ba6..4e38a5ee7e 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -13,7 +13,7 @@ import './script_helpers'; import './sidebar/form_sidebar'; import './footer/footer'; import './form_tour'; -import {UndoManager } from './undo_manager'; +import { UndoManager } from './undo_manager'; frappe.ui.form.Controller = class FormController { constructor(opts) { @@ -1785,7 +1785,7 @@ frappe.ui.form.Form = class FrappeForm { return sum; } - scroll_to_field(fieldname) { + scroll_to_field(fieldname, focus=true) { let field = this.get_field(fieldname); if (!field) return; @@ -1805,7 +1805,9 @@ frappe.ui.form.Form = class FrappeForm { frappe.utils.scroll_to($el, true, 15); // focus if text field - $el.find('input, select, textarea').focus(); + if (focus) { + $el.find('input, select, textarea').focus(); + } // highlight control inside field let control_element = $el.find('.form-control') diff --git a/frappe/public/js/frappe/form/undo_manager.js b/frappe/public/js/frappe/form/undo_manager.js index 5dd94b82d2..cfffe0611b 100644 --- a/frappe/public/js/frappe/form/undo_manager.js +++ b/frappe/public/js/frappe/form/undo_manager.js @@ -20,20 +20,34 @@ export class UndoManager { docname, is_child, }); - console.log(this.undo_stack[this.undo_stack.length - 1]); + } + + erase_history() { + this.undo_stack = []; + this.redo_stack = []; } undo() { const change = this.undo_stack.pop(); if (change) { - this.apply_change(change); - this.push_reverse_entry(change, this.redo_stack); + this.#apply_change(change); + this.#push_reverse_entry(change, this.redo_stack); } else { - this.show_alert(__("Nothing left to undo")); + this.#show_alert(__("Nothing left to undo")); } } - push_reverse_entry(change, stack) { + redo() { + const change = this.redo_stack.pop(); + if (change) { + this.#apply_change(change); + this.#push_reverse_entry(change, this.undo_stack); + } else { + this.#show_alert(__("Nothing left to redo")); + } + } + + #push_reverse_entry(change, stack) { stack.push({ ...change, new_value: change.old_value, @@ -41,17 +55,7 @@ export class UndoManager { }); } - redo() { - const change = this.redo_stack.pop(); - if (change) { - this.apply_change(change); - this.push_reverse_entry(change, this.undo_stack); - } else { - this.show_alert(__("Nothing left to redo")); - } - } - - apply_change(change) { + #apply_change(change) { if (change.is_child) { frappe.model.set_value( change.doctype, @@ -61,17 +65,13 @@ export class UndoManager { ); } else { this.frm.set_value(change.fieldname, change.old_value); + this.frm.scroll_to_field(change.fieldname, false); } } - show_alert(msg) { + #show_alert(msg) { // reduce duration // keyboard interactions shouldn't have long running annoying toasts frappe.show_alert(msg, 3); } - - erase_history() { - this.undo_stack = []; - this.redo_stack = []; - } } From e73f4aa8ab79ddfc2034bb0a7d45788b32a6878f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 12 Jul 2022 23:22:53 +0530 Subject: [PATCH 187/201] fix: avoid nonsensical change triggers if both values are same then logging it doesn't make much sense. --- frappe/public/js/frappe/form/undo_manager.js | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/form/undo_manager.js b/frappe/public/js/frappe/form/undo_manager.js index cfffe0611b..c09c3902b7 100644 --- a/frappe/public/js/frappe/form/undo_manager.js +++ b/frappe/public/js/frappe/form/undo_manager.js @@ -12,6 +12,10 @@ export class UndoManager { docname, is_child, }) { + if (old_value == new_value) { + return; + } + this.undo_stack.push({ fieldname, old_value, @@ -30,24 +34,24 @@ export class UndoManager { undo() { const change = this.undo_stack.pop(); if (change) { - this.#apply_change(change); - this.#push_reverse_entry(change, this.redo_stack); + this._apply_change(change); + this._push_reverse_entry(change, this.redo_stack); } else { - this.#show_alert(__("Nothing left to undo")); + this._show_alert(__("Nothing left to undo")); } } redo() { const change = this.redo_stack.pop(); if (change) { - this.#apply_change(change); - this.#push_reverse_entry(change, this.undo_stack); + this._apply_change(change); + this._push_reverse_entry(change, this.undo_stack); } else { - this.#show_alert(__("Nothing left to redo")); + this._show_alert(__("Nothing left to redo")); } } - #push_reverse_entry(change, stack) { + _push_reverse_entry(change, stack) { stack.push({ ...change, new_value: change.old_value, @@ -55,7 +59,7 @@ export class UndoManager { }); } - #apply_change(change) { + _apply_change(change) { if (change.is_child) { frappe.model.set_value( change.doctype, @@ -69,7 +73,7 @@ export class UndoManager { } } - #show_alert(msg) { + _show_alert(msg) { // reduce duration // keyboard interactions shouldn't have long running annoying toasts frappe.show_alert(msg, 3); From 5ea96ced3ac369bd39a29faf9c50e29c66022265 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 19 Jul 2022 20:29:56 +0530 Subject: [PATCH 188/201] test: undo/redo, jump to field UI tests --- cypress/integration/form.js | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cypress/integration/form.js b/cypress/integration/form.js index b395ff77b2..53b87994d7 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -6,6 +6,7 @@ context('Form', () => { return frappe.call("frappe.tests.ui_test_helpers.create_contact_records"); }); }); + it('create a new form', () => { cy.visit('/app/todo/new'); cy.get_field('description', 'Text Editor').type('this is a test todo', {force: true}).wait(200); @@ -95,4 +96,63 @@ context('Form', () => { }) }) }); + + it('let user undo/redo field value changes', { scrollBehavior: false }, () => { + const jump_to_field = (field_label) => { + cy.get("body") + .type("{esc}") // lose focus if any + .type("{ctrl+j}") // jump to field + .type(field_label) + .wait(500) + .type("{enter}") + .wait(200) + .type("{enter}") + .wait(500); + }; + + const type_value = (value) => { + cy.focused() + .clear() + .type(value) + .type("{esc}"); + }; + + const undo = () => cy.get("body").type("{esc}").type("{ctrl+z}").wait(500); + const redo = () => cy.get("body").type("{esc}").type("{ctrl+y}").wait(500); + + cy.new_form('User'); + + jump_to_field("Email"); + type_value("admin@example.com"); + + jump_to_field("Username"); + type_value("admin42"); + + jump_to_field("Birth Date"); + type_value("12-31-01"); + + jump_to_field("Send Welcome Email"); + cy.focused().uncheck() + + // make a mistake + jump_to_field("Username"); + type_value("admin24"); + + // undo behaviour + undo(); + cy.get_field("username").should('have.value', 'admin42'); + + // redo behaviour + redo(); + cy.get_field("username").should('have.value', 'admin24'); + + // undo everything & redo everything, ensure same values at the end + undo(); undo(); undo(); undo(); undo(); + redo(); redo(); redo(); redo(); redo(); + + cy.get_field("username").should('have.value', 'admin24'); + cy.get_field("email").should('have.value', 'admin@example.com'); + cy.get_field("birth_date").should('have.value', '12-31-2001'); // parsed value + cy.get_field("send_welcome_email").should('not.be.checked'); + }); }); From 85a3837b14c78816c9647f180d761e09d18bce2d Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 21 Jul 2022 07:34:47 +0000 Subject: [PATCH 189/201] fix: ensure 2FA patch sets parent only for 2FA keys (#17575) * fix: ensure 2FA patch sets parent only for 2FA keys * fix: try to rebuild defaults * fix: set other app defaults as well * fix: set POS profile defaults * fix: exists params * chore: remove unnecessary change * fix: handle case where POS Profile doesnt exist * chore: move erpnext code to erpnext --- .../system_settings/system_settings.py | 15 +++++---- frappe/patches.txt | 1 + frappe/patches/v13_0/encrypt_2fa_secrets.py | 1 + .../patches/v13_0/reset_corrupt_defaults.py | 33 +++++++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 frappe/patches/v13_0/reset_corrupt_defaults.py diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index fbdc188742..4bd41be974 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -44,12 +44,7 @@ class SystemSettings(Document): frappe.flags.update_last_reset_password_date = True def on_update(self): - for df in self.meta.get("fields"): - if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): - frappe.db.set_default(df.fieldname, self.get(df.fieldname)) - - if self.language: - set_default_language(self.language) + self.set_defaults() frappe.cache().delete_value("system_settings") frappe.cache().delete_value("time_zone") @@ -57,6 +52,14 @@ class SystemSettings(Document): if frappe.flags.update_last_reset_password_date: update_last_reset_password_date() + def set_defaults(self): + for df in self.meta.get("fields"): + if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): + frappe.db.set_default(df.fieldname, self.get(df.fieldname)) + + if self.language: + set_default_language(self.language) + def update_last_reset_password_date(): frappe.db.sql( diff --git a/frappe/patches.txt b/frappe/patches.txt index f79cadae87..ee2eb0d2a1 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -184,6 +184,7 @@ frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v13_0.set_first_day_of_the_week frappe.patches.v13_0.encrypt_2fa_secrets +frappe.patches.v13_0.reset_corrupt_defaults execute:frappe.reload_doc('custom', 'doctype', 'custom_field') frappe.patches.v14_0.update_workspace2 # 20.09.2021 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021 diff --git a/frappe/patches/v13_0/encrypt_2fa_secrets.py b/frappe/patches/v13_0/encrypt_2fa_secrets.py index 3b220f485f..1814ff50c5 100644 --- a/frappe/patches/v13_0/encrypt_2fa_secrets.py +++ b/frappe/patches/v13_0/encrypt_2fa_secrets.py @@ -39,6 +39,7 @@ def execute(): .set(table.parent, PARENT_FOR_DEFAULTS) .set(table.defvalue, defvalue_cases) .where(table.parent == OLD_PARENT) + .where(table.defkey.like("%_otpsecret")) ).run() clear_defaults_cache() diff --git a/frappe/patches/v13_0/reset_corrupt_defaults.py b/frappe/patches/v13_0/reset_corrupt_defaults.py new file mode 100644 index 0000000000..10e81c7ff1 --- /dev/null +++ b/frappe/patches/v13_0/reset_corrupt_defaults.py @@ -0,0 +1,33 @@ +import frappe +from frappe.patches.v13_0.encrypt_2fa_secrets import DOCTYPE +from frappe.patches.v13_0.encrypt_2fa_secrets import PARENT_FOR_DEFAULTS as TWOFACTOR_PARENT +from frappe.utils import cint + + +def execute(): + """ + This patch is needed to fix parent incorrectly set as `__2fa` because of + https://github.com/frappe/frappe/commit/a822092211533ff17ff9b92dd86f6f868ed63e2e + """ + + if not frappe.db.get_value( + DOCTYPE, {"parent": TWOFACTOR_PARENT, "defkey": ("not like", "%_otp%")}, "defkey" + ): + return + + # system settings + system_settings = frappe.get_single("System Settings") + system_settings.set_defaults() + + # home page + frappe.db.set_default( + "desktop:home_page", "workspace" if cint(system_settings.setup_complete) else "setup-wizard" + ) + + # letter head + try: + letter_head = frappe.get_doc("Letter Head", {"is_default": 1}) + letter_head.set_as_default() + + except frappe.DoesNotExistError: + pass From d2177d16a1d21e794c21373dccbf3638543280a5 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 21 Jul 2022 16:54:41 +0530 Subject: [PATCH 190/201] ci: bump node version to node16 (#17564) * ci: bump node version to node16 * chore: remove `node-sass` - no longer used Co-authored-by: Sagar Vora --- .github/helper/install.sh | 5 ----- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/publish-assets-develop.yml | 2 +- .github/workflows/publish-assets-releases.yml | 4 +++- .github/workflows/release.yml | 6 +++--- .github/workflows/semantic-commits.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 2 +- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/ui-tests.yml | 2 +- 9 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index b36c1e4b12..e59fa36627 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -56,11 +56,6 @@ bench -v setup requirements --dev if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi -# install node-sass which is required for website theme test -cd ./apps/frappe || exit -yarn add node-sass@4.13.1 -cd ../.. - bench start & bench --site test_site reinstall --yes if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 1e21ae8549..655f99e9dd 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -49,7 +49,7 @@ jobs: if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index b216718b99..467922e766 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -15,7 +15,7 @@ jobs: path: 'frappe' - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 - uses: actions/setup-python@v4 with: python-version: '3.10' diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index 2612c45bea..ff1656e55d 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -16,9 +16,11 @@ jobs: - uses: actions/checkout@v3 with: path: 'frappe' + - uses: actions/setup-node@v3 with: - python-version: '12.x' + node-version: 16 + - uses: actions/setup-python@v4 with: python-version: '3.10' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f73bed09c7..010022b7f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,10 +16,10 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Setup Node.js v14 + - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 - name: Setup dependencies run: | npm install @semantic-release/git @semantic-release/exec --no-save @@ -31,4 +31,4 @@ jobs: GIT_AUTHOR_EMAIL: "developers@frappe.io" GIT_COMMITTER_NAME: "Frappe PR Bot" GIT_COMMITTER_EMAIL: "developers@frappe.io" - run: npx semantic-release \ No newline at end of file + run: npx semantic-release diff --git a/.github/workflows/semantic-commits.yml b/.github/workflows/semantic-commits.yml index a3536d5019..7afa02d1b9 100644 --- a/.github/workflows/semantic-commits.yml +++ b/.github/workflows/semantic-commits.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 check-latest: true - name: Check commit titles diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 29d88fd9a5..84bb4ba8dc 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -56,7 +56,7 @@ jobs: - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 8f015f43e6..09bfad4304 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -59,7 +59,7 @@ jobs: - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: '14' + node-version: '16' check-latest: true - name: Add to Hosts diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index da6b095451..1d0d28223e 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -55,7 +55,7 @@ jobs: - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts From 9e87598ddaa19b565099685c2bdad636b29b3d4f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 21 Jul 2022 17:03:52 +0530 Subject: [PATCH 191/201] refactor: Use FrappeTestCase as it rolls back test data --- frappe/tests/test_boot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_boot.py b/frappe/tests/test_boot.py index 8f26a078b0..11cbeb7621 100644 --- a/frappe/tests/test_boot.py +++ b/frappe/tests/test_boot.py @@ -3,9 +3,10 @@ import unittest import frappe from frappe.boot import get_unseen_notes from frappe.desk.doctype.note.note import mark_as_seen +from frappe.tests.utils import FrappeTestCase -class TestBootData(unittest.TestCase): +class TestBootData(FrappeTestCase): def test_get_unseen_notes(self): frappe.db.delete("Note") frappe.db.delete("Note Seen By") From b1e9bc8d12d35d20e9b9bee0f29922753902d0ca Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Thu, 21 Jul 2022 20:46:03 +0530 Subject: [PATCH 192/201] fix: error while genarating date for blog post google search preview (#17581) --- frappe/website/doctype/blog_post/blog_post.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.js b/frappe/website/doctype/blog_post/blog_post.js index 97916b6fc6..08bc55f07c 100644 --- a/frappe/website/doctype/blog_post/blog_post.js +++ b/frappe/website/doctype/blog_post/blog_post.js @@ -2,21 +2,21 @@ // For license information, please see license.txt frappe.ui.form.on('Blog Post', { - refresh: function(frm) { + refresh: function (frm) { frappe.db.get_single_value('Blog Settings', 'show_cta_in_blog').then(value => { frm.set_df_property("hide_cta", "hidden", !value); }); generate_google_search_preview(frm); }, - title: function(frm) { + title: function (frm) { generate_google_search_preview(frm); frm.trigger('set_route'); }, - meta_description: function(frm) { + meta_description: function (frm) { generate_google_search_preview(frm); }, - blog_intro: function(frm) { + blog_intro: function (frm) { generate_google_search_preview(frm); }, blog_category(frm) { @@ -36,8 +36,8 @@ function generate_google_search_preview(frm) { if (!(frm.doc.meta_title || frm.doc.title)) return; let google_preview = frm.get_field("google_preview"); let seo_title = (frm.doc.meta_title || frm.doc.title).slice(0, 60); - let seo_description = (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160); - let date = frm.doc.published_on ? new frappe.datetime.datetime(frm.doc.published_on).moment.format('ll') + ' - ' : ''; + let seo_description = (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160); + let date = frm.doc.published_on ? moment(frm.doc.published_on).format('ll') + '-' : ''; let route_array = frm.doc.route ? frm.doc.route.split('/') : []; route_array.pop(); @@ -49,10 +49,10 @@ function generate_google_search_preview(frm) { › ${route_array.join(' › ')}
- ${ seo_title } + ${seo_title}

- ${ date } ${ seo_description } + ${date} ${seo_description}

`); From 006ebcbedeb41fb21babf0a6c11a709a0a3525eb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Jul 2022 11:17:02 +0530 Subject: [PATCH 193/201] refactor: Use pymysql over mariadb client This is supposed to be a temporary switch to make the parent PR easier to digest. MariaDB client has some issues with release, and system dependencies. This commit may be reverted to enable mariadb client again. --- frappe/__init__.py | 1 - frappe/commands/site.py | 2 - frappe/database/mariadb/database.py | 253 ++++++---------------------- frappe/utils/bench_helper.py | 2 - pyproject.toml | 1 - 5 files changed, 55 insertions(+), 204 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 3d3288c16d..20fddb0267 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -48,7 +48,6 @@ __title__ = "Frappe Framework" controllers = {} local = Local() STANDARD_USERS = ("Guest", "Administrator") -DISABLE_DATABASE_CONNECTION_POOLING = None _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) _qb_patched = {} diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 62451b6013..e3c7de32a3 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -69,8 +69,6 @@ def new_site( "Create a new site" from frappe.installer import _new_site - frappe.DISABLE_DATABASE_CONNECTION_POOLING = True - frappe.init(site=site, new_site=True) _new_site( diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 71317b5884..e89168194e 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,114 +1,87 @@ import re -from collections import defaultdict -from decimal import Decimal -from typing import TYPE_CHECKING -import mariadb -from mariadb.constants import ERR, FIELD_TYPE -from pymysql.converters import escape_sequence, escape_string +import pymysql +from pymysql.constants import ER, FIELD_TYPE +from pymysql.converters import conversions, escape_string import frappe -from frappe.database.database import Database, QueryValues +from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable -from frappe.utils import UnicodeWithAttrs, get_datetime, get_table_name +from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name -if TYPE_CHECKING: - from mariadb import ConnectionPool - -_FIND_ITER_PATTERN = re.compile("%s") _PARAM_COMP = re.compile(r"%\([\w]*\)s") -_SITE_POOLS = defaultdict(frappe._dict) -_MAX_POOL_SIZE = 64 -_POOL_SIZE = 1 - -# _POOL_SIZE is selected "arbitrarily" to avoid overloading the server and being mindful of multitenancy -# init size of connection pool will be _POOL_SIZE for each site. Replica setups will have separate pool. -# This means each site with a replica setup can have 2 active pools of size _POOL_SIZE each. Each pool may -# expand up to _MAX_POOL_SIZE as per requirement. This cannot be a function of @@global.max_connections, -# no. of sites since there may be multiple processes holding connections; and this defines the size for each -# of those processes/workers. Check MariaDBConnectionUtil for connection & pool management. - - -def is_connection_pooling_enabled() -> bool: - """Set `frappe.DISABLE_CONNECTION_POOLING` to enable/disable connection pooling for all on current - process. This will override config key `disable_database_connection_pooling`. Set key - `disable_database_connection_pooling` in site config for persistent settings across workers.""" - - if frappe.DISABLE_DATABASE_CONNECTION_POOLING is not None: - return not frappe.DISABLE_DATABASE_CONNECTION_POOLING - return not frappe.local.conf.disable_database_connection_pooling class MariaDBExceptionUtil: - ProgrammingError = mariadb.ProgrammingError - TableMissingError = mariadb.ProgrammingError - OperationalError = mariadb.OperationalError - InternalError = mariadb.InternalError - SQLError = mariadb.ProgrammingError - DataError = mariadb.DataError + ProgrammingError = pymysql.ProgrammingError + TableMissingError = pymysql.ProgrammingError + OperationalError = pymysql.OperationalError + InternalError = pymysql.InternalError + SQLError = pymysql.ProgrammingError + DataError = pymysql.DataError # match ER_SEQUENCE_RUN_OUT - https://mariadb.com/kb/en/mariadb-error-codes/ - SequenceGeneratorLimitExceeded = mariadb.OperationalError + SequenceGeneratorLimitExceeded = pymysql.OperationalError SequenceGeneratorLimitExceeded.errno = 4084 @staticmethod - def is_deadlocked(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_LOCK_DEADLOCK + def is_deadlocked(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_DEADLOCK @staticmethod - def is_timedout(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_LOCK_WAIT_TIMEOUT + def is_timedout(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_WAIT_TIMEOUT @staticmethod - def is_table_missing(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_NO_SUCH_TABLE + def is_table_missing(e: pymysql.Error) -> bool: + return e.args[0] == ER.NO_SUCH_TABLE @staticmethod - def is_missing_table(e: mariadb.Error) -> bool: + def is_missing_table(e: pymysql.Error) -> bool: return MariaDBDatabase.is_table_missing(e) @staticmethod - def is_missing_column(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_BAD_FIELD_ERROR + def is_missing_column(e: pymysql.Error) -> bool: + return e.args[0] == ER.BAD_FIELD_ERROR @staticmethod - def is_duplicate_fieldname(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_DUP_FIELDNAME + def is_duplicate_fieldname(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_FIELDNAME @staticmethod - def is_duplicate_entry(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_DUP_ENTRY + def is_duplicate_entry(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_ENTRY @staticmethod - def is_access_denied(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_ACCESS_DENIED_ERROR + def is_access_denied(e: pymysql.Error) -> bool: + return e.args[0] == ER.ACCESS_DENIED_ERROR @staticmethod - def cant_drop_field_or_key(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_CANT_DROP_FIELD_OR_KEY + def cant_drop_field_or_key(e: pymysql.Error) -> bool: + return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY @staticmethod - def is_syntax_error(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_PARSE_ERROR + def is_syntax_error(e: pymysql.Error) -> bool: + return e.args[0] == ER.PARSE_ERROR @staticmethod - def is_data_too_long(e: mariadb.Error) -> bool: - return getattr(e, "errno", None) == ERR.ER_DATA_TOO_LONG + def is_data_too_long(e: pymysql.Error) -> bool: + return e.args[0] == ER.DATA_TOO_LONG @staticmethod - def is_primary_key_violation(e: mariadb.Error) -> bool: + def is_primary_key_violation(e: pymysql.Error) -> bool: return ( MariaDBDatabase.is_duplicate_entry(e) - and "PRIMARY" in e.errmsg - and isinstance(e, mariadb.IntegrityError) + and "PRIMARY" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) ) @staticmethod - def is_unique_key_violation(e: mariadb.Error) -> bool: + def is_unique_key_violation(e: pymysql.Error) -> bool: return ( MariaDBDatabase.is_duplicate_entry(e) - and "Duplicate" in e.errmsg - and isinstance(e, mariadb.IntegrityError) + and "Duplicate" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) ) @@ -118,90 +91,21 @@ class MariaDBConnectionUtil: conn.auto_reconnect = True return conn - def _get_connection(self) -> "mariadb.Connection": - """Return MariaDB connection object. - - If frappe.conf.disable_database_connection_pooling is set, return a new connection - object and close existing pool if exists. Else, return a connection from the pool. - """ - global _SITE_POOLS - - # don't pool root connections - if self.user == "root": - return self.create_connection() - - if not is_connection_pooling_enabled(): - self.close_connection_pools() - return self.create_connection() - - if frappe.local.site not in _SITE_POOLS: - site_pool = self.create_connection_pool() - else: - site_pool = self.get_connection_pool() - - try: - conn = site_pool.get_connection() - except mariadb.PoolError: - # PoolError is raised when the pool is exhausted - conn = self.create_connection() - try: - site_pool.add_connection(conn) - # log this via frappe.logger & continue - site needs bigger pool...over _POOL_SIZE - except mariadb.PoolError: - # PoolError is raised when size limit is reached - # log this via frappe.logger & continue - site needs a much bigger pool...over _MAX_POOL_SIZE - pass - - return conn - - def close_connection_pools(self): - if frappe.local.site in _SITE_POOLS: - pools = _SITE_POOLS[frappe.local.site] - for pool in pools.values(): - try: - pool.close() - except Exception: - pass - _SITE_POOLS.pop(frappe.local.site, None) - - def get_pool_name(self) -> str: - pool_type = "read-only" if self.read_only else "default" - return f"{frappe.local.site}-{pool_type}" - - def get_connection_pool(self) -> "ConnectionPool": - """Return MariaDB connection pool object. - - If `read_only` is True, return a read only pool. - """ - return _SITE_POOLS[frappe.local.site]["read_only" if self.read_only else "default"] - - def create_connection_pool(self): - pool = mariadb.ConnectionPool( - pool_name=self.get_pool_name(), - pool_size=_MAX_POOL_SIZE, - pool_reset_connection=False, - ) - pool.set_config(**self.get_connection_settings()) - - if self.read_only: - _SITE_POOLS[frappe.local.site].read_only = pool - else: - _SITE_POOLS[frappe.local.site].default = pool - - for _ in range(_POOL_SIZE): - pool.add_connection() - - return pool + def _get_connection(self): + """Return MariaDB connection object.""" + return self.create_connection() def create_connection(self): - return mariadb.connect(**self.get_connection_settings()) + return pymysql.connect(**self.get_connection_settings()) def get_connection_settings(self) -> dict: conn_settings = { "host": self.host, "user": self.user, "password": self.password, - "converter": self.CONVERSION_MAP, + "conv": self.CONVERSION_MAP, + "charset": "utf8mb4", + "use_unicode": True, } if self.user != "root": @@ -215,63 +119,15 @@ class MariaDBConnectionUtil: if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: ssl_params = { - "ssl": True, - "ssl_ca": frappe.conf.db_ssl_ca, - "ssl_cert": frappe.conf.db_ssl_cert, - "ssl_key": frappe.conf.db_ssl_key, + "ca": frappe.conf.db_ssl_ca, + "cert": frappe.conf.db_ssl_cert, + "key": frappe.conf.db_ssl_key, } conn_settings.update(ssl_params) return conn_settings -class MariaDBCursorPatchUtil: - """Patch mariadb.cursor.Cursor to handle things not supported by pinned version of MariaDB client.""" - - def _transform_query(self, query: str, values: QueryValues) -> tuple: - """Transform the query to handle things not supported by pinned version of MariaDB client. - - Transformations: - - Escape sequences in values - """ - _values = [] - - if isinstance(values, (tuple, list)): - for val in values: - if isinstance(val, (tuple, list)): - _values.append(escape_sequence(val, charset=self._conn.character_set)) - else: - _values.append(val) - values = _values - else: - for token in _PARAM_COMP.findall(query): - key = token[2:-2] - try: - val = values[key] - except KeyError: - raise self.ProgrammingError(f"Missing value for key '{key}'") - if isinstance(val, (tuple, list)): - values[key] = escape_sequence(val, charset=self._conn.character_set) - - return query, values or [] - - def _transform_result(self, result: list[tuple]) -> list[tuple]: - # ref: https://jira.mariadb.org/projects/CONPY/issues/CONPY-213 - _result = [] - for row in result: - _row = [] - for el in row: - if isinstance(el, Decimal): - el = float(el) - elif isinstance(el, UnicodeWithAttrs): - el = escape_string(el) - _row.append(el) - _result.append(tuple(_row)) - return _result - - -class MariaDBDatabase( - MariaDBCursorPatchUtil, MariaDBConnectionUtil, MariaDBExceptionUtil, Database -): +class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): REGEX_CHARACTER = "regexp" # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, @@ -282,7 +138,7 @@ class MariaDBDatabase( # using the system after a restore. # issue link: https://jira.mariadb.org/browse/MDEV-21786 SEQUENCE_CACHE = 50 - CONVERSION_MAP = { + CONVERSION_MAP = conversions | { FIELD_TYPE.NEWDECIMAL: float, FIELD_TYPE.DATETIME: get_datetime, UnicodeWithAttrs: escape_string, @@ -342,7 +198,8 @@ class MariaDBDatabase( return db_size[0].get("database_size") def log_query(self, query, values, debug, explain): - self.last_query = super().log_query(query, values, debug, explain) + self.last_query = self._cursor._last_executed + self._log_query(query, debug, explain) return self.last_query @staticmethod @@ -368,11 +225,11 @@ class MariaDBDatabase( # column type @staticmethod def is_type_number(code): - return code == mariadb.NUMBER + return code == pymysql.NUMBER @staticmethod def is_type_datetime(code): - return code == mariadb.DATETIME + return code == pymysql.DATETIME def rename_table(self, old_name: str, new_name: str) -> list | tuple: old_name = get_table_name(old_name) diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index 10ace1b1b6..a0b011acc1 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -106,6 +106,4 @@ if __name__ == "__main__": if not frappe._dev_server: warnings.simplefilter("ignore") - frappe.DISABLE_DATABASE_CONNECTION_POOLING = not int(os.environ.get("DATABASE_POOLING", "0")) - main() diff --git a/pyproject.toml b/pyproject.toml index c3ef944b85..5eeb6f46dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dependencies = [ "html5lib~=1.1", "ipython~=8.4.0", "ldap3~=2.9", - "mariadb~=1.1.2", "markdown2~=2.4.0", "maxminddb-geolite2==2018.703", "num2words~=0.5.10", From 63e618c7097a0049a0d0a25f46e7198d9268b8a9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Jul 2022 12:31:55 +0530 Subject: [PATCH 194/201] test: More resilient tests for sequences --- frappe/tests/test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 0e172c8f5b..bb64d0278d 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -574,7 +574,7 @@ class TestDDLCommandsMaria(unittest.TestCase): self.test_table_name = new_table_name def test_describe(self) -> None: - self.assertEqual( + self.assertSequenceEqual( [ ("id", "int(11)", "NO", "PRI", None, ""), ("content", "text", "YES", "", None, ""), @@ -798,7 +798,7 @@ class TestDDLCommandsPost(unittest.TestCase): self.test_table_name = new_table_name def test_describe(self) -> None: - self.assertEqual([("id",), ("content",)], frappe.db.describe(self.test_table_name)) + self.assertSequenceEqual([("id",), ("content",)], frappe.db.describe(self.test_table_name)) def test_change_type(self) -> None: from psycopg2.errors import DatatypeMismatch From e411132c6e8a6bebb719c37303e350958894026f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Jul 2022 16:27:54 +0530 Subject: [PATCH 195/201] fix(db): Revert breaking change of wrapping NoneType in sequence --- frappe/database/database.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index f205041acf..40529d0ec9 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -9,7 +9,6 @@ import string import traceback from contextlib import contextmanager from time import time -from types import NoneType from pypika.terms import Criterion, NullValue @@ -105,7 +104,7 @@ class Database: raise NotImplementedError def _transform_query(self, query: Query, values: QueryValues) -> tuple: - return query, values or None + return query, values def _transform_result(self, result: list[tuple]) -> list[tuple]: return result @@ -113,7 +112,7 @@ class Database: def sql( self, query: Query, - values: QueryValues = None, + values: QueryValues = (), as_dict=0, as_list=0, formatted=0, @@ -176,7 +175,7 @@ class Database: if debug: time_start = time() - if not isinstance(values, (NoneType, tuple, dict, list)): + if not isinstance(values, (tuple, dict, list)): values = (values,) query, values = self._transform_query(query, values) From 1a610e135df4cc330fc6ab592839282c7efc36a7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Jul 2022 17:38:40 +0530 Subject: [PATCH 196/201] fix(db): Use sentinel object for default values paramters --- frappe/database/database.py | 8 +++++--- frappe/database/postgres/database.py | 6 +++--- frappe/database/utils.py | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 40529d0ec9..e55af037f2 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -16,7 +16,7 @@ import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.database.utils import LazyMogrify, Query, QueryValues, is_query_type +from frappe.database.utils import EmptyQueryValues, LazyMogrify, Query, QueryValues, is_query_type from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count @@ -112,7 +112,7 @@ class Database: def sql( self, query: Query, - values: QueryValues = (), + values: QueryValues = EmptyQueryValues, as_dict=0, as_list=0, formatted=0, @@ -175,7 +175,9 @@ class Database: if debug: time_start = time() - if not isinstance(values, (tuple, dict, list)): + if values == EmptyQueryValues: + values = None + elif not isinstance(values, (tuple, dict, list)): values = (values,) query, values = self._transform_query(query, values) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index ec3e0084f9..85e4f2f0f7 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -18,7 +18,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable -from frappe.database.utils import LazyDecode +from frappe.database.utils import EmptyQueryValues, LazyDecode from frappe.utils import cstr, get_table_name # cast decimals as floats @@ -188,7 +188,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): return db_size[0].get("database_size") # pylint: disable=W0221 - def sql(self, query, values=(), *args, **kwargs): + def sql(self, query, values=EmptyQueryValues, *args, **kwargs): return super().sql(modify_query(query), modify_values(values), *args, **kwargs) def lazy_mogrify(self, *args, **kwargs) -> str: @@ -419,7 +419,7 @@ def modify_values(values): return value - if not values: + if not values or values == EmptyQueryValues: return values if isinstance(values, dict): diff --git a/frappe/database/utils.py b/frappe/database/utils.py index e71ffbf0d6..4bff2f78e6 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -10,6 +10,8 @@ from frappe.query_builder.builder import MariaDB, Postgres Query = str | MariaDB | Postgres QueryValues = tuple | list | dict | NoneType +EmptyQueryValues = object() + def is_query_type(query: str, query_type: str | tuple[str]) -> bool: return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) From bcfa8c276e46d00e6927924ce190d6ea2b663c95 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 22 Jul 2022 13:16:26 +0530 Subject: [PATCH 197/201] fix: Set default port attribute for Database classes db.default_port wil be available as a class attribute to hold defaults for DB types. Usage: frappe.conf.db_port or frappe.db.default_port Why: I couldn't run the mariadb command because the defaults aren't set for my system. server is remote / containerized. Setting port in equivalent mysql command fixes this. --- frappe/commands/utils.py | 30 +++++++++++++++------------- frappe/database/mariadb/database.py | 1 + frappe/database/postgres/database.py | 1 + frappe/tests/test_db.py | 1 - frappe/utils/backups.py | 6 +----- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index d0024aed73..3658a35992 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -523,22 +523,24 @@ def postgres(context): def _mariadb(): + from frappe.database.mariadb.database import MariaDBDatabase + mysql = find_executable("mysql") - os.execv( + command = [ mysql, - [ - mysql, - "-u", - frappe.conf.db_name, - "-p" + frappe.conf.db_password, - frappe.conf.db_name, - "-h", - frappe.conf.db_host or "localhost", - "--pager=less -SFX", - "--safe-updates", - "-A", - ], - ) + "--port", + frappe.conf.db_port or MariaDBDatabase.default_port, + "-u", + frappe.conf.db_name, + f"-p{frappe.conf.db_password}", + frappe.conf.db_name, + "-h", + frappe.conf.db_host or "localhost", + "--pager=less -SFX", + "--safe-updates", + "-A", + ] + os.execv(mysql, command) def _psql(): diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index e89168194e..4bdc1688e9 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -143,6 +143,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): FIELD_TYPE.DATETIME: get_datetime, UnicodeWithAttrs: escape_string, } + default_port = "3306" def setup_type_map(self): self.db_type = "mariadb" diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 85e4f2f0f7..58b63e6547 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -108,6 +108,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database): # to the next non-cached value hence not using cache in postgres. # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers SEQUENCE_CACHE = 0 + default_port = "5432" def setup_type_map(self): self.db_type = "postgres" diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index bb64d0278d..b658919ef5 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -97,7 +97,6 @@ class TestDB(unittest.TestCase): ) def test_get_value_limits(self): - # check both dict and list style filters filters = [{"enabled": 1}, [["enabled", "=", 1]]] for filter in filters: diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index e6068fd299..f7b38b0055 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -73,11 +73,7 @@ class BackupGenerator: if not self.db_type: self.db_type = "mariadb" - if not self.db_port: - if self.db_type == "mariadb": - self.db_port = 3306 - if self.db_type == "postgres": - self.db_port = 5432 + self.db_port = self.db_port or frappe.db.default_port site = frappe.local.site or frappe.generate_hash(length=8) self.site_slug = site.replace(".", "_") From 1dbc0b4d3ca15a53b78349a21fcd5a8f72c503cd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 22 Jul 2022 13:19:25 +0530 Subject: [PATCH 198/201] fix(db*): Use common fallback Datetime str There existed inconsistencies between db_query & db's fallback for min datetime in str format - missing decimal seconds places. Now, we're storing the default string once and re-using it to reduce inconsistencies or room for human errors. --- frappe/database/database.py | 11 +++++++++-- frappe/database/utils.py | 1 + frappe/model/db_query.py | 9 +++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index e55af037f2..68286507ba 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -16,7 +16,14 @@ import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.database.utils import EmptyQueryValues, LazyMogrify, Query, QueryValues, is_query_type +from frappe.database.utils import ( + EmptyQueryValues, + FallBackDateTimeStr, + LazyMogrify, + Query, + QueryValues, + is_query_type, +) from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count @@ -1071,7 +1078,7 @@ class Database: @staticmethod def format_datetime(datetime): if not datetime: - return "0001-01-01 00:00:00.000000" + return FallBackDateTimeStr if isinstance(datetime, str): if ":" not in datetime: diff --git a/frappe/database/utils.py b/frappe/database/utils.py index 4bff2f78e6..c4d8cb4953 100644 --- a/frappe/database/utils.py +++ b/frappe/database/utils.py @@ -11,6 +11,7 @@ Query = str | MariaDB | Postgres QueryValues = tuple | list | dict | NoneType EmptyQueryValues = object() +FallBackDateTimeStr = "0001-01-01 00:00:00.000000" def is_query_type(query: str, query_type: str | tuple[str]) -> bool: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 4c62992e21..a29ede37bf 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -13,6 +13,7 @@ import frappe.permissions import frappe.share from frappe import _ from frappe.core.doctype.server_script.server_script_utils import get_server_script_map +from frappe.database.utils import FallBackDateTimeStr from frappe.model import optional_fields from frappe.model.meta import get_table_columns from frappe.model.utils.user_settings import get_user_settings, update_user_settings @@ -632,11 +633,11 @@ class DatabaseQuery: date_range = get_date_range(f.operator.lower(), f.value) f.operator = "Between" f.value = date_range - fallback = "'0001-01-01 00:00:00'" + fallback = f"'{FallBackDateTimeStr}'" if f.operator in (">", "<") and (f.fieldname in ("creation", "modified")): value = cstr(f.value) - fallback = "'0001-01-01 00:00:00'" + fallback = f"'{FallBackDateTimeStr}'" elif f.operator.lower() in ("between") and ( f.fieldname in ("creation", "modified") @@ -644,7 +645,7 @@ class DatabaseQuery: ): value = get_between_date_filter(f.value, df) - fallback = "'0001-01-01 00:00:00'" + fallback = f"'{FallBackDateTimeStr}'" elif f.operator.lower() == "is": if f.value == "set": @@ -665,7 +666,7 @@ class DatabaseQuery: elif (df and df.fieldtype == "Datetime") or isinstance(f.value, datetime): value = frappe.db.format_datetime(f.value) - fallback = "'0001-01-01 00:00:00'" + fallback = f"'{FallBackDateTimeStr}'" elif df and df.fieldtype == "Time": value = get_time(f.value).strftime("%H:%M:%S.%f") From 2bef29bb466c4ad2b23888421e0ac4a9f0c03690 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 22 Jul 2022 13:21:29 +0530 Subject: [PATCH 199/201] chore: Deprecate backup script + minor refactors --- frappe/database/mariadb/database.py | 2 +- frappe/tests/test_db.py | 2 +- frappe/tests/test_fmt_datetime.py | 2 +- frappe/utils/backups.py | 8 ++++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 4bdc1688e9..5e6d62f842 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -123,7 +123,7 @@ class MariaDBConnectionUtil: "cert": frappe.conf.db_ssl_cert, "key": frappe.conf.db_ssl_key, } - conn_settings.update(ssl_params) + conn_settings |= ssl_params return conn_settings diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index b658919ef5..b1ff0f81df 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -135,7 +135,7 @@ class TestDB(unittest.TestCase): test_inputs = [ {"fieldtype": fieldtype, "value": value} for fieldtype, value in values_dict.items() ] - for fieldtype in values_dict.keys(): + for fieldtype in values_dict: create_custom_field( "Print Settings", { diff --git a/frappe/tests/test_fmt_datetime.py b/frappe/tests/test_fmt_datetime.py index 031b8c323c..706aca6d7c 100644 --- a/frappe/tests/test_fmt_datetime.py +++ b/frappe/tests/test_fmt_datetime.py @@ -125,5 +125,5 @@ class TestFmtDatetime(unittest.TestCase): for time_fmt, valid_time in test_time_formats.items(): frappe.db.set_default("time_format", time_fmt) frappe.local.user_time_format = None - valid_fmt = valid_date + " " + valid_time + valid_fmt = f"{valid_date} {valid_time}" self.assertEqual(format_datetime(test_datetime), valid_fmt) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index f7b38b0055..f5e13c0873 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -697,6 +697,14 @@ def backup( if __name__ == "__main__": import sys + from frappe.utils.commands import warn + + warn( + "Calling the backup script directly is deprecated. " + "Use the backup command instead. This script will be removed in Frappe v15.", + category=DeprecationWarning, + ) + cmd = sys.argv[1] db_type = "mariadb" From d1fbab1c45c0f04f126bfc2ccf2a1c14a1bccd10 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 22 Jul 2022 13:30:47 +0530 Subject: [PATCH 200/201] test(db): Add tests for untested db methods --- frappe/tests/test_db.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index b1ff0f81df..bb6db76bf5 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -9,9 +9,11 @@ from random import choice from unittest.mock import patch import frappe +from frappe.core.utils import find from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.database import savepoint from frappe.database.database import Database +from frappe.database.utils import FallBackDateTimeStr from frappe.query_builder import Field from frappe.query_builder.functions import Concat_ws from frappe.tests.test_query_builder import db_type_is, run_only_if @@ -20,6 +22,20 @@ from frappe.utils.testutils import clear_custom_fields class TestDB(unittest.TestCase): + def test_datetime_format(self): + now_str = now() + self.assertEqual(frappe.db.format_datetime(None), FallBackDateTimeStr) + self.assertEqual(frappe.db.format_datetime(now_str), now_str) + + @run_only_if(db_type_is.MARIADB) + def test_get_column_type(self): + desc_data = frappe.db.sql("desc `tabUser`", as_dict=1) + user_name_type = find(desc_data, lambda x: x["Field"] == "name")["Type"] + self.assertEqual(frappe.db.get_column_type("User", "name"), user_name_type) + + def test_get_database_size(self): + self.assertIsInstance(frappe.db.get_database_size(), (float, int)) + def test_get_value(self): self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator") From f40214bca8dee3564ec7a0c073c0e1ab3cd67502 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 22 Jul 2022 13:34:04 +0530 Subject: [PATCH 201/201] fix(get_last_doc): Allow for_update as kwarg only Co-authored-by: Ankush Menat --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 20fddb0267..e1fa902eba 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1182,7 +1182,7 @@ def get_doc(*args, **kwargs) -> "Document": return doc -def get_last_doc(doctype, filters=None, order_by="creation desc", for_update=False): +def get_last_doc(doctype, filters=None, order_by="creation desc", *, for_update=False): """Get last created document of this type.""" d = get_all(doctype, filters=filters, limit_page_length=1, order_by=order_by, pluck="name") if d: