From 7c77bedbf26156e0e34c470f0b7a8f447a4ea9a3 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 18 May 2022 16:57:54 +0530 Subject: [PATCH 1/5] refactor: Simplify logic + Add typing hints --- frappe/database/database.py | 12 ++++-------- frappe/database/query.py | 8 ++------ frappe/model/db_query.py | 7 +++++-- frappe/utils/data.py | 2 +- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1a03ac3889..42135f3cd5 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1026,14 +1026,10 @@ class Database(object): if cache_count is not None: return cache_count query = self.query.get_sql(table=dt, filters=filters, fields=Count("*"), distinct=distinct) - if filters: - count = self.sql(query, debug=debug)[0][0] - return count - else: - count = self.sql(query, debug=debug)[0][0] - if cache: - frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) - return count + count = self.sql(query, debug=debug)[0][0] + if not filters and cache: + frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) + return count @staticmethod def format_date(date): diff --git a/frappe/database/query.py b/frappe/database/query.py index b107759af0..2af84cc29a 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -269,12 +269,8 @@ class Query: conditions = conditions.where(make_function(key, value)) continue if isinstance(value, (list, tuple)): - if isinstance(value[1], (list, tuple)) or value[0] in list(OPERATOR_MAP.keys())[-4:]: - _operator = OPERATOR_MAP[value[0]] - conditions = conditions.where(_operator(Field(key), value[1])) - else: - _operator = OPERATOR_MAP[value[0]] - conditions = conditions.where(_operator(Field(key), value[1])) + _operator = OPERATOR_MAP[value[0].casefold()] + conditions = conditions.where(_operator(Field(key), value[1])) else: if value is not None: conditions = conditions.where(_operator(Field(key), value)) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 005b7e3741..54331f5124 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -1041,7 +1041,7 @@ def get_additional_filter_field(additional_filters_config, f, value): return f -def get_date_range(operator, value): +def get_date_range(operator: str, value: str): timespan_map = { "1 week": "week", "1 month": "month", @@ -1054,7 +1054,10 @@ def get_date_range(operator, value): "next": "next", } - timespan = period_map[operator] + " " + timespan_map[value] if operator != "timespan" else value + if operator != "timespan": + timespan = f"{period_map[operator]} {timespan_map[value]}" + else: + timespan = value return get_timespan_date_range(timespan) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 7311756fcd..6a9ffc81a6 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -714,7 +714,7 @@ def get_weekday(datetime: Optional[datetime.datetime] = None) -> str: return weekdays[datetime.weekday()] -def get_timespan_date_range(timespan): +def get_timespan_date_range(timespan: str) -> Tuple[datetime.datetime, datetime.datetime]: today = nowdate() date_range_map = { "last week": lambda: ( From 5ebaf424689c52cd88cc428842feb917ad7857e0 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 18 May 2022 16:59:10 +0530 Subject: [PATCH 2/5] feat(db.query): Add support for timespan operator --- frappe/database/query.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/frappe/database/query.py b/frappe/database/query.py index 2af84cc29a..42b2a60af5 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Tuple, Union import frappe from frappe import _ +from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table @@ -90,6 +91,20 @@ def func_is(key, value): return Field(key).isnotnull() if value.lower() == "set" else Field(key).isnull() +def func_timespan(key: Field, value: str) -> frappe.qb: + """Wrapper method for `TIMESPAN` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `TIMESPAN` + """ + + return func_between(key, get_timespan_date_range(value)) + + def make_function(key: Any, value: Union[int, str]): """returns fucntion query @@ -141,6 +156,7 @@ OPERATOR_MAP = { "regex": func_regex, "between": func_between, "is": func_is, + "timespan": func_timespan, } From 72ad582b8a37f0419719e9122b7b47908e7fea88 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 18 May 2022 16:59:53 +0530 Subject: [PATCH 3/5] fix(db.query): Casefold key to match operators --- frappe/database/query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 42b2a60af5..ad7b3f83b4 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -115,7 +115,7 @@ def make_function(key: Any, value: Union[int, str]): Returns: frappe.qb: frappe.qb object """ - return OPERATOR_MAP[value[0]](key, value[1]) + return OPERATOR_MAP[value[0].casefold()](key, value[1]) def change_orderby(order: str): @@ -138,7 +138,7 @@ def change_orderby(order: str): return order[0], Order.desc -OPERATOR_MAP = { +OPERATOR_MAP: Dict[str, "function"] = { "+": operator.add, "=": operator.eq, "-": operator.sub, @@ -243,14 +243,14 @@ class Query: if isinstance(filters, list): for f in filters: if not isinstance(f, (list, tuple)): - _operator = OPERATOR_MAP[filters[1]] + _operator = OPERATOR_MAP[filters[1].casefold()] if not isinstance(filters[0], str): conditions = make_function(filters[0], filters[2]) break conditions = conditions.where(_operator(Field(filters[0]), filters[2])) break else: - _operator = OPERATOR_MAP[f[-2]] + _operator = OPERATOR_MAP[f[-2].casefold()] if len(f) == 4: table_object = self.get_table(f[0]) _field = table_object[f[1]] From db11af2a5cf39f7862c5335a95b5ed010d7a2924 Mon Sep 17 00:00:00 2001 From: gavin Date: Thu, 19 May 2022 17:31:35 +0530 Subject: [PATCH 4/5] feat(wip): Custom filters in db.query engine * Added provision for semi-implemneted version * Hard to fix it completely given it's broken on develop / desk * Added TODO for adding nestedset related filters --- frappe/database/query.py | 34 ++++++++++++++++++++++++++++++---- frappe/utils/commands.py | 4 ++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index ad7b3f83b4..6297e297a4 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,9 +1,11 @@ import operator import re +from functools import cached_property from typing import Any, Dict, List, Tuple, Union import frappe from frappe import _ +from frappe.boot import get_additional_filters_from_hooks from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table @@ -138,6 +140,7 @@ def change_orderby(order: str): return order[0], Order.desc +# default operators OPERATOR_MAP: Dict[str, "function"] = { "+": operator.add, "=": operator.eq, @@ -157,12 +160,35 @@ OPERATOR_MAP: Dict[str, "function"] = { "between": func_between, "is": func_is, "timespan": func_timespan, + # TODO: Add support for nested set + # TODO: Add support for custom operators (WIP) - via filters_config hooks } class Query: tables: dict = {} + @cached_property + def OPERATOR_MAP(self): + # default operators + all_operators = OPERATOR_MAP.copy() + + # update with site-specific custom operators + additional_filters_config = get_additional_filters_from_hooks() + + if additional_filters_config: + from frappe.utils.commands import warn + + warn("'filters_config' hook is not completely implemented yet in frappe.db.query engine") + + for operator, function in additional_filters_config.items(): + if callable(function): + all_operators.update({operator.casefold(): function}) + elif isinstance(function, dict): + all_operators[operator.casefold()] = frappe.get_attr(function.get("get_field"))()["operator"] + + return all_operators + def get_condition(self, table: Union[str, Table], **kwargs) -> frappe.qb: """Get initial table object @@ -243,14 +269,14 @@ class Query: if isinstance(filters, list): for f in filters: if not isinstance(f, (list, tuple)): - _operator = OPERATOR_MAP[filters[1].casefold()] + _operator = self.OPERATOR_MAP[filters[1].casefold()] if not isinstance(filters[0], str): conditions = make_function(filters[0], filters[2]) break conditions = conditions.where(_operator(Field(filters[0]), filters[2])) break else: - _operator = OPERATOR_MAP[f[-2].casefold()] + _operator = self.OPERATOR_MAP[f[-2].casefold()] if len(f) == 4: table_object = self.get_table(f[0]) _field = table_object[f[1]] @@ -279,13 +305,13 @@ class Query: for key in filters: value = filters.get(key) - _operator = OPERATOR_MAP["="] + _operator = self.OPERATOR_MAP["="] if not isinstance(key, str): conditions = conditions.where(make_function(key, value)) continue if isinstance(value, (list, tuple)): - _operator = OPERATOR_MAP[value[0].casefold()] + _operator = self.OPERATOR_MAP[value[0].casefold()] conditions = conditions.where(_operator(Field(key), value[1])) else: if value is not None: diff --git a/frappe/utils/commands.py b/frappe/utils/commands.py index a610872f03..bbc09b3034 100644 --- a/frappe/utils/commands.py +++ b/frappe/utils/commands.py @@ -59,7 +59,7 @@ def log(message, colour=""): print(colour + message + end_line) -def warn(message, category=None): +def warn(message, category=None, stacklevel=2): from warnings import warn - warn(message=message, category=category, stacklevel=2) + warn(message=message, category=category, stacklevel=stacklevel) From a20800ee2b9c2472acbb1584db95aa58769dcbba Mon Sep 17 00:00:00 2001 From: gavin Date: Thu, 19 May 2022 17:45:22 +0530 Subject: [PATCH 5/5] fix: mport module from line N shadowed by loop variable Flake8 F402 reported by sider --- frappe/database/query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index 6297e297a4..70127dd9d9 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -181,11 +181,11 @@ class Query: warn("'filters_config' hook is not completely implemented yet in frappe.db.query engine") - for operator, function in additional_filters_config.items(): + for _operator, function in additional_filters_config.items(): if callable(function): - all_operators.update({operator.casefold(): function}) + all_operators.update({_operator.casefold(): function}) elif isinstance(function, dict): - all_operators[operator.casefold()] = frappe.get_attr(function.get("get_field"))()["operator"] + all_operators[_operator.casefold()] = frappe.get_attr(function.get("get_field"))()["operator"] return all_operators