Merge branch 'develop' of github.com:frappe/frappe into mariadb-client-refactor

.github/helper/flake8.conf

@@ -69,7 +69,6 @@ ignore =
     F841,
     E713,
     E712,
-    E722,
 
 
 max-line-length = 200

.github/helper/roulette.py

@@ -5,10 +5,10 @@ import shlex
 import subprocess
 import sys
 import urllib.request
-from functools import cache
+from functools import lru_cache
 
 
-@cache
+@lru_cache(maxsize=None)
 def fetch_pr_data(pr_number, repo, endpoint):
 	api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}"
 

.github/workflows/patch-mariadb-tests.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
 
       - name: Setup Node
         uses: actions/setup-node@v3

.github/workflows/publish-assets-develop.yml

@@ -18,7 +18,7 @@ jobs:
           node-version: 14
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/publish-assets-releases.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: '12.x'
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/server-mariadb-tests.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
 
       - name: Check if build should be run
         id: check-build

.github/workflows/server-postgres-tests.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
 
       - name: Check if build should be run
         id: check-build

.github/workflows/ui-tests.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.8'
 
       - name: Check if build should be run
         id: check-build

frappe/__init__.py

@@ -22,7 +22,12 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 import click
 from werkzeug.local import Local, release_local
 
-from frappe.query_builder import get_query_builder, patch_query_aggregation, patch_query_execute
+from frappe.query_builder import (
+	get_qb_engine,
+	get_query_builder,
+	patch_query_aggregation,
+	patch_query_execute,
+)
 from frappe.utils.caching import request_cache
 from frappe.utils.data import cstr, sbool
 
@@ -241,7 +246,7 @@ def init(site, sites_path=None, new_site=False):
 	local.session = _dict()
 	local.dev_server = _dev_server
 	local.qb = get_query_builder(local.conf.db_type or "mariadb")
-
+	local.qb.engine = get_qb_engine()
 	setup_module_map()
 
 	if not _qb_patched.get(local.conf.db_type):

frappe/build.py

@@ -205,7 +205,7 @@ def symlink(target, link_name, overwrite=False):
 			os.replace(temp_link_name, link_name)
 		except AttributeError:
 			os.renames(temp_link_name, link_name)
-	except:
+	except Exception:
 		if os.path.islink(temp_link_name):
 			os.remove(temp_link_name)
 		raise

frappe/commands/scheduler.py

@@ -5,6 +5,7 @@ import click
 import frappe
 from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
+from frappe.utils import cint
 
 
 @click.command("trigger-scheduler-event", help="Trigger a scheduler event")

frappe/core/doctype/doctype/test_doctype.py

@@ -318,7 +318,7 @@ class TestDocType(unittest.TestCase):
 			self.assertListEqual(
 				test_doctype_json["field_order"], ["field_4", "field_5", "field_1", "field_2"]
 			)
-		except:
+		except Exception:
 			raise
 		finally:
 			frappe.flags.allow_doctype_export = 0

frappe/core/doctype/user/user.py

@@ -586,7 +586,7 @@ class User(Document):
 			for p in self.social_logins:
 				if p.provider == provider:
 					return p.userid
-		except:
+		except Exception:
 			return None
 
 	def set_social_login_userid(self, provider, userid, username=None):

frappe/database/database.py

@@ -11,13 +11,12 @@ from contextlib import contextmanager
 from time import time
 from typing import Dict, List, Optional, Tuple, Union
 
-from pypika.terms import Criterion, NullValue, PseudoColumn
+from pypika.terms import Criterion, NullValue
 
 import frappe
 import frappe.defaults
 import frappe.model.meta
 from frappe import _
-from frappe.database.query import Query as FilterEngine
 from frappe.database.utils import LazyMogrify, Query, QueryValues, is_query_type
 from frappe.exceptions import DoesNotExistError
 from frappe.model.utils.link_count import flush_local_link_count
@@ -84,12 +83,6 @@ class Database(object):
 		# self.db_type: str
 		# self.last_query (lazy) attribute of last sql query executed
 
-	@property
-	def query(self):
-		if not hasattr(self, "_filter_engine"):
-			self._filter_engine = FilterEngine()
-		return self._filter_engine
-
 	def setup_type_map(self):
 		pass
 
@@ -279,12 +272,12 @@ class Database(object):
 
 		try:
 			return self._cursor.mogrify(query, values)
-		except BaseException:  # noqa: E722
+		except BaseException:
 			if isinstance(values, dict):
 				return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()}
 			elif isinstance(values, (list, tuple)):
 				return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values)
-			return (query, values)
+			return query, values
 
 	def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify:
 		"""Wrap the object with str to generate mogrified query."""
@@ -612,7 +605,7 @@ class Database(object):
 				return [map(values.get, fields)]
 
 		else:
-			r = self.query.get_sql(
+			r = frappe.qb.engine.get_query(
 				"Singles",
 				filters={"field": ("in", tuple(fields)), "doctype": doctype},
 				fields=["field", "value"],
@@ -645,7 +638,7 @@ class Database(object):
 		        # Get coulmn and value of the single doctype Accounts Settings
 		        account_settings = frappe.db.get_singles_dict("Accounts Settings")
 		"""
-		queried_result = self.query.get_sql(
+		queried_result = frappe.qb.engine.get_query(
 			"Singles",
 			filters={"doctype": doctype},
 			fields=["field", "value"],
@@ -718,7 +711,7 @@ class Database(object):
 		if cache and fieldname in self.value_cache[doctype]:
 			return self.value_cache[doctype][fieldname]
 
-		val = self.query.get_sql(
+		val = frappe.qb.engine.get_query(
 			table="Singles",
 			filters={"doctype": doctype, "field": fieldname},
 			fields="value",
@@ -760,14 +753,7 @@ class Database(object):
 	):
 		field_objects = []
 
-		if not isinstance(fields, Criterion):
-			for field in fields:
-				if "(" in str(field) or " as " in str(field):
-					field_objects.append(PseudoColumn(field))
-				else:
-					field_objects.append(field)
-
-		query = self.query.get_sql(
+		query = frappe.qb.engine.get_query(
 			table=doctype,
 			filters=filters,
 			orderby=order_by,
@@ -877,7 +863,7 @@ class Database(object):
 					frappe.clear_document_cache(dt, docname)
 
 			else:
-				query = self.query.build_conditions(table=dt, filters=dn, update=True)
+				query = frappe.qb.engine.build_conditions(table=dt, filters=dn, update=True)
 				# TODO: Fix this; doesn't work rn - gavin@frappe.io
 				# frappe.cache().hdel_keys(dt, "document_cache")
 				# Workaround: clear all document caches
@@ -1062,7 +1048,9 @@ class Database(object):
 			cache_count = frappe.cache().get_value("doctype:count:{}".format(dt))
 			if cache_count is not None:
 				return cache_count
-		query = self.query.get_sql(table=dt, filters=filters, fields=Count("*"), distinct=distinct)
+		query = frappe.qb.engine.get_query(
+			table=dt, filters=filters, fields=Count("*"), distinct=distinct
+		)
 		count = self.sql(query, debug=debug)[0][0]
 		if not filters and cache:
 			frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400)
@@ -1207,7 +1195,7 @@ class Database(object):
 		Doctype name can be passed directly, it will be pre-pended with `tab`.
 		"""
 		filters = filters or kwargs.get("conditions")
-		query = self.query.build_conditions(table=doctype, filters=filters).delete()
+		query = frappe.qb.engine.build_conditions(table=doctype, filters=filters).delete()
 		if "debug" not in kwargs:
 			kwargs["debug"] = debug
 		return query.run(**kwargs)

frappe/database/query.py

@@ -1,15 +1,22 @@
 import operator
 import re
+from ast import literal_eval
 from functools import cached_property
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Union
 
 import frappe
 from frappe import _
 from frappe.model.db_query import get_timespan_date_range
-from frappe.query_builder import Criterion, Field, Order, Table
+from frappe.query_builder import Criterion, Field, Order, Table, functions
+from frappe.query_builder.functions import SqlFunctions
 
 TAB_PATTERN = re.compile("^tab")
 WORDS_PATTERN = re.compile(r"\w+")
+BRACKETS_PATTERN = re.compile(r"\(.*?\)|$")
+SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions]
+
+if TYPE_CHECKING:
+	from pypika.functions import Function
 
 
 def like(key: Field, value: str) -> frappe.qb:
@@ -92,7 +99,7 @@ def func_between(key: Field, value: Union[List, Tuple]) -> frappe.qb:
 
 def func_is(key, value):
 	"Wrapper for IS"
-	return Field(key).isnotnull() if value.lower() == "set" else Field(key).isnull()
+	return key.isnotnull() if value.lower() == "set" else key.isnull()
 
 
 def func_timespan(key: Field, value: str) -> frappe.qb:
@@ -142,6 +149,13 @@ def change_orderby(order: str):
 	return order[0], Order.desc
 
 
+def literal_eval_(literal):
+	try:
+		return literal_eval(literal)
+	except (ValueError, SyntaxError):
+		return literal
+
+
 # default operators
 OPERATOR_MAP: Dict[str, Callable] = {
 	"+": operator.add,
@@ -167,7 +181,7 @@ OPERATOR_MAP: Dict[str, Callable] = {
 }
 
 
-class Query:
+class Engine:
 	tables: dict = {}
 
 	@cached_property
@@ -232,7 +246,7 @@ class Query:
 		Returns:
 		        conditions (frappe.qb): frappe.qb object
 		"""
-		if kwargs.get("orderby"):
+		if kwargs.get("orderby") and kwargs.get("orderby") != "KEEP_DEFAULT_ORDERING":
 			orderby = kwargs.get("orderby")
 			if isinstance(orderby, str) and len(orderby.split()) > 1:
 				for ordby in orderby.split(","):
@@ -244,6 +258,7 @@ class Query:
 
 		if kwargs.get("limit"):
 			conditions = conditions.limit(kwargs.get("limit"))
+			conditions = conditions.offset(kwargs.get("offset", 0))
 
 		if kwargs.get("distinct"):
 			conditions = conditions.distinct()
@@ -251,6 +266,9 @@ class Query:
 		if kwargs.get("for_update"):
 			conditions = conditions.for_update()
 
+		if kwargs.get("groupby"):
+			conditions = conditions.groupby(kwargs.get("groupby"))
+
 		return conditions
 
 	def misc_query(self, table: str, filters: Union[List, Tuple] = None, **kwargs):
@@ -302,6 +320,10 @@ class Query:
 			conditions = self.add_conditions(conditions, **kwargs)
 			return conditions
 
+		for key, value in filters.items():
+			if isinstance(value, bool):
+				filters.update({key: str(int(value))})
+
 		for key in filters:
 			value = filters.get(key)
 			_operator = self.OPERATOR_MAP["="]
@@ -311,7 +333,8 @@ class Query:
 				continue
 			if isinstance(value, (list, tuple)):
 				_operator = self.OPERATOR_MAP[value[0].casefold()]
-				conditions = conditions.where(_operator(Field(key), value[1]))
+				_value = value[1] if value[1] else ("",)
+				conditions = conditions.where(_operator(Field(key), _value))
 			else:
 				if value is not None:
 					conditions = conditions.where(_operator(Field(key), value))
@@ -348,7 +371,117 @@ class Query:
 
 		return criterion
 
-	def get_sql(
+	def get_function_object(self, field: str) -> "Function":
+		"""Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object"""
+		func = field.split("(", maxsplit=1)[0].capitalize()
+		args_start, args_end = len(func) + 1, field.index(")")
+		args = field[args_start:args_end].split(",")
+
+		to_cast = "*" not in args
+		_args = []
+
+		for arg in args:
+			field = literal_eval_(arg.strip())
+			if to_cast:
+				field = Field(field)
+			_args.append(field)
+
+		return getattr(functions, func)(*_args)
+
+	def function_objects_from_string(self, fields):
+		functions = ""
+		for func in SQL_FUNCTIONS:
+			if f"{func}(" in fields:
+				functions = str(func) + str(BRACKETS_PATTERN.search(fields).group())
+				return [self.get_function_object(functions)]
+		if not functions:
+			return []
+
+	def function_objects_from_list(self, fields):
+		functions = []
+		for field in fields:
+			field = field.casefold() if isinstance(field, str) else field
+			if not issubclass(type(field), Criterion):
+				if any([func in field and f"{func}(" in field for func in SQL_FUNCTIONS]):
+					functions.append(field)
+		return [self.get_function_object(function) for function in functions]
+
+	def remove_string_functions(self, fields, function_objects):
+		"""Remove string functions from fields which have already been converted to function objects"""
+		for function in function_objects:
+			if isinstance(fields, str):
+				fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), ""))
+			else:
+				updated_fields = []
+				for field in fields:
+					if isinstance(field, str):
+						updated_fields.append(
+							BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "")
+						)
+					else:
+						updated_fields.append(field)
+
+					fields = [field for field in updated_fields if field]
+
+		return fields
+
+	def set_fields(self, fields, **kwargs):
+		fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name"
+		if isinstance(fields, list) and None in fields and Field not in fields:
+			return None
+
+		function_objects = []
+
+		is_list = isinstance(fields, (list, tuple, set))
+		if is_list and len(fields) == 1:
+			fields = fields[0]
+			is_list = False
+
+		if is_list:
+			function_objects += self.function_objects_from_list(fields=fields)
+
+		is_str = isinstance(fields, str)
+		if is_str:
+			fields = fields.casefold()
+			function_objects += self.function_objects_from_string(fields=fields)
+
+		fields = self.remove_string_functions(fields, function_objects)
+
+		if is_str and "," in fields:
+			fields = [field.replace(" ", "") if "as" not in field else field for field in fields.split(",")]
+			is_list, is_str = True, False
+
+		if is_str:
+			if fields == "*":
+				return fields
+			if " as " in fields:
+				fields, reference = fields.split(" as ")
+				fields = Field(fields).as_(reference)
+
+		if not is_str and fields:
+			if issubclass(type(fields), Criterion):
+				return fields
+			updated_fields = []
+			if "*" in fields:
+				return fields
+			for field in fields:
+				if not isinstance(field, Criterion) and field:
+					if " as " in field:
+						field, reference = field.split(" as ")
+						updated_fields.append(Field(field.strip()).as_(reference))
+					else:
+						updated_fields.append(Field(field))
+
+					fields = updated_fields
+
+		# Need to check instance again since fields modified.
+		if not isinstance(fields, (list, tuple, set)):
+			fields = [fields] if fields else []
+
+		fields.extend(function_objects)
+		return fields
+
+	def get_query(
 		self,
 		table: str,
 		fields: Union[List, Tuple],
@@ -358,15 +491,20 @@ class Query:
 		# Clean up state before each query
 		self.tables = {}
 		criterion = self.build_conditions(table, filters, **kwargs)
+		fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs)
+
+		join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join"
 
 		if len(self.tables) > 1:
 			primary_table = self.tables[table]
 			del self.tables[table]
 			for table_object in self.tables.values():
-				criterion = criterion.left_join(table_object).on(table_object.parent == primary_table.name)
+				criterion = getattr(criterion, join)(table_object).on(
+					table_object.parent == primary_table.name
+				)
 
 		if isinstance(fields, (list, tuple)):
-			query = criterion.select(*kwargs.get("field_objects", fields))
+			query = criterion.select(*fields)
 
 		elif isinstance(fields, Criterion):
 			query = criterion.select(fields)

frappe/desk/doctype/number_card/number_card.py

@@ -204,7 +204,7 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters):
 	if txt:
 		search_conditions = [numberCard[field].like("%{txt}%".format(txt=txt)) for field in searchfields]
 
-	condition_query = frappe.db.query.build_conditions(doctype, filters)
+	condition_query = frappe.qb.engine.build_conditions(doctype, filters)
 
 	return (
 		condition_query.select(numberCard.name, numberCard.label, numberCard.document_type)

frappe/desk/doctype/system_console/system_console.py

@@ -19,7 +19,7 @@ class SystemConsole(Document):
 				self.output = "\n".join(frappe.debug_log)
 			elif self.type == "SQL":
 				self.output = frappe.as_json(read_sql(self.console, as_dict=1))
-		except:  # noqa: E722
+		except Exception:
 			self.output = frappe.get_traceback()
 
 		if self.commit:

frappe/desk/listview.py

@@ -37,7 +37,7 @@ def get_group_by_count(doctype: str, current_filters: str, field: str) -> List[D
 		ToDo = DocType("ToDo")
 		User = DocType("User")
 		count = Count("*").as_("count")
-		filtered_records = frappe.db.query.build_conditions(doctype, current_filters).select("name")
+		filtered_records = frappe.qb.engine.build_conditions(doctype, current_filters).select("name")
 
 		return (
 			frappe.qb.from_(ToDo)

frappe/desk/reportview.py

@@ -156,6 +156,8 @@ def setup_group_by(data):
 					**data
 				)
 			)
+			if data.aggregate_on_field:
+				data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`")
 		else:
 			raise_invalid_field(data.aggregate_on_field)
 
@@ -433,20 +435,11 @@ def append_totals_row(data):
 def get_labels(fields, doctype):
 	"""get column labels based on column names"""
 	labels = []
-	doctype = doctype.lower()
 	for key in fields:
-		aggregate_function = ""
-
-		key = key.casefold().split(" as ", maxsplit=1)[0]
+		key = key.split(" as ")[0]
 
 		if key.startswith(("count(", "sum(", "avg(")):
-			# Get aggregate function and _aggregate_column
-			# key = 'sum(`tabDocType`.`fieldname`)'
-			if not key.rstrip().endswith(")"):
-				continue
-			_agg_fn, _key = key.split("(", maxsplit=1)
-			aggregate_function = _agg_fn.lower()  # aggregate_function = 'sum'
-			key = _key[:-1]  # key = `tabDocType`.`fieldname`
+			continue
 
 		if "." in key:
 			parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
@@ -462,10 +455,7 @@ def get_labels(fields, doctype):
 			if parenttype != doctype:
 				# If the column is from a child table, append the child doctype.
 				# For example, "Item Code (Sales Invoice Item)".
-				label += f" ({ _(parenttype.title()) })"
-
-		if aggregate_function:
-			label = _("{0} of {1}").format(aggregate_function.capitalize(), label)
+				label += f" ({ _(parenttype) })"
 
 		labels.append(label)
 
@@ -474,7 +464,7 @@ def get_labels(fields, doctype):
 
 def handle_duration_fieldtype_values(doctype, data, fields):
 	for field in fields:
-		key = field.casefold().split(" as ", maxsplit=1)[0]
+		key = field.split(" as ")[0]
 
 		if key.startswith(("count(", "sum(", "avg(")):
 			continue

frappe/email/doctype/newsletter/newsletter.py

@@ -66,7 +66,7 @@ class Newsletter(WebsiteGenerator):
 				response = requests.head(url, verify=False, timeout=5)
 				if response.status_code >= 400:
 					broken_links.append(url)
-			except:
+			except Exception:
 				broken_links.append(url)
 		return broken_links
 

frappe/email/doctype/notification/notification.py

@@ -140,7 +140,7 @@ def get_context(context):
 			if self.channel == "System Notification" or self.send_system_notification:
 				self.create_system_notification(doc, context)
 
-		except:
+		except Exception:
 			self.log_error("Failed to send Notification")
 
 		if self.set_property_after_alert:

frappe/email/queue.py

@@ -69,7 +69,7 @@ def get_emails_sent_today(email_account=None):
 
 def get_unsubscribe_message(
 	unsubscribe_message: str, expose_recipients: str
-) -> frappe._dict[str, str]:
+) -> "frappe._dict[str, str]":
 	unsubscribe_message = unsubscribe_message or _("Unsubscribe")
 	unsubscribe_link = f'<a href="<!--unsubscribe_url-->" target="_blank">{unsubscribe_message}</a>'
 	unsubscribe_html = _("{0} to stop receiving emails of this type").format(unsubscribe_link)

frappe/email/receive.py

@@ -377,7 +377,7 @@ class EmailServer:
 			try:
 				# retrieve headers
 				incoming_mail = Email(b"\n".join(self.pop.top(msg_num, 5)[1]))
-			except:
+			except Exception:
 				pass
 
 		if incoming_mail:
@@ -437,7 +437,7 @@ class Email:
 				utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
 				utc_dt = datetime.datetime.utcfromtimestamp(utc)
 				self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S")
-			except:
+			except Exception:
 				self.date = now()
 		else:
 			self.date = now()
@@ -572,7 +572,7 @@ class Email:
 				try:
 					fname = fname.replace("\n", " ").replace("\r", "")
 					fname = cstr(decode_header(fname)[0][0])
-				except:
+				except Exception:
 					fname = get_random_filename(content_type=content_type)
 			else:
 				fname = get_random_filename(content_type=content_type)

frappe/installer.py

@@ -707,7 +707,7 @@ def extract_files(site_name, file_path):
 			subprocess.check_output(["tar", "xvf", tar_path, "--strip", "2"], cwd=abs_site_path)
 		elif file_path.endswith(".tgz"):
 			subprocess.check_output(["tar", "zxvf", tar_path, "--strip", "2"], cwd=abs_site_path)
-	except:
+	except Exception:
 		raise
 	finally:
 		frappe.destroy()

frappe/integrations/doctype/razorpay_settings/razorpay_settings.py

@@ -141,8 +141,8 @@ class RazorpaySettings(Document):
 				)
 				if not resp.get("id"):
 					frappe.log_error(message=str(resp), title="Razorpay Failed while creating subscription")
-		except:
-			frappe.log_error(frappe.get_traceback())
+		except Exception:
+			frappe.log_error()
 			# failed
 			pass
 
@@ -181,10 +181,8 @@ class RazorpaySettings(Document):
 			else:
 				frappe.log_error(message=str(resp), title="Razorpay Failed while creating subscription")
 
-		except:
-			frappe.log_error(frappe.get_traceback())
-			# failed
-			pass
+		except Exception:
+			frappe.log_error()
 
 	def prepare_subscription_details(self, settings, **kwargs):
 		if not kwargs.get("subscription_id"):
@@ -283,10 +281,8 @@ class RazorpaySettings(Document):
 			else:
 				frappe.log_error(message=str(resp), title="Razorpay Payment not authorized")
 
-		except:
-			frappe.log_error(frappe.get_traceback())
-			# failed
-			pass
+		except Exception:
+			frappe.log_error()
 
 		status = frappe.flags.integration_request.status_code
 

frappe/model/db_query.py

@@ -417,8 +417,6 @@ class DatabaseQuery(object):
 			"extract(",
 			"locate(",
 			"strpos(",
-		]
-		aggregate_functions = [
 			"count(",
 			"sum(",
 			"avg(",
@@ -429,9 +427,6 @@ class DatabaseQuery(object):
 				if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field):
 					continue
 
-				if any(x for x in aggregate_functions if x in field):
-					field = field.split("(", 1)[1][:-1]
-
 				table_name = field.split(".")[0]
 
 				if table_name.lower().startswith("group_concat("):

frappe/model/mapper.py

@@ -243,7 +243,7 @@ def map_fetch_fields(target_doc, df, no_copy_fields):
 			if not linked_doc:
 				try:
 					linked_doc = frappe.get_doc(df.options, target_doc.get(df.fieldname))
-				except:
+				except Exception:
 					return
 
 			val = linked_doc.get(source_fieldname)

frappe/patches/v13_0/generate_theme_files_in_public_folder.py

@@ -14,6 +14,6 @@ def execute():
 		try:
 			doc.generate_bootstrap_theme()
 			doc.save()
-		except:  # noqa: E722
+		except Exception:
 			print("Ignoring....")
 			print(frappe.get_traceback())

frappe/printing/page/print_format_builder/print_format_builder_column_selector.html

@@ -20,7 +20,7 @@
 						<input type="checkbox"
 							data-fieldname="{{ f.fieldname }}"
 							{{ selected ? "checked" : "" }}>
-							{{ f.label }}
+							{{ __(f.label) }}
 					</label>
 				</div>
 			</div>

frappe/public/js/frappe/ui/sort_selector.html

@@ -2,7 +2,7 @@
 	<div class="btn-group">
 		<button class="btn btn-default btn-sm btn-order"
 			data-value="{{ sort_order }}"
-			title="{{ sort_order==="desc" ? "descending" : "ascending" }}">
+			title="{{ sort_order==="desc" ? __("descending") : __("ascending") }}">
 			<span class="sort-order">
 				<svg class="icon icon-sm">
 					<use href="#icon-{{ sort_order==="desc" ? "sort-descending" : "sort-ascending" }}"></use>

frappe/public/js/frappe/ui/sort_selector.js

@@ -25,7 +25,7 @@ frappe.ui.SortSelector = class SortSelector {
 		this.wrapper.find('.btn-order').on('click', function() {
 			let btn = $(this);
 			const order = $(this).attr('data-value') === 'desc' ? 'asc' : 'desc';
-			const title = $(this).attr('data-value' )=== 'desc' ? 'ascending' : 'descending';
+			const title = $(this).attr('data-value' )=== 'desc' ? __('ascending') : __('descending');
 
 			btn.attr('data-value', order);
 			btn.attr('title', title);

frappe/public/js/frappe/ui/toolbar/toolbar.js

@@ -97,7 +97,7 @@ frappe.ui.toolbar.Toolbar = class {
 				$("<a>", {
 					href: url,
 					class: "dropdown-item",
-					text: link.label,
+					text: __(link.label),
 					target: "_blank"
 				}).appendTo($help_links);
 			}

frappe/public/js/frappe/utils/dashboard_utils.js

@@ -16,7 +16,7 @@ frappe.dashboard_utils = {
 					<a data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
 						<button class="btn btn-secondary btn-xs">
 			 				${icon_html}
-							<span class="filter-label">${filter.label}</span>
+							<span class="filter-label">${__(filter.label)}</span>
 							${frappe.utils.icon('select', 'xs')}
 						</button>
 				</a>`;
@@ -24,10 +24,12 @@ frappe.dashboard_utils = {
 
 			if (filter.fieldnames) {
 				options_html = filter.options.map((option, i) =>
+					// TODO: Make option translatable - be careful, since the text of the a tag is later used to perform some action
 					`<li>
 						<a class="dropdown-item" data-fieldname="${filter.fieldnames[i]}">${option}</a>
 					</li>`).join('');
 			} else {
+				// TODO: Make option translatable - be careful, since the text of the a tag is later used to perform some action
 				options_html = filter.options.map( option => `<li><a class="dropdown-item">${option}</a></li>`).join('');
 			}
 

frappe/public/js/frappe/views/kanban/kanban_column.html

@@ -17,7 +17,7 @@
 	</div>
 	<div class="add-card">
 		<div class="ellipsis">
-			+ {{ __("Add " + doctype) }}
+			+ {{ __("Add {0}", [__(doctype)]) }}
 		</div>
 	</div>
 	<div class="kanban-card new-card-area">

frappe/query_builder/__init__.py

@@ -7,6 +7,7 @@ from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValue
 from frappe.query_builder.utils import (
 	Column,
 	DocType,
+	get_qb_engine,
 	get_query_builder,
 	patch_query_aggregation,
 	patch_query_execute,

frappe/query_builder/builder.py

@@ -1,3 +1,5 @@
+import typing
+
 from pypika import MySQLQuery, Order, PostgreSQLQuery, terms
 from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
 from pypika.queries import QueryBuilder, Schema, Table
@@ -13,6 +15,13 @@ class Base:
 	Schema = Schema
 	Table = Table
 
+	# Added dynamic type hints for engine attribute
+	# which is to be assigned later.
+	if typing.TYPE_CHECKING:
+		from frappe.database.query import Engine
+
+		engine: Engine
+
 	@staticmethod
 	def functions(name: str, *args, **kwargs) -> Function:
 		return Function(name, *args, **kwargs)

frappe/query_builder/functions.py

@@ -1,8 +1,9 @@
+from enum import Enum
+
 from pypika.functions import *
 from pypika.terms import Arithmetic, ArithmeticExpression, CustomFunction, Function
 
 import frappe
-from frappe.database.query import Query
 from frappe.query_builder.custom import GROUP_CONCAT, MATCH, STRING_AGG, TO_TSVECTOR
 from frappe.query_builder.utils import ImportMapper, db_type_is
 
@@ -14,6 +15,11 @@ class Concat_ws(Function):
 		super(Concat_ws, self).__init__("CONCAT_WS", *terms, **kwargs)
 
 
+class Locate(Function):
+	def __init__(self, *terms, **kwargs):
+		super(Locate, self).__init__("LOCATE", *terms, **kwargs)
+
+
 GroupConcat = ImportMapper({db_type_is.MARIADB: GROUP_CONCAT, db_type_is.POSTGRES: STRING_AGG})
 
 Match = ImportMapper({db_type_is.MARIADB: MATCH, db_type_is.POSTGRES: TO_TSVECTOR})
@@ -73,14 +79,24 @@ class Cast_(Function):
 
 def _aggregate(function, dt, fieldname, filters, **kwargs):
 	return (
-		Query()
-		.build_conditions(dt, filters)
+		frappe.qb.engine.build_conditions(dt, filters)
 		.select(function(PseudoColumn(fieldname)))
 		.run(**kwargs)[0][0]
 		or 0
 	)
 
 
+class SqlFunctions(Enum):
+	DayOfYear = "dayofyear"
+	Extract = "extract"
+	Locate = "locate"
+	Count = "count"
+	Sum = "sum"
+	Avg = "avg"
+	Max = "max"
+	Min = "min"
+
+
 def _max(dt, fieldname, filters=None, **kwargs):
 	return _aggregate(Max, dt, fieldname, filters, **kwargs)
 

frappe/query_builder/utils.py

@@ -45,6 +45,12 @@ def get_query_builder(type_of_db: str) -> Union[Postgres, MariaDB]:
 	return picks[db]
 
 
+def get_qb_engine():
+	from frappe.database.query import Engine
+
+	return Engine()
+
+
 def get_attr(method_string):
 	modulename = ".".join(method_string.split(".")[:-1])
 	methodname = method_string.split(".")[-1]

frappe/tests/test_db_query.py

@@ -143,7 +143,7 @@ class TestReportview(unittest.TestCase):
 			)
 
 	def test_none_filter(self):
-		query = frappe.db.query.get_sql("DocType", fields="name", filters={"restrict_to_domain": None})
+		query = frappe.qb.engine.get_query("DocType", fields="name", filters={"restrict_to_domain": None})
 		sql = str(query).replace("`", "").replace('"', "")
 		condition = "restrict_to_domain IS NULL"
 		self.assertIn(condition, sql)
@@ -643,7 +643,9 @@ class TestReportview(unittest.TestCase):
 		)
 
 		response = execute_cmd("frappe.desk.reportview.get")
-		self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column"])
+		self.assertListEqual(
+			response["keys"], ["field_label", "field_name", "_aggregate_column", "columns"]
+		)
 
 	def test_cast_name(self):
 		from frappe.core.doctype.doctype.test_doctype import new_doctype

frappe/tests/test_query.py

@@ -1,14 +1,15 @@
 import unittest
 
 import frappe
+from frappe.query_builder import Field
 from frappe.tests.test_query_builder import db_type_is, run_only_if
 
 
-@run_only_if(db_type_is.MARIADB)
 class TestQuery(unittest.TestCase):
+	@run_only_if(db_type_is.MARIADB)
 	def test_multiple_tables_in_filters(self):
 		self.assertEqual(
-			frappe.db.query.get_sql(
+			frappe.qb.engine.get_query(
 				"DocType",
 				["*"],
 				[
@@ -18,3 +19,56 @@ class TestQuery(unittest.TestCase):
 			).get_sql(),
 			"SELECT * FROM `tabDocType` LEFT JOIN `tabBOM Update Log` ON `tabBOM Update Log`.`parent`=`tabDocType`.`name` WHERE `tabBOM Update Log`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'",
 		)
+
+	def test_string_fields(self):
+		self.assertEqual(
+			frappe.qb.engine.get_query(
+				"User", fields="name, email", filters={"name": "Administrator"}
+			).get_sql(),
+			frappe.qb.from_("User")
+			.select(Field("name"), Field("email"))
+			.where(Field("name") == "Administrator")
+			.get_sql(),
+		)
+
+		self.assertEqual(
+			frappe.qb.engine.get_query(
+				"User", fields=["name, email"], filters={"name": "Administrator"}
+			).get_sql(),
+			frappe.qb.from_("User")
+			.select(Field("name"), Field("email"))
+			.where(Field("name") == "Administrator")
+			.get_sql(),
+		)
+
+	def test_functions_fields(self):
+		from frappe.query_builder.functions import Count, Max
+
+		self.assertEqual(
+			frappe.qb.engine.get_query("User", fields="Count(name)", filters={}).get_sql(),
+			frappe.qb.from_("User").select(Count(Field("name"))).get_sql(),
+		)
+
+		self.assertEqual(
+			frappe.qb.engine.get_query("User", fields=["Count(name)", "Max(name)"], filters={}).get_sql(),
+			frappe.qb.from_("User").select(Count(Field("name")), Max(Field("name"))).get_sql(),
+		)
+
+		self.assertEqual(
+			frappe.qb.engine.get_query("User", fields=[Count("*")], filters={}).get_sql(),
+			frappe.qb.from_("User").select(Count("*")).get_sql(),
+		)
+
+	def test_qb_fields(self):
+		user_doctype = frappe.qb.DocType("User")
+		self.assertEqual(
+			frappe.qb.engine.get_query(
+				user_doctype, fields=[user_doctype.name, user_doctype.email], filters={}
+			).get_sql(),
+			frappe.qb.from_(user_doctype).select(user_doctype.name, user_doctype.email).get_sql(),
+		)
+
+		self.assertEqual(
+			frappe.qb.engine.get_query(user_doctype, fields=user_doctype.email, filters={}).get_sql(),
+			frappe.qb.from_(user_doctype).select(user_doctype.email).get_sql(),
+		)

frappe/twofactor.py

@@ -301,7 +301,7 @@ def send_token_via_sms(otpsecret, token=None, phone_no=None):
 	"""Send token as sms to user."""
 	try:
 		from frappe.core.doctype.sms_settings.sms_settings import send_request
-	except:
+	except Exception:
 		return False
 
 	if not phone_no:

frappe/utils/background_jobs.py

@@ -166,7 +166,7 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True,
 			frappe.log_error(title=method_name)
 			raise
 
-	except:
+	except Exception:
 		frappe.db.rollback()
 		frappe.log_error(title=method_name)
 		frappe.db.commit()

frappe/utils/boilerplate.py

@@ -478,7 +478,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: 3.8
 
       - name: Setup Node
         uses: actions/setup-node@v2

frappe/utils/data.py

@@ -966,7 +966,7 @@ def floor(s):
 	"""
 	try:
 		num = cint(math.floor(flt(s)))
-	except:
+	except Exception:
 		num = 0
 	return num
 
@@ -988,7 +988,7 @@ def ceil(s):
 	"""
 	try:
 		num = cint(math.ceil(flt(s)))
-	except:
+	except Exception:
 		num = 0
 	return num
 

frappe/utils/goal.py

@@ -25,7 +25,7 @@ def get_monthly_results(
 	date_format = "%m-%Y" if frappe.db.db_type != "postgres" else "MM-YYYY"
 
 	return dict(
-		frappe.db.query.build_conditions(table=goal_doctype, filters=filters)
+		frappe.qb.engine.build_conditions(table=goal_doctype, filters=filters)
 		.select(
 			DateFormat(Table[date_col], date_format).as_("month_year"),
 			Function(aggregation, goal_field),

frappe/utils/pdf.py

@@ -173,7 +173,7 @@ def read_options_from_html(html):
 			match = pattern.findall(html)
 			if match:
 				options[attr] = str(match[-1][3]).strip()
-		except:
+		except Exception:
 			pass
 
 	return str(soup), options

frappe/utils/scheduler.py

@@ -84,7 +84,7 @@ def enqueue_events_for_site(site):
 			frappe.logger("scheduler").debug("Access denied for site {0}".format(site))
 		else:
 			log_and_raise()
-	except:
+	except Exception:
 		log_and_raise()
 
 	finally: