ci: this adds universal runtime typechecking during tests to test runners (#28554)

* ci: this adds universal runtime typechecking during tests to test runners

* ci: add configuration options for test-time type checking

frappe/parallel_test_runner.py

@@ -14,6 +14,7 @@ import requests
 import frappe
 from frappe.tests.utils import make_test_records
 
+from .testing.environment import _decorate_all_methods_and_functions_with_type_checker
 from .testing.result import TestResult
 
 click_ctx = click.get_current_context(True)
@@ -49,6 +50,7 @@ class ParallelTestRunner:
 		frappe.flags.in_test = True
 		frappe.clear_cache()
 		frappe.utils.scheduler.disable_scheduler()
+		_decorate_all_methods_and_functions_with_type_checker()
 		self.before_test_setup()
 
 	def before_test_setup(self):

frappe/testing/environment.py

@@ -19,9 +19,13 @@ These functions and classes are typically used by the test runner to set up
 and tear down the test environment before and after test execution.
 """
 
+import functools
+import inspect
 import logging
 import unittest
 
+import tomllib
+
 import frappe
 import frappe.utils.scheduler
 from frappe.tests.utils import make_test_records
@@ -52,6 +56,8 @@ def _initialize_test_environment(site, config):
 	frappe.flags.print_messages = logger.getEffectiveLevel() < logging.INFO
 	frappe.flags.tests_verbose = logger.getEffectiveLevel() < logging.INFO
 
+	_decorate_all_methods_and_functions_with_type_checker()
+
 
 def _cleanup_after_tests():
 	"""Perform cleanup operations after running tests"""
@@ -77,6 +83,86 @@ def _disable_scheduler_if_needed():
 		frappe.utils.scheduler.disable_scheduler()
 
 
+@debug_timer
+def _decorate_all_methods_and_functions_with_type_checker():
+	from frappe.utils.typing_validations import validate_argument_types
+
+	def _get_config_from_pyproject(app_path):
+		try:
+			with open(f"{app_path}/pyproject.toml", "rb") as f:
+				config = tomllib.load(f)
+				return (
+					config.get("tool", {})
+					.get("frappe", {})
+					.get("testing", {})
+					.get("function_type_validation", {})
+				)
+		except FileNotFoundError:
+			return {}
+		except tomllib.TOMLDecodeError:
+			logger.warning(f"Failed to parse pyproject.toml for app {app_path}")
+			return {}
+
+	def _decorate_callable(obj, apps, parent_module):
+		# whitelisted methods are already checked, see frappe.whitelist
+		if getattr(obj, "__func__", obj) in frappe.whitelisted:
+			return obj
+		# Check if the function is already decorated
+		elif hasattr(obj, "_is_decorated_for_validate_argument_types"):
+			return obj
+		elif module := getattr(obj, "__module__", ""):
+			if (app := module.split(".", 1)[0]) and app not in apps:
+				return obj
+			config = _get_config_from_pyproject(frappe.get_app_source_path(app))
+			skip_namespaces = config.get("skip_namespaces", [])
+			if any(module.startswith(n) for n in skip_namespaces):
+				return obj
+
+		@functools.wraps(obj)
+		def wrapper(*args, **kwargs):
+			try:
+				return validate_argument_types(obj)(*args, **kwargs)
+			except TypeError as e:
+				# breakpoint()
+				raise e
+
+		wrapper._is_decorated_for_validate_argument_types = True
+
+		logger.debug(f"... patching {obj.__module__}.{obj.__name__} in {parent_module.__name__}")
+
+		return wrapper
+
+	def _decorate_module(module, apps, current_depth, max_depth):
+		if current_depth >= max_depth:
+			return
+		if (app := module.__name__.split(".", 1)[0]) and app not in apps:
+			return
+		for name in dir(module):
+			obj = getattr(module, name)
+			if inspect.isfunction(obj):
+				if not hasattr(obj, "__annotations__"):
+					continue
+				setattr(module, name, _decorate_callable(obj, apps, module))
+			elif inspect.ismodule(obj):
+				if hasattr(obj, "_is_decorated_for_validate_argument_types"):
+					continue
+				obj._is_decorated_for_validate_argument_types = True
+				_decorate_module(obj, apps, current_depth + 1, max_depth)
+
+	for app in (apps := frappe.get_installed_apps()):
+		config = _get_config_from_pyproject(frappe.get_app_source_path(app))
+		max_depth = config.get("max_module_depth", float("inf"))
+		logger.info(
+			f"Decorating callables with type validator up to module depth {max_depth+1} in {app!r} ..."
+		)
+		for module_name in frappe.local.app_modules.get(app) or []:
+			try:
+				module = frappe.get_module(f"{app}.{module_name}")
+				_decorate_module(module, apps, 0, max_depth)
+			except ImportError:
+				logger.error(f"Error importing module {app}.{module_name}")
+
+
 class IntegrationTestPreparation:
 	def __init__(self, cfg):
 		self.cfg = cfg

frappe/utils/typing_validations.py

@@ -69,9 +69,18 @@ def raise_type_error(
 
 @lru_cache(maxsize=2048)
 def TypeAdapter(type_):
+	from pydantic import PydanticUserError
 	from pydantic import TypeAdapter as PyTypeAdapter
 
-	return PyTypeAdapter(type_, config=FrappePydanticConfig)
+	try:
+		return PyTypeAdapter(type_, config=FrappePydanticConfig)
+	except PydanticUserError as e:
+		match e.code:
+			case "type-adapter-config-unused":
+				# Unless they set their custom __pydantic_config__, this will be the case on BaseModule, TypedDict and dataclass - ignore
+				return PyTypeAdapter(type_)
+			case _:
+				raise e
 
 
 def transform_parameter_types(func: Callable, args: tuple, kwargs: dict):

pyproject.toml

@@ -138,6 +138,13 @@ test = [
 requires = ["flit_core >=3.4,<4"]
 build-backend = "flit_core.buildapi"
 
+[tool.frappe.testing.function_type_validation]
+max_module_depth = 1
+skip_namespaces = [
+    "frappe.deprecation_dumpster",
+    "frappe.utils.typing_validations",
+]
+
 [tool.bench.dev-dependencies]
 coverage = "~=6.5.0"
 Faker = "~=18.10.1"