feat(DX): type annotated python controllers

frappe/core/doctype/doctype/doctype.py

@@ -409,6 +409,7 @@ class DocType(Document):
 			self.export_doc()
 			self.make_controller_template()
 			self.set_base_class_for_controller()
+			self.export_types_to_controller()
 
 		# update index
 		if not self.custom:
@@ -719,6 +720,18 @@ class DocType(Document):
 			make_boilerplate("templates/controller.html", self.as_dict())
 			make_boilerplate("templates/controller_row.html", self.as_dict())
 
+	def export_types_to_controller(self):
+		from frappe.modules.utils import get_module_app
+		from frappe.types.exporter import TypeExporter
+
+		try:
+			app = get_module_app(self.module)
+		except frappe.DoesNotExistError:
+			return
+
+		if any(frappe.get_hooks("export_python_type_annotations", app_name=app)):
+			TypeExporter(self).export_types()
+
 	def make_amendable(self):
 		"""If is_submittable is set, add amended_from docfields."""
 		if self.is_submittable:

frappe/core/doctype/doctype/test_doctype.py

@@ -648,6 +648,44 @@ class TestDocType(FrappeTestCase):
 	def test_no_delete_doc(self):
 		self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address")
 
+	@patch.dict(frappe.conf, {"developer_mode": 1})
+	def test_export_types(self):
+		"""Export python types."""
+		import ast
+
+		from frappe.types.exporter import TypeExporter
+
+		def validate(code):
+			ast.parse(code)
+
+		doctype = new_doctype(custom=0).insert()
+
+		exporter = TypeExporter(doctype)
+		code = exporter.controller_path.read_text()
+		validate(code)
+
+		# regenerate and verify and file is same word to word.
+		exporter.export_types()
+		new_code = exporter.controller_path.read_text()
+		validate(new_code)
+
+		self.assertEqual(code, new_code)
+
+		# Add fields and save
+
+		fieldname = "test_type"
+		doctype.append("fields", {"fieldname": fieldname, "fieldtype": "Int"})
+		doctype.save()
+
+		new_field_code = exporter.controller_path.read_text()
+		validate(new_field_code)
+
+		self.assertIn(fieldname, new_field_code)
+		self.assertIn("Int", new_field_code)
+
+		doctype.delete()
+		frappe.db.commit()
+
 	@patch.dict(frappe.conf, {"developer_mode": 1})
 	def test_custom_field_deletion(self):
 		"""Custom child tables whose doctype doesn't exist should be auto deleted."""

frappe/hooks.py

@@ -429,3 +429,5 @@ extend_bootinfo = [
 	"frappe.utils.telemetry.add_bootinfo",
 	"frappe.core.doctype.user_permission.user_permission.send_user_permissions",
 ]
+
+export_python_type_annotations = True

frappe/model/document.py

@@ -19,6 +19,7 @@ from frappe.model.docstatus import DocStatus
 from frappe.model.naming import set_new_name, validate_name
 from frappe.model.utils import is_virtual_doctype
 from frappe.model.workflow import set_workflow_state_on_action, validate_workflow
+from frappe.types import DF
 from frappe.utils import compare, cstr, date_diff, file_lock, flt, get_datetime_str, now
 from frappe.utils.data import get_absolute_url
 from frappe.utils.global_search import update_global_search
@@ -81,6 +82,15 @@ def get_doc(*args, **kwargs):
 class Document(BaseDocument):
 	"""All controllers inherit from `Document`."""
 
+	doctype: DF.Data
+	name: DF.Data | None
+	flags: frappe._dict[str, Any]
+	owner: DF.Link
+	creation: DF.Datetime
+	modified: DF.Datetime
+	modified_by: DF.Link
+	idx: DF.Int
+
 	def __init__(self, *args, **kwargs):
 		"""Constructor.
 

frappe/types/DF.py

@@ -0,0 +1,38 @@
+"""This file defines all frappe types."""
+
+from datetime import date, datetime, time
+from typing import Literal
+
+# DocField types
+Data = str
+Text = str
+Autocomplete = Data
+Attach = Data
+AttachImage = Data
+Barcode = Data
+Check = int
+Code = Text
+Color = str
+Currency = float
+Date = str | date
+Datetime = str | datetime
+Duration = int
+DynamicLink = Data
+Float = str
+HTMLEditor = Text
+Int = int
+JSON = Text
+Link = Data
+LongText = Text
+MarkdownEditor = Text
+Password = Data
+Percent = float
+Phone = Data
+ReadOnly = Data
+Rating = float
+Select = Literal
+SmallText = Text
+TextEditor = Text
+Time = str | time
+Table = list
+TableMultiSelect = list

frappe/types/exporter.py

@@ -0,0 +1,205 @@
+"""Creates/Updates types in python controller when schema is updated.
+
+Design goal:
+	- Developer should be able to see schema in same file.
+	- Type checkers should assist with field names and basic validation in same file.
+	- `get_doc` outside of same file without explicit annotation is out of scope.
+	- Customizations like change of fieldtype and addition of fields are out of scope.
+"""
+
+import ast
+import inspect
+import re
+import textwrap
+import tokenize
+from keyword import iskeyword
+from pathlib import Path
+
+import frappe
+from frappe.types import DF
+
+field_template = "{field}: {type}"
+
+start_block = "# begin: auto-generated types"
+end_block = "# end: auto-generated types"
+
+type_code_block_template = """{start_block}
+# This code is auto-generated. Do not modify anything in this block.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+{imports}
+
+{fields}
+{end_block}"""
+
+
+non_nullable_types = {
+	"Check",
+	"Currency",
+	"Float",
+	"Int",
+	"Percent",
+	"Rating",
+	"Select",
+	"Table",
+	"TableMultiSelect",
+}
+
+
+class TypeExporter:
+	def __init__(self, doc):
+		from frappe.model.base_document import get_controller
+
+		self.doc = doc
+		self.doctype = doc.name
+		self.field_types = {}
+
+		self.imports = {"from frappe.types import DF"}
+		self.indent = "\t"
+		self.controller_path = Path(inspect.getfile(get_controller(self.doctype)))
+
+	def export_types(self):
+		self._guess_indetation()
+		new_code = self._generate_code()
+		self._replace_or_add_code(new_code)
+
+	def _replace_or_add_code(self, new_code: str):
+		despaced_name = self.doctype.replace(" ", "")
+
+		class_definition = f"class {despaced_name}("  # )
+		code = self.controller_path.read_text()
+
+		first_line, *_, last_line = new_code.splitlines()
+		if first_line in code and last_line in code:  # Replace
+			existing_block_start = code.find(first_line)
+			existing_block_end = code.find(last_line) + len(last_line)
+
+			code = code[:existing_block_start] + new_code + code[existing_block_end:]
+		elif class_definition in code:  # Add just after class definition
+			# Regex by default will only match till line ends, span end is when we need to stop
+			if class_def := re.search(rf"class {despaced_name}\(.*", code):  # )
+				class_definition_end = class_def.span()[1] + 1
+				code = code[:class_definition_end] + new_code + "\n" + code[class_definition_end:]
+
+		if self._validate_code(code):
+			self.controller_path.write_text(code)
+
+	def _generate_code(self):
+		for field in self.doc.fields:
+			if iskeyword(field.fieldname):
+				continue
+			if python_type := self._map_fieldtype(field):
+				self.field_types[field.fieldname] = python_type
+
+		if self.doc.istable:
+			for parent_field in ("parent", "parentfield", "parenttype"):
+				self.field_types[parent_field] = "DF.Data"
+
+		if self.doc.autoname == "autoincrement":
+			self.field_types["name"] = "DF.Int | None"
+
+		fields_code_block = self._create_fields_code_block()
+		imports = self._create_imports_block()
+
+		return textwrap.indent(
+			type_code_block_template.format(
+				start_block=start_block,
+				end_block=end_block,
+				fields=textwrap.indent(fields_code_block, self.indent),
+				imports=textwrap.indent(imports, self.indent),
+			),
+			self.indent,
+		)
+
+	def _create_fields_code_block(self):
+		fields = []
+
+		for field, typehint in self.field_types.items():
+			fields.append(field_template.format(field=field, type=typehint))
+		return "\n".join(sorted(fields))
+
+	def _create_imports_block(self) -> str:
+		return "\n".join(sorted(self.imports))
+
+	def _get_doctype_imports(self, doctype):
+		from frappe.model.base_document import get_controller
+
+		doctype_module = get_controller(doctype)
+
+		filepath = doctype_module.__module__
+		class_name = doctype_module.__name__
+
+		return f"from {filepath} import {class_name}", class_name
+
+	def _map_fieldtype(self, field) -> type | None:
+		fieldtype = field.fieldtype.replace(" ", "")
+		field_definition = ""
+
+		if fieldtype == "Select":
+			field_definition += "DF.Literal"
+		elif getattr(DF, fieldtype, None):
+			field_definition += f"DF.{fieldtype}"
+		else:
+			return
+
+		if parameter_definition := self._generic_parameters(field):
+			field_definition += parameter_definition
+
+		if self._is_nullable(field):
+			field_definition += " | None"
+
+		return field_definition
+
+	def _is_nullable(self, field) -> bool:
+		"""If value can be `None`"""
+
+		if field.fieldtype in non_nullable_types:
+			return False
+
+		return not bool(field.reqd)
+
+	def _generic_parameters(self, field) -> str | None:
+		"""If field is container type then return element type."""
+		if field.fieldtype in ("Table", "Table MultiSelect"):
+			doctype = field.options
+			if not doctype:
+				return
+
+			import_statment, cls_name = self._get_doctype_imports(doctype)
+			self.imports.add(import_statment)
+			return f"[{cls_name}]"
+
+		elif field.fieldtype == "Select":
+			if not field.options:
+				# Could be dynamic
+				return
+			options = [o.strip() for o in field.options.split("\n")]
+			return repr(options)
+
+	@staticmethod
+	def _validate_code(code) -> bool:
+		"""Make sure whatever code Frappe adds dynamically is valid python."""
+		try:
+			ast.parse(code)
+			return True
+		except Exception:
+			frappe.msgprint(frappe._("Failed to export python type hints"), alert=True)
+			return False
+
+	def _guess_indetation(
+		self,
+	) -> str:
+		from token import INDENT
+
+		with self.controller_path.open() as f:
+			for token in tokenize.generate_tokens(f.readline):
+				if token.type == INDENT:
+					if "\t" in token.string:
+						self.indent = "\t"
+					else:
+						# TODO: any other custom indent not supported
+						# Ideally this should be longest common substring but I don't l33tc0de.
+						# If someone really needs it, add support via hooks.
+						self.indent = " " * 4