From 0bc5d1dc3b7aa296a6ea88e83ec89f36dbd7c5d2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 23 Jul 2023 13:41:16 +0530 Subject: [PATCH] feat(DX): type annotated python controllers --- frappe/core/doctype/doctype/doctype.py | 13 ++ frappe/core/doctype/doctype/test_doctype.py | 38 ++++ frappe/hooks.py | 2 + frappe/model/document.py | 10 + frappe/types/DF.py | 38 ++++ frappe/types/__init__.py | 0 frappe/types/exporter.py | 205 ++++++++++++++++++++ 7 files changed, 306 insertions(+) create mode 100644 frappe/types/DF.py create mode 100644 frappe/types/__init__.py create mode 100644 frappe/types/exporter.py diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index c127335b16..7accea40ff 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -409,6 +409,7 @@ class DocType(Document): self.export_doc() self.make_controller_template() self.set_base_class_for_controller() + self.export_types_to_controller() # update index if not self.custom: @@ -719,6 +720,18 @@ class DocType(Document): make_boilerplate("templates/controller.html", self.as_dict()) make_boilerplate("templates/controller_row.html", self.as_dict()) + def export_types_to_controller(self): + from frappe.modules.utils import get_module_app + from frappe.types.exporter import TypeExporter + + try: + app = get_module_app(self.module) + except frappe.DoesNotExistError: + return + + if any(frappe.get_hooks("export_python_type_annotations", app_name=app)): + TypeExporter(self).export_types() + def make_amendable(self): """If is_submittable is set, add amended_from docfields.""" if self.is_submittable: diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index fead7672fe..c819663962 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -648,6 +648,44 @@ class TestDocType(FrappeTestCase): def test_no_delete_doc(self): self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address") + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_export_types(self): + """Export python types.""" + import ast + + from frappe.types.exporter import TypeExporter + + def validate(code): + ast.parse(code) + + doctype = new_doctype(custom=0).insert() + + exporter = TypeExporter(doctype) + code = exporter.controller_path.read_text() + validate(code) + + # regenerate and verify and file is same word to word. + exporter.export_types() + new_code = exporter.controller_path.read_text() + validate(new_code) + + self.assertEqual(code, new_code) + + # Add fields and save + + fieldname = "test_type" + doctype.append("fields", {"fieldname": fieldname, "fieldtype": "Int"}) + doctype.save() + + new_field_code = exporter.controller_path.read_text() + validate(new_field_code) + + self.assertIn(fieldname, new_field_code) + self.assertIn("Int", new_field_code) + + doctype.delete() + frappe.db.commit() + @patch.dict(frappe.conf, {"developer_mode": 1}) def test_custom_field_deletion(self): """Custom child tables whose doctype doesn't exist should be auto deleted.""" diff --git a/frappe/hooks.py b/frappe/hooks.py index af801e9e7b..bb464b193b 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -429,3 +429,5 @@ extend_bootinfo = [ "frappe.utils.telemetry.add_bootinfo", "frappe.core.doctype.user_permission.user_permission.send_user_permissions", ] + +export_python_type_annotations = True diff --git a/frappe/model/document.py b/frappe/model/document.py index 7130a727e0..591576c962 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -19,6 +19,7 @@ from frappe.model.docstatus import DocStatus from frappe.model.naming import set_new_name, validate_name from frappe.model.utils import is_virtual_doctype from frappe.model.workflow import set_workflow_state_on_action, validate_workflow +from frappe.types import DF from frappe.utils import compare, cstr, date_diff, file_lock, flt, get_datetime_str, now from frappe.utils.data import get_absolute_url from frappe.utils.global_search import update_global_search @@ -81,6 +82,15 @@ def get_doc(*args, **kwargs): class Document(BaseDocument): """All controllers inherit from `Document`.""" + doctype: DF.Data + name: DF.Data | None + flags: frappe._dict[str, Any] + owner: DF.Link + creation: DF.Datetime + modified: DF.Datetime + modified_by: DF.Link + idx: DF.Int + def __init__(self, *args, **kwargs): """Constructor. diff --git a/frappe/types/DF.py b/frappe/types/DF.py new file mode 100644 index 0000000000..2050d08119 --- /dev/null +++ b/frappe/types/DF.py @@ -0,0 +1,38 @@ +"""This file defines all frappe types.""" + +from datetime import date, datetime, time +from typing import Literal + +# DocField types +Data = str +Text = str +Autocomplete = Data +Attach = Data +AttachImage = Data +Barcode = Data +Check = int +Code = Text +Color = str +Currency = float +Date = str | date +Datetime = str | datetime +Duration = int +DynamicLink = Data +Float = str +HTMLEditor = Text +Int = int +JSON = Text +Link = Data +LongText = Text +MarkdownEditor = Text +Password = Data +Percent = float +Phone = Data +ReadOnly = Data +Rating = float +Select = Literal +SmallText = Text +TextEditor = Text +Time = str | time +Table = list +TableMultiSelect = list diff --git a/frappe/types/__init__.py b/frappe/types/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/types/exporter.py b/frappe/types/exporter.py new file mode 100644 index 0000000000..aa906330d0 --- /dev/null +++ b/frappe/types/exporter.py @@ -0,0 +1,205 @@ +"""Creates/Updates types in python controller when schema is updated. + +Design goal: + - Developer should be able to see schema in same file. + - Type checkers should assist with field names and basic validation in same file. + - `get_doc` outside of same file without explicit annotation is out of scope. + - Customizations like change of fieldtype and addition of fields are out of scope. +""" + +import ast +import inspect +import re +import textwrap +import tokenize +from keyword import iskeyword +from pathlib import Path + +import frappe +from frappe.types import DF + +field_template = "{field}: {type}" + +start_block = "# begin: auto-generated types" +end_block = "# end: auto-generated types" + +type_code_block_template = """{start_block} +# This code is auto-generated. Do not modify anything in this block. + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: +{imports} + +{fields} +{end_block}""" + + +non_nullable_types = { + "Check", + "Currency", + "Float", + "Int", + "Percent", + "Rating", + "Select", + "Table", + "TableMultiSelect", +} + + +class TypeExporter: + def __init__(self, doc): + from frappe.model.base_document import get_controller + + self.doc = doc + self.doctype = doc.name + self.field_types = {} + + self.imports = {"from frappe.types import DF"} + self.indent = "\t" + self.controller_path = Path(inspect.getfile(get_controller(self.doctype))) + + def export_types(self): + self._guess_indetation() + new_code = self._generate_code() + self._replace_or_add_code(new_code) + + def _replace_or_add_code(self, new_code: str): + despaced_name = self.doctype.replace(" ", "") + + class_definition = f"class {despaced_name}(" # ) + code = self.controller_path.read_text() + + first_line, *_, last_line = new_code.splitlines() + if first_line in code and last_line in code: # Replace + existing_block_start = code.find(first_line) + existing_block_end = code.find(last_line) + len(last_line) + + code = code[:existing_block_start] + new_code + code[existing_block_end:] + elif class_definition in code: # Add just after class definition + # Regex by default will only match till line ends, span end is when we need to stop + if class_def := re.search(rf"class {despaced_name}\(.*", code): # ) + class_definition_end = class_def.span()[1] + 1 + code = code[:class_definition_end] + new_code + "\n" + code[class_definition_end:] + + if self._validate_code(code): + self.controller_path.write_text(code) + + def _generate_code(self): + for field in self.doc.fields: + if iskeyword(field.fieldname): + continue + if python_type := self._map_fieldtype(field): + self.field_types[field.fieldname] = python_type + + if self.doc.istable: + for parent_field in ("parent", "parentfield", "parenttype"): + self.field_types[parent_field] = "DF.Data" + + if self.doc.autoname == "autoincrement": + self.field_types["name"] = "DF.Int | None" + + fields_code_block = self._create_fields_code_block() + imports = self._create_imports_block() + + return textwrap.indent( + type_code_block_template.format( + start_block=start_block, + end_block=end_block, + fields=textwrap.indent(fields_code_block, self.indent), + imports=textwrap.indent(imports, self.indent), + ), + self.indent, + ) + + def _create_fields_code_block(self): + fields = [] + + for field, typehint in self.field_types.items(): + fields.append(field_template.format(field=field, type=typehint)) + return "\n".join(sorted(fields)) + + def _create_imports_block(self) -> str: + return "\n".join(sorted(self.imports)) + + def _get_doctype_imports(self, doctype): + from frappe.model.base_document import get_controller + + doctype_module = get_controller(doctype) + + filepath = doctype_module.__module__ + class_name = doctype_module.__name__ + + return f"from {filepath} import {class_name}", class_name + + def _map_fieldtype(self, field) -> type | None: + fieldtype = field.fieldtype.replace(" ", "") + field_definition = "" + + if fieldtype == "Select": + field_definition += "DF.Literal" + elif getattr(DF, fieldtype, None): + field_definition += f"DF.{fieldtype}" + else: + return + + if parameter_definition := self._generic_parameters(field): + field_definition += parameter_definition + + if self._is_nullable(field): + field_definition += " | None" + + return field_definition + + def _is_nullable(self, field) -> bool: + """If value can be `None`""" + + if field.fieldtype in non_nullable_types: + return False + + return not bool(field.reqd) + + def _generic_parameters(self, field) -> str | None: + """If field is container type then return element type.""" + if field.fieldtype in ("Table", "Table MultiSelect"): + doctype = field.options + if not doctype: + return + + import_statment, cls_name = self._get_doctype_imports(doctype) + self.imports.add(import_statment) + return f"[{cls_name}]" + + elif field.fieldtype == "Select": + if not field.options: + # Could be dynamic + return + options = [o.strip() for o in field.options.split("\n")] + return repr(options) + + @staticmethod + def _validate_code(code) -> bool: + """Make sure whatever code Frappe adds dynamically is valid python.""" + try: + ast.parse(code) + return True + except Exception: + frappe.msgprint(frappe._("Failed to export python type hints"), alert=True) + return False + + def _guess_indetation( + self, + ) -> str: + from token import INDENT + + with self.controller_path.open() as f: + for token in tokenize.generate_tokens(f.readline): + if token.type == INDENT: + if "\t" in token.string: + self.indent = "\t" + else: + # TODO: any other custom indent not supported + # Ideally this should be longest common substring but I don't l33tc0de. + # If someone really needs it, add support via hooks. + self.indent = " " * 4