dataface 0.1.2__py3-none-any.whl

dataface/core/compile/introspection.py

@@ -0,0 +1,527 @@
+"""Pydantic model introspection — Layer 1 of the two-layer schema IR.
+
+Walks AuthoredFace and all reachable authored models once, producing an
+AuthorableSchema IR. Renderers (json_schema, prompt, markdown, erd) consume
+this IR — no rendering logic lives here.
+
+Entry point: introspect() -> AuthorableSchema
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import typing
+from enum import Enum
+from typing import Any, ForwardRef, Literal, get_args, get_origin
+
+from pydantic import BaseModel
+from pydantic.fields import FieldInfo
+from pydantic_core import PydanticUndefinedType
+
+from dataface.core.compile.models.face.authored import AuthoredFace
+from dataface.core.compile.models.factories import _PatchBase
+from dataface.core.compile.models.query.authored import AuthoredQuery
+from dataface.core.compile.models.source import (
+    BigQuerySourceConfig,
+    CsvSourceConfig,
+    DbtProfileSourceConfig,
+    DuckDBSourceConfig,
+    HttpSourceConfig,
+    JsonSourceConfig,
+    MySQLSourceConfig,
+    ParquetSourceConfig,
+    PostgresSourceConfig,
+    RedshiftSourceConfig,
+    SnowflakeSourceConfig,
+)
+from dataface.core.compile.models.style.compiled import PaginationConfig
+
+
+@dataclasses.dataclass
+class SchemaField:
+    name: str
+    description: str
+    type_repr: str
+    required: bool
+    default: Any
+    default_repr: (
+        str | None
+    )  # human-readable default for display (None = no meaningful default)
+    enum_values: list[str] | None
+    nested_models: list[str]  # class names of nested AuthorableModels (empty = none)
+    container: str | None = None  # "list" or "dict" if the model is in a container
+    extra_union_types: list[str] = dataclasses.field(
+        default_factory=list
+    )  # primitive type names alongside models or enums in a union
+    cascades: bool = False  # True when value propagates (face or style cascade)
+    cascade_key: str | None = None  # face cascade: context key
+    cascade_targets: tuple[str, ...] = ()  # style cascade: sibling fields this fills in
+    is_extra_key: bool = (
+        False  # True for synthetic '<name>' fields from __pydantic_extra__
+    )
+
+
+@dataclasses.dataclass
+class AuthorableModel:
+    name: str
+    doc: str
+    fields: list[SchemaField]
+    generated: bool = False  # True for _PatchBase subclasses (build_patch_model output)
+
+
+@dataclasses.dataclass
+class AuthorableSchema:
+    root: str
+    models: dict[str, AuthorableModel]
+
+
+def _unwrap_annotated(annotation: Any) -> Any:
+    """Strip Annotated[X, *metadata] → X. Idempotent on plain types."""
+    if hasattr(annotation, "__metadata__"):
+        return annotation.__origin__
+    return annotation
+
+
+def _is_authored_model(cls: Any) -> bool:
+    """Return True if cls should be included in the IR.
+
+    Includes authored.* models, primitives, and generated _PatchBase subclasses.
+    """
+    if not (isinstance(cls, type) and issubclass(cls, BaseModel)):
+        return False
+    if issubclass(cls, _PatchBase):
+        return True
+    module = getattr(cls, "__module__", "")
+    return ".authored" in module or module.endswith("primitives")
+
+
+def _extract_enum_values(annotation: Any) -> list[str] | None:
+    """Return string enum values if annotation is an Enum or Literal, else None."""
+    annotation = _unwrap_annotated(annotation)
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+
+    # Literal["foo", "bar", ...]
+    if origin is Literal:
+        return [str(a) for a in args if a is not None]
+
+    # str Enum subclass
+    if isinstance(annotation, type) and issubclass(annotation, Enum):
+        return [e.value for e in annotation]
+
+    # Union — look for a Literal or Enum inside (e.g. ChartType | None)
+    if origin is type(None) or origin is None:
+        return None
+    # handle Union (typing.Union has origin = types.UnionType or typing.Union)
+    if args:
+        for arg in args:
+            result = _extract_enum_values(arg)
+            if result:
+                return result
+
+    return None
+
+
+def _nested_model_names(annotation: Any) -> list[str]:
+    """Return all authored model class names reachable from annotation.
+
+    Returns an empty list if no authored models are found.
+    For plain types, returns a one-element list; for discriminated unions,
+    returns one entry per branch.
+    """
+    annotation = _unwrap_annotated(annotation)
+    if _is_authored_model(annotation):
+        return [annotation.__name__]
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+    if not args:
+        return []
+
+    # dict[K, V] — recurse into the value type only
+    if origin is dict and len(args) == 2:
+        return _nested_model_names(args[1])
+
+    # list[SomeModel] or Union/Optional — collect all authored branches, dedup by class name
+    result: list[str] = []
+    for arg in args:
+        result.extend(_nested_model_names(arg))
+    return list(dict.fromkeys(result))
+
+
+def _all_model_names_in_annotation(annotation: Any) -> list[str]:
+    """Return class names of ALL BaseModel subclasses reachable from annotation.
+
+    Unlike _nested_model_names, this is not limited to "authored" modules.
+    Used for __pydantic_extra__ synthetic fields where the value type may
+    reference source connector configs (models.source module).
+    """
+    annotation = _unwrap_annotated(annotation)
+    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
+        return [annotation.__name__]
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+    if not args:
+        return []
+
+    if origin is dict and len(args) == 2:
+        return _all_model_names_in_annotation(args[1])
+
+    result: list[str] = []
+    for arg in args:
+        if arg is not type(None):
+            result.extend(_all_model_names_in_annotation(arg))
+    return result
+
+
+def _type_repr(annotation: Any) -> str:
+    """Human-readable type string for display in docs/prompts."""
+    annotation = _unwrap_annotated(annotation)
+    if annotation is None or annotation is type(None):
+        return "None"
+    if isinstance(annotation, ForwardRef):
+        return annotation.__forward_arg__
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+
+    if origin is Literal:
+        values = [repr(a) for a in args]
+        return f"one of: {', '.join(values)}"
+
+    # Guard: only match plain types, not generic aliases (list[X], dict[K,V])
+    # On Python 3.10, isinstance(list[X], type) can return True for GenericAlias.
+    if origin is None and isinstance(annotation, type):
+        if issubclass(annotation, Enum):
+            return f"one of: {', '.join(e.value for e in annotation)}"
+        return annotation.__name__
+
+    if origin is list:
+        if args:
+            return f"list[{_type_repr(args[0])}]"
+        return "list"
+
+    if origin is dict:
+        if len(args) == 2:
+            return f"dict[{_type_repr(args[0])}, {_type_repr(args[1])}]"
+        return "dict"
+
+    # Union types (X | Y | None) — deduplicate branches (discriminated unions may
+    # have the same class under multiple Tag() aliases, e.g. BarChart for bar+histogram)
+    if args:
+        non_none = [a for a in args if a is not type(None)]
+        nullable = any(a is type(None) for a in args)
+        parts = list(dict.fromkeys(_type_repr(a) for a in non_none))
+        result = " | ".join(parts)
+        if nullable and len(non_none) > 0:
+            result += " | None"
+        return result
+
+    if annotation is Any:
+        return "Any"
+    return str(annotation)
+
+
+def _get_container(annotation: Any) -> str | None:
+    """Return 'list' or 'dict' if annotation wraps a nested model in a container."""
+    annotation = _unwrap_annotated(annotation)
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+    if origin is list:
+        return "list"
+    if origin is dict:
+        return "dict"
+    if args:
+        non_none = [a for a in args if a is not type(None)]
+        if len(non_none) == 1:
+            return _get_container(non_none[0])
+    return None
+
+
+_PRIMITIVE_NAMES = frozenset({"str", "int", "float", "bool"})
+
+
+def _default_repr(field_info: FieldInfo) -> str | None:
+    """Return a human-readable default string for display, or None if no meaningful default."""
+    if field_info.is_required():
+        return None
+    default = field_info.default
+    if isinstance(default, PydanticUndefinedType):
+        factory = field_info.default_factory
+        if factory is dict:
+            return "{}"
+        if factory is list:
+            return "[]"
+        return None
+    if default is None:
+        return None
+    if isinstance(default, bool):
+        return str(default).lower()
+    if isinstance(default, str):
+        return f'"{default}"'
+    if isinstance(default, (int, float)):
+        return str(default)
+    return None
+
+
+def _extra_union_types(annotation: Any) -> list[str]:
+    """Primitive type names that sit alongside models or enums in a union.
+
+    Returns the names (e.g. ["str", "bool"]) so the renderer can include
+    them in anyOf alongside $ref or enum branches.
+    Only meaningful for bare Union types — list/dict containers return [].
+    """
+    annotation = _unwrap_annotated(annotation)
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+    if not args or origin in (list, dict):
+        return []
+
+    result: list[str] = []
+    for arg in args:
+        if arg is type(None):
+            continue
+        if _is_authored_model(arg):
+            continue
+        arg_origin = get_origin(arg)
+        if arg_origin is Literal:
+            continue
+        if isinstance(arg, type) and issubclass(arg, Enum):
+            continue
+        if (
+            arg_origin is None
+            and isinstance(arg, type)
+            and arg.__name__ in _PRIMITIVE_NAMES
+        ):
+            result.append(arg.__name__)
+    return result
+
+
+def _build_schema_field(
+    name: str, field_info: FieldInfo, annotation: Any
+) -> SchemaField:
+    from dataface.core.compile.models.markers import Cascade
+
+    required = field_info.is_required()
+    default = field_info.default if not required else dataclasses.MISSING
+    cascade = next(
+        (m for m in (field_info.metadata or []) if isinstance(m, Cascade)), None
+    )
+    return SchemaField(
+        name=name,
+        description=field_info.description or "",
+        type_repr=_type_repr(annotation),
+        required=required,
+        default=default,
+        default_repr=_default_repr(field_info),
+        enum_values=_extract_enum_values(annotation),
+        nested_models=_nested_model_names(annotation),
+        container=_get_container(annotation),
+        extra_union_types=_extra_union_types(annotation),
+        cascades=cascade is not None,
+        cascade_key=(cascade.key or name) if cascade is not None else None,
+        cascade_targets=cascade.targets if cascade is not None else (),
+    )
+
+
+def _collect_models(
+    model_cls: type[BaseModel],
+    collected: dict[str, AuthorableModel],
+) -> None:
+    """Recursively walk model_cls and all reachable authored sub-models."""
+    name = model_cls.__name__
+    if name in collected:
+        return
+
+    # Use __doc__ directly (not inspect.getdoc) to avoid MRO walk that pulls in
+    # Pydantic BaseModel's "[Models](../concepts/models.md)" doc for generated patches.
+    raw_doc = model_cls.__doc__ or ""
+    doc = ""
+    for line in raw_doc.split("\n"):
+        stripped = line.strip()
+        if stripped and not stripped.startswith(("!!! ", "??? ", ":::")):
+            doc = stripped
+            break
+
+    fields: list[SchemaField] = []
+
+    for field_name, field_info in model_cls.model_fields.items():
+        if field_info.exclude:
+            continue
+        annotation = field_info.annotation
+        display_name = field_info.alias or field_name
+        sf = _build_schema_field(display_name, field_info, annotation)
+        fields.append(sf)
+
+    # Synthesize a '<name>' field for models with typed __pydantic_extra__.
+    # get_type_hints() is needed because from __future__ import annotations turns
+    # annotations into strings; vars() gives the raw (unresolved) form.
+    try:
+        hints = typing.get_type_hints(model_cls)
+        extra_annotation = hints.get("__pydantic_extra__")
+    except NameError:
+        extra_annotation = None
+    if extra_annotation is not None:
+        ea_origin = get_origin(extra_annotation)
+        ea_args = get_args(extra_annotation)
+        if ea_origin is dict and len(ea_args) == 2:
+            value_annotation = ea_args[1]
+            fields.append(
+                SchemaField(
+                    name="<name>",
+                    description="Additional named entry.",
+                    type_repr=_type_repr(value_annotation),
+                    required=False,
+                    default=None,
+                    default_repr=None,
+                    enum_values=None,
+                    nested_models=_all_model_names_in_annotation(value_annotation),
+                    is_extra_key=True,
+                )
+            )
+
+    collected[name] = AuthorableModel(
+        name=name,
+        doc=doc,
+        fields=fields,
+        generated=issubclass(model_cls, _PatchBase),
+    )
+
+    # Recurse into nested authored models
+    for sf in fields:
+        for model_name in sf.nested_models:
+            if model_name not in collected:
+                nested_cls = _find_nested_class(model_cls, model_name)
+                if nested_cls is not None and _is_authored_model(nested_cls):
+                    _collect_models(nested_cls, collected)
+
+
+def _find_nested_class(parent_cls: type[BaseModel], class_name: str) -> type | None:
+    """Find a nested model class by name from a parent model's field annotations."""
+    for field_info in parent_cls.model_fields.values():
+        annotation = field_info.annotation
+        cls = _find_in_annotation(annotation, class_name)
+        if cls is not None:
+            return cls
+    return None
+
+
+def _find_in_annotation(annotation: Any, class_name: str) -> type | None:
+    """Recursively search annotation tree for a class with the given name."""
+    if isinstance(annotation, type) and annotation.__name__ == class_name:
+        return annotation
+    args = get_args(annotation)
+    for arg in args:
+        result = _find_in_annotation(arg, class_name)
+        if result is not None:
+            return result
+    return None
+
+
+def introspect() -> AuthorableSchema:
+    """Walk AuthoredFace and all reachable authored models, returning an IR.
+
+    The returned AuthorableSchema is the single source of truth consumed by
+    render_json_schema, render_prompt, render_markdown, and render_erd.
+    """
+    collected: dict[str, AuthorableModel] = {}
+    _collect_models(AuthoredFace, collected)
+    # Walk extra roots not reachable via AuthoredFace type annotations.
+    # AuthoredQuery uses untyped dicts in the authored YAML (normalizer coerces).
+    # Source connector configs are reached via sources: dict[str, Any] — no type annotation.
+    # Per-family chart patch classes are not reachable via AuthoredFace annotations either
+    # (AuthoredChart is a discriminated union alias, not a typed field on AuthoredFace).
+    from dataface.core.compile.models.chart.authored import (  # noqa: PLC0415
+        AreaChart,
+        BarChart,
+        CalloutChart,
+        GeoshapeChart,
+        HeatmapChart,
+        KpiChart,
+        LayeredChart,
+        LineChart,
+        PieChart,
+        PointMapChart,
+        ScatterChart,
+        SparkBarChart,
+        TableChart,
+    )
+
+    for extra in (
+        BarChart,
+        LineChart,
+        AreaChart,
+        ScatterChart,
+        HeatmapChart,
+        PieChart,
+        KpiChart,
+        TableChart,
+        PointMapChart,
+        GeoshapeChart,
+        LayeredChart,
+        CalloutChart,
+        SparkBarChart,
+        AuthoredQuery,
+        PaginationConfig,
+        PostgresSourceConfig,
+        SnowflakeSourceConfig,
+        BigQuerySourceConfig,
+        RedshiftSourceConfig,
+        MySQLSourceConfig,
+        DuckDBSourceConfig,
+        CsvSourceConfig,
+        ParquetSourceConfig,
+        JsonSourceConfig,
+        HttpSourceConfig,
+        DbtProfileSourceConfig,
+    ):
+        _collect_models(extra, collected)
+    # Build a synthetic "AuthoredChart" IR entry as a union of all per-family patches.
+    # This gives prompt.py and vscode_schema.py a single named surface to BFS from.
+    # The entry lists all fields across all families — renderers that need per-family
+    # discrimination should walk the individual *Patch entries directly.
+    all_patch_fields: dict[str, SchemaField] = {}
+    for patch_name in (
+        "BarChart",
+        "LineChart",
+        "AreaChart",
+        "ScatterChart",
+        "HeatmapChart",
+        "PieChart",
+        "KpiChart",
+        "TableChart",
+        "PointMapChart",
+        "GeoshapeChart",
+        "LayeredChart",
+        "CalloutChart",
+        "SparkBarChart",
+    ):
+        if patch_name not in collected:
+            continue
+        for f in collected[patch_name].fields:
+            if f.name not in all_patch_fields:
+                all_patch_fields[f.name] = f
+    if all_patch_fields:
+        from dataface.core.compile.models.chart.authored import (  # noqa: PLC0415
+            ChartType,
+        )
+
+        chart_type_values = [t.value for t in ChartType]
+        chart_patch_fields = [
+            (
+                dataclasses.replace(f, enum_values=chart_type_values)
+                if f.name == "type"
+                else f
+            )
+            for f in all_patch_fields.values()
+        ]
+        collected["AuthoredChart"] = AuthorableModel(
+            name="AuthoredChart",
+            doc=(
+                "Discriminated union of per-family chart patches. "
+                "type: is mandatory; missing or unknown type raises ValidationError."
+            ),
+            fields=chart_patch_fields,
+        )
+    return AuthorableSchema(root="AuthoredFace", models=collected)