dataface 0.1.2__py3-none-any.whl

dataface/core/compile/normalizer.py

@@ -0,0 +1,543 @@
+"""Normalization module for transforming input types to compiled types.
+
+Stage: COMPILE (Step 3 of 4)
+Purpose: Transform AuthoredFace input types to Face output types.
+
+Entry Points:
+    - normalize_face(face: AuthoredFace) -> Face
+
+This is the core transformation step of compilation:
+1. Resolve all chart references (queries, extends, partials)
+2. Resolve remote dataface references
+3. Generate unique IDs for all entities
+4. Apply defaults (title from ID, empty description)
+5. Transform input types → compiled types
+6. Create unified layout structure
+
+After this step, all references are resolved and downstream code can
+rely on guaranteed field presence.
+
+Cross-file Query References:
+    Charts can reference queries from external files using the syntax:
+    `query: path/to/file.yml#query_name`
+
+    The path is resolved relative to the current file's directory.
+    External queries are loaded once and cached in the query registry.
+
+Dependencies:
+    - .models.face.authored (AuthoredFace)
+    - .models.face.compiled (Face, Layout)
+    - .models.chart.compiled (Chart)
+    - .models.query.compiled (AnyQuery)
+    - .jinja (resolve_jinja_template)
+    - .errors (ReferenceError, CompilationError)
+
+See also:
+    - compile/validator.py: Previous step
+    - compile/sizing.py: Next step
+"""
+
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from dataface.core.compile.errors import CompilationError
+from dataface.core.compile.models.chart.compiled import Chart
+from dataface.core.compile.models.face.authored import AuthoredFace
+from dataface.core.compile.models.face.compiled import (
+    Face,
+    VariableValues,
+)
+from dataface.core.compile.models.query.compiled import AnyQuery
+from dataface.core.compile.models.style.authored import StylePatch
+from dataface.core.compile.models.style.merged import (
+    MergedStyle,
+    resolve_style,
+)
+from dataface.core.compile.models.variable.authored import Variable
+from dataface.core.compile.normalize_charts import (
+    _collect_charts_from_layout,
+    _normalize_chart,
+)
+from dataface.core.compile.normalize_layout import build_unified_layout
+from dataface.core.compile.normalize_queries import (
+    normalize_query,  # noqa: F401 — re-exported (used by compiler.py, tests)
+)
+from dataface.core.compile.normalize_variables import (
+    _build_variable_registry,
+    _compute_variable_dependencies,
+    _detect_variable_input_type,
+    _expand_chart_variable_dependencies,
+    _generate_layout_variables,
+    _promote_inline_option_queries,
+    _validate_variable_references,
+    _validate_variable_value,
+)
+
+
+def compile_board_resolved_style(
+    face_style: StylePatch | None,
+    parent_resolved: MergedStyle | None,
+    theme_name: str | None = None,
+) -> MergedStyle:
+    """Build authoritative MergedStyle for this board scope.
+
+    Cascade:
+    1. Theme defaults (theme_name → compiled theme → resolve_style)
+    2. Parent semantic tokens (muted, accent) — propagate tonal identity
+    3. Face's style: block as StylePatch — all fields cascade normally
+    """
+    from dataface.core.compile.config import get_compiled_theme, get_config
+
+    base = get_compiled_theme(theme_name) if theme_name else get_config().style
+
+    parent_cascade = None
+    if parent_resolved is not None:
+        parent_cascade = StylePatch.model_validate(
+            {"muted": parent_resolved.muted, "accent": parent_resolved.accent}
+        )
+
+    if face_style is None:
+        return parent_resolved or resolve_style(base)
+
+    return (
+        resolve_style(base, parent_cascade, face_style)
+        if parent_cascade is not None
+        else resolve_style(base, face_style)
+    )
+
+
+def get_cascade_context(
+    face: AuthoredFace, parent_context: dict[str, Any]
+) -> dict[str, Any]:
+    """Build the cascade dict from Cascade-annotated fields on AuthoredFace.
+
+    Iterates model_fields, detects Cascade markers, and resolves each cascade
+    key's effective value (child's explicit value, or parent's propagated value).
+    Fields sharing a context key (e.g. source + sources → default_source) are
+    deduplicated: the first Cascade with that key wins, calling via= once.
+    """
+    from dataface.core.compile.models.markers import Cascade
+
+    result: dict[str, Any] = {}
+    seen_keys: set[str] = set()
+
+    for field_name, field_info in AuthoredFace.model_fields.items():
+        cascade = next(
+            (m for m in (field_info.metadata or []) if isinstance(m, Cascade)), None
+        )
+        if cascade is None:
+            continue
+        key = cascade.key or field_name
+        if key in seen_keys:
+            continue
+        seen_keys.add(key)
+        value = (
+            getattr(face, cascade.via)() if cascade.via else getattr(face, field_name)
+        )
+        result[key] = value or parent_context.get(key)
+
+    return result
+
+
+def sync_face_resolved_style(
+    face: Any, parent_resolved: MergedStyle | None = None
+) -> None:
+    """Re-resolve face.resolved_style from face.theme and recurse into nested faces.
+
+    Passes the freshly-computed resolved style as parent_resolved to each
+    nested sub-board so the parent→child semantic color cascade (muted, accent)
+    is preserved. Call this when face.theme has been mutated after compile.
+    """
+    face.resolved_style = compile_board_resolved_style(
+        face.authored_style, parent_resolved, theme_name=face.theme
+    )
+    for item in face.layout.items or []:
+        if item.face is not None:
+            sync_face_resolved_style(item.face, face.resolved_style)
+
+
+def _propagate_resolved_style(
+    layout: Any,
+    parent_resolved: MergedStyle,
+) -> None:
+    """Walk the layout tree and set resolved_style on nested Faces."""
+    from dataface.core.compile.models.face.compiled import Layout
+
+    if not isinstance(layout, Layout):
+        return
+
+    for item in layout.items:
+        if item.face is not None:
+            item.face.resolved_style = compile_board_resolved_style(
+                item.face.authored_style,
+                parent_resolved,
+                theme_name=item.face.theme,
+            )
+            _propagate_resolved_style(item.face.layout, item.face.resolved_style)
+
+
+def slugify(text: str) -> str:
+    """Convert text to a URL-safe slug.
+
+    "Raw Data" → "raw_data"
+    "My Tab!" → "my_tab"
+    """
+    slug = re.sub(r"[^\w\s-]", "", text.lower().strip())
+    return re.sub(r"[\s-]+", "_", slug).strip("_")
+
+
+def normalize_face(
+    face: AuthoredFace,
+    face_id: str | None = None,
+    parent_context: dict[str, Any] | None = None,
+    query_registry: dict[str, AnyQuery] | None = None,
+    chart_registry: dict[str, Any] | None = None,
+    depth: int = 0,
+    base_path: Path | None = None,
+    parent_level: int = 0,
+) -> Face:
+    """Normalize a AuthoredFace into a Face with guaranteed structure.
+
+    Stage: COMPILE (Step 3 of 4: Normalization)
+
+    This is the core transformation from user-provided YAML structure to the
+    internal compiled representation. After this step, all references are resolved,
+    IDs are assigned, and downstream code can rely on guaranteed field presence.
+
+    Normalization performs:
+    - Resolve all chart references (queries, extends, partials)
+    - Generate unique IDs for all entities
+    - Apply defaults (title from ID, empty description)
+    - Transform input types to compiled types
+    - Create unified Layout structure
+    - Propagate default source to queries without explicit source
+
+    Args:
+        face: Input AuthoredFace from parsing/validation step.
+        face_id: Optional explicit ID for this face
+        parent_context: Context from parent face (for nested faces)
+        query_registry: Complete query registry for reference resolution
+        chart_registry: Complete chart registry for reference resolution
+        depth: Structural nesting depth (for cycle detection and root detection)
+        base_path: Base path for resolving external file references
+        parent_level: Semantic heading level of the parent face. The current
+            face's level = parent_level + (1 if face.title else 0).
+            Root call passes 0 so a titled root face gets level=1.
+
+    Returns:
+        Face with guarantees:
+            - All fields required by compiled types are present
+            - All references resolved to actual objects
+            - All IDs are unique and deterministic
+            - Layout is unified (rows/cols/grid/tabs → Layout type)
+
+    Raises:
+        ReferenceError: When a chart references an undefined query
+        CompilationError: When normalization fails
+
+    Example:
+        >>> face = parse_yaml(yaml_content)
+        >>> errors = validate_face(face)
+        >>> if not errors:
+        ...     compiled = normalize_face(face)
+        ...     print(compiled.charts['revenue'].id)  # Guaranteed to exist
+        'revenue'
+    """
+    # Prevent infinite recursion
+    MAX_DEPTH = 50
+    if depth > MAX_DEPTH:
+        raise CompilationError(f"Maximum nesting depth ({MAX_DEPTH}) exceeded")
+
+    if depth > 0 and face.card_gap:
+        raise CompilationError(
+            "card_gap can only be set on the root face, not on nested faces"
+        )
+
+    # Root-level faces must have renderable content (layout or text). A title-only
+    # face is valid at depth>0 (AuthoredFace.validate_layout allows it for section
+    # headers and style wrappers), but is not renderable as a standalone dashboard.
+    if (
+        depth == 0
+        and not face.text
+        and all(f is None for f in (face.rows, face.cols, face.grid, face.tabs))
+    ):
+        raise CompilationError("Face must have at least one layout type or text")
+
+    parent_context = parent_context or {}
+    if base_path is None:
+        base_path = parent_context.get("base_path")
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 1: Setup — default source, registries, face ID
+    # ════════════════════════════════════════════════════════════════════
+    _cascade = get_cascade_context(face, parent_context)
+    default_source = _cascade.get("default_source")
+
+    # Build registries if not provided (allows direct normalize_face() calls in tests)
+    if query_registry is None:
+        from dataface.core.compile.compiler import build_query_registry
+
+        query_registry = build_query_registry(
+            face, base_dir=base_path, default_source=default_source
+        )
+
+    if chart_registry is None:
+        from dataface.core.compile.compiler import build_chart_registry
+
+        chart_registry = build_chart_registry(face, base_dir=base_path)
+
+    if not face_id:
+        face_id = _generate_face_id(face, depth, parent_context.get("parent_id"))
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 2: Variables
+    # ════════════════════════════════════════════════════════════════════
+    # Variable names are globally unique across the face tree; the renderer
+    # builds the global registry at render time. Here we collect local defs only.
+    local_variables: dict[str, Variable] = {}
+    if face.variables:
+        for var_name, var_def in face.variables.items():
+            if isinstance(var_def, Variable):
+                local_variables[var_name] = var_def
+            else:
+                # VariableRef — cross-file reference (e.g., "file.variables.var_name")
+                from dataface.core.compile.compiler import load_from_reference
+
+                local_variables[var_name] = load_from_reference(
+                    var_def, base_dir=base_path
+                )
+
+    # Auto-detect input type when set to "auto" (the default)
+    for var in local_variables.values():
+        was_auto = var.input == "auto"
+        var.input = _detect_variable_input_type(var)
+        # Only flag select as refinable — multiselect has multi-value semantics
+        # that can't map to datepicker/checkbox/slider, and strong structural
+        # signals (bool → checkbox, min/max → slider) shouldn't be overridden.
+        if was_auto and var.input == "select":
+            var.input_auto_detected = True
+
+    # Promote inline SQL in variable options.query to synthetic named queries
+    _promote_inline_option_queries(local_variables, query_registry, default_source)
+
+    # Compute cascading dropdown dependencies from Jinja variable references
+    _compute_variable_dependencies(local_variables, query_registry)
+
+    # Validate and collect local defaults (global defaults built by renderer)
+    variable_defaults: VariableValues = {}
+    for var_name, var in local_variables.items():
+        if var.default is not None:
+            _validate_variable_value(var_name, var, var.default)
+            variable_defaults[var_name] = var.default
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 3: Queries
+    # ════════════════════════════════════════════════════════════════════
+    # Extract locally defined queries from the global registry (for Face.queries).
+    local_queries: dict[str, AnyQuery] = {}
+    if face.queries:
+        for query_name in face.queries:
+            if query_name in query_registry:
+                local_queries[query_name] = query_registry[query_name]
+            else:
+                raise CompilationError(
+                    f"Query '{query_name}' not found in registry. "
+                    "This may indicate a compilation error."
+                )
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 4: Charts
+    # ════════════════════════════════════════════════════════════════════
+    # Normalize all charts from the global registry — layout resolution needs
+    # all of them by name. Then extract locally defined ones for Face.charts.
+    normalized_charts: dict[str, Chart] = {}
+    for chart_name, chart_def in chart_registry.items():
+        normalized_charts[chart_name] = _normalize_chart(
+            chart_name,
+            chart_def,
+            query_registry,
+            base_path,
+            default_source,
+            source_path=f"charts.{chart_name}",
+        )
+
+    local_charts: dict[str, Chart] = {}
+    if face.charts:
+        for chart_name in face.charts:
+            if chart_name in normalized_charts:
+                local_charts[chart_name] = normalized_charts[chart_name]
+            else:
+                raise CompilationError(
+                    f"Chart '{chart_name}' not found in registry. "
+                    "Charts must be defined in the face tree or imported."
+                )
+
+    charts = local_charts
+
+    # Expand transitive variable dependencies (A→B→C means chart depends on A, B, C)
+    _expand_chart_variable_dependencies(charts, local_variables)
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 5: Style & theme
+    # ════════════════════════════════════════════════════════════════════
+    # Faces should inherit the same effective theme choice as their charts so
+    # nested boards resolve typography/palette consistently. The face canvas
+    # background itself is intentionally not copied from chart themes: board
+    # chrome follows face/style defaults (white in Playground review mode),
+    # while chart surfaces keep the theme-owned canvas/background colors.
+    # If YAML does not specify a theme explicitly, inherit the parent theme or
+    # fall back to the configured default theme.
+    from dataface.core.compile.config import get_default_theme_name
+
+    theme = _cascade.get("theme") or get_default_theme_name()
+
+    # Compile board-scoped resolved style before layout so nested faces can inherit
+    resolved_style = compile_board_resolved_style(
+        face.style,
+        parent_context.get("resolved_style"),
+        theme_name=theme,
+    )
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 6: Layout
+    # ════════════════════════════════════════════════════════════════════
+    # parent_variables: inherited compile-time variables (foreach loop vars) from
+    # parent + local variable defaults. These are needed by foreach to generate
+    # static per-iteration content at compile time. Interactive variable defaults
+    # are included here so foreach items get them baked in (correct, since foreach
+    # produces static faces). For regular nested boards, _resolve_dict_templates is
+    # only called when _resolve_jinja=True, so interactive defaults don't get baked
+    # into regular board content at compile time.
+    parent_variables: dict[str, Any] = {}
+    if parent_context.get("variables"):
+        parent_variables.update(parent_context["variables"])
+    for var_name, var in local_variables.items():
+        if var.default is not None:
+            parent_variables[var_name] = var.default
+
+    # Semantic heading level: count only titled ancestors.
+    # Bare wrappers (title is empty/None) do not advance the level counter.
+    # If style.title.level is an integer override (from theme or face YAML), it wins;
+    # that value also becomes the parent_level for descendants (cascade as if structural truth).
+    style_level_override = resolved_style.title.level
+    if isinstance(style_level_override, int):
+        this_level = style_level_override
+    else:
+        this_level = parent_level + (1 if face.title else 0)
+
+    _resolve_jinja = parent_context.get("_resolve_jinja", False)
+    layout = build_unified_layout(
+        face,
+        normalized_charts,
+        query_registry,
+        face_id,
+        depth,
+        base_path,
+        default_source,
+        chart_registry,
+        theme,
+        parent_variables,
+        _resolve_jinja=_resolve_jinja,
+        resolved_style=resolved_style,
+        parent_level=this_level,
+    )
+
+    _propagate_resolved_style(layout, resolved_style)
+
+    # Auto-generate hidden variables for tabs/details widgets
+    auto_variables = _generate_layout_variables(layout)
+    local_variables.update(auto_variables)
+    for var_name, var in auto_variables.items():
+        if var.default is not None:
+            variable_defaults[var_name] = var.default
+
+    # Collect inline charts from layout so all charts are accessible via face.charts
+    # without traversing the layout tree.
+    all_charts = dict(charts)
+    _collect_charts_from_layout(layout, all_charts)
+    charts = all_charts
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 7: Assemble Face
+    # ════════════════════════════════════════════════════════════════════
+
+    compiled_face = Face(
+        id=face_id,
+        title=face.title or "",
+        description=face.description or "",
+        tags=face.tags or [],
+        text=face.text or "",
+        variables=local_variables,  # Local variables for UI controls
+        queries=local_queries,
+        charts=charts,
+        layout=layout,
+        variable_defaults=variable_defaults,  # Local variable defaults
+        card_gap=face.card_gap,
+        authored_style=face.style,
+        theme=theme,
+        resolved_style=resolved_style,
+        level=this_level,
+        meta={
+            "compiled_at": datetime.now(timezone.utc).isoformat(),
+            "version": "0.1.0",
+        },
+    )
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 8: Variable registry (root face only)
+    # ════════════════════════════════════════════════════════════════════
+    if depth == 0:
+        compiled_face.variable_registry = _build_variable_registry(compiled_face)
+
+        # Pre-compute global defaults to avoid repeated comprehension at render time
+        compiled_face.variable_defaults = {
+            name: var.default
+            for name, var in compiled_face.variable_registry.items()
+            if var.default is not None
+        }
+
+        # Validate that all referenced variables are defined
+        warnings = _validate_variable_references(compiled_face)
+        if warnings:
+            compiled_face.meta["variable_warnings"] = warnings
+
+    # ════════════════════════════════════════════════════════════════════
+    # STEP 9: Chart focus (root face only)
+    # ════════════════════════════════════════════════════════════════════
+    # Narrows the face to a single chart and its dependent variables.
+    if depth == 0 and face.chart_focus:
+        from dataface.core.compile.chart_focus import focus_on_chart
+
+        compiled_face = focus_on_chart(compiled_face, face.chart_focus)
+
+    return compiled_face
+
+
+def _generate_face_id(
+    face: AuthoredFace,
+    depth: int,
+    parent_id: str | None = None,
+) -> str:
+    """Generate a unique face ID.
+
+    Args:
+        face: AuthoredFace to generate ID for
+        depth: Nesting depth
+        parent_id: Parent face ID if nested
+
+    Returns:
+        Unique face ID
+    """
+    if face.id:
+        return face.id
+
+    if parent_id:
+        return f"{parent_id}_nested{depth}"
+
+    if face.title:
+        slug = slugify(face.title)
+        return slug or "untitled_face"
+
+    return "untitled_face"