dataface 0.1.2__py3-none-any.whl

dataface/core/compile/normalize_charts.py

@@ -0,0 +1,494 @@
+"""Chart normalization and inline chart handling."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel
+
+from dataface.core.compile.custom_chart_types import BUILTIN_CHART_TYPE_VALUES
+from dataface.core.compile.errors import CompilationError, ReferenceError
+from dataface.core.compile.jinja import extract_variable_dependencies
+from dataface.core.compile.models.chart.authored import (
+    VALID_LAYER_TYPES,
+    Layer,
+)
+from dataface.core.compile.models.chart.compiled import (
+    Chart,
+)
+from dataface.core.compile.models.face.compiled import Layout
+from dataface.core.compile.models.query.compiled import AnyQuery
+from dataface.core.compile.models.style.authored import ChartStylePatch
+from dataface.core.compile.normalize_queries import (
+    normalize_query,
+    resolve_external_query,
+)
+from dataface.core.compile.parser import looks_like_sql
+from dataface.core.utils import slug_to_text as _slug_to_text
+
+if TYPE_CHECKING:
+    from dataface.core.compile.custom_chart_types import CustomChartTypeRegistry
+
+# Chart type aliases: rewrite alternate names to canonical forms.
+# Applied before any other normalization so all downstream code sees only
+# canonical names.  donut → pie is handled separately in _normalize_chart()
+# because it also sets style.inner_radius=0.6, not just a type rename.
+CHART_TYPE_ALIASES: dict[str, str] = {
+    "point": "scatter",
+    "choropleth": "map",
+}
+
+# Maps authored chart-type values to the sub-key on ChartStylePatch.
+# Used both at normalize time (wrapping flat authored style into the monolithic
+# shape) and at cascade/pipeline time (finding the family sub-patch for a
+# given chart type).
+_AUTHORED_FAMILY_TO_MONOLITHIC_KEY: dict[str, str] = {
+    "bar": "bar",
+    "histogram": "bar",
+    "line": "line",
+    "area": "area",
+    "scatter": "scatter",
+    "pie": "pie",
+    "donut": "pie",
+    "kpi": "kpi",
+    "table": "table",
+    "spark_bar": "spark_bar",
+    "heatmap": "heatmap",
+    "map": "geoshape",
+    "geoshape": "geoshape",
+    "point_map": "point_map",
+    "callout": "callout",
+    "layered": "layered",
+}
+
+
+def _wrap_authored_style_for_compiled(
+    chart_type: str | None, authored_style: dict[str, Any]
+) -> dict[str, Any]:
+    """Re-wrap a flat per-family authored style dict into the monolithic shape.
+
+    After the authored-side move to per-family styles, model_dump() produces
+    flat dicts (e.g. PieChartStylePatch → {"inner_radius": 0.3}). The compiled
+    Chart.style expects the monolithic ChartStylePatch shape with family sub-keys
+    (e.g. {"pie": {"inner_radius": 0.3}}). Layered charts use a flat
+    LayeredChartStyle that wraps to {"layered": {...}}.
+    Families without a monolithic slot (heatmap, geo) also pass through; any
+    family-specific fields they contain will be unknown to ChartStylePatch, but
+    those families have no authored chart-style extensions yet.
+    """
+    key = _AUTHORED_FAMILY_TO_MONOLITHIC_KEY.get(chart_type or "")
+    if key is None:
+        return authored_style
+    return {key: authored_style}
+
+
+def _kpi_subtitle_error(chart_id: str | None) -> str:
+    where = f"KPI chart '{chart_id}' " if chart_id else "KPI charts "
+    return (
+        f"{where}uses `subtitle:` which is no longer part of the KPI "
+        "surface. Use the structured `support: {value, label}` block instead."
+    )
+
+
+def _collect_charts_from_layout(
+    layout: Layout,
+    charts: dict[str, Chart],
+) -> None:
+    """Collect all inline charts from layout into the charts dict.
+
+    This normalizes the data model so ALL charts end up in face.charts,
+    making chart lookup a simple dict access (face.charts[chart_id])
+    instead of requiring layout traversal.
+
+    Args:
+        layout: The layout to extract charts from
+        charts: Dict to add charts to (modified in place)
+    """
+    for item in layout.items:
+        if item.type == "chart" and item.chart:
+            # Add inline chart to charts dict if not already there
+            chart_id = item.chart.id
+            if chart_id and chart_id not in charts:
+                charts[chart_id] = item.chart
+
+        elif item.type == "face" and item.face:
+            # Recurse into nested faces
+            _collect_charts_from_layout(item.face.layout, charts)
+            # Also collect charts from nested face's charts dict
+            if item.face.charts:
+                for chart_id, chart in item.face.charts.items():
+                    if chart_id not in charts:
+                        charts[chart_id] = chart
+
+
+def _normalize_chart(
+    chart_id: str,
+    chart_def: Any,
+    query_registry: dict[str, AnyQuery],
+    base_path: Path | None = None,
+    default_source: str | None = None,
+    source_path: str | None = None,
+    custom_chart_types: CustomChartTypeRegistry | None = None,
+) -> Chart:
+    """Normalize a chart definition to Chart.
+
+    Supports external query references using the syntax:
+    `query: path/to/file.yml#query_name`
+
+    Args:
+        chart_id: Chart identifier
+        chart_def: Chart definition
+        query_registry: Query registry for resolution
+        base_path: Base path for resolving external file references
+        default_source: Default source to apply to inline queries
+        source_path: Path in YAML for edit-back support (e.g., "charts.revenue")
+
+    Returns:
+        Chart with resolved query
+
+    Raises:
+        ReferenceError: If chart references unknown query
+    """
+    if isinstance(chart_def, BaseModel):
+        # Covers all per-family *Patch classes (including CalloutChart which
+        # does not inherit _SharedChartFields but is still a BaseModel).
+        chart_dict = chart_def.model_dump(exclude_none=True)
+    elif isinstance(chart_def, dict):
+        chart_dict = dict(chart_def)
+    else:
+        raise CompilationError(f"Invalid chart definition: {chart_def}")
+
+    # Set ID
+    chart_dict["id"] = chart_id
+
+    # Auto-derive the chart-id slug into the appropriate slot. KPI charts
+    # use ``label`` (the text rendered above the headline value); every
+    # other chart type uses ``title`` (the section heading). Authoring an
+    # explicit empty value is honored so authors can opt out — only a
+    # missing key falls through to the slug.
+    raw_chart_type = chart_dict.get("type")
+    canonical_chart_type = (
+        CHART_TYPE_ALIASES.get(raw_chart_type, raw_chart_type)
+        if isinstance(raw_chart_type, str)
+        else raw_chart_type
+    )
+    if canonical_chart_type == "kpi":
+        if "label" not in chart_dict:
+            chart_dict["label"] = _slug_to_text(chart_id)
+    else:
+        if "title" not in chart_dict:
+            chart_dict["title"] = _slug_to_text(chart_id)
+
+    # Ensure description (might be None or missing)
+    if not chart_dict.get("description"):
+        chart_dict["description"] = ""
+    if not chart_dict.get("subtitle"):
+        chart_dict["subtitle"] = ""
+
+    # Rewrite chart type aliases to canonical forms
+    raw_type = chart_dict.get("type")
+    if raw_type in CHART_TYPE_ALIASES:
+        chart_dict["type"] = CHART_TYPE_ALIASES[raw_type]
+
+    # Set type to "auto" if not specified (enables auto-detection at render time)
+    if "type" not in chart_dict or chart_dict.get("type") is None:
+        chart_dict["type"] = "auto"
+
+    # Validate chart type against builtins + custom registry.
+    # Unknown types are always rejected.  When a custom_chart_types
+    # registry is provided, registered custom types are also accepted.
+    _chart_type = chart_dict.get("type")
+    if _chart_type and _chart_type not in BUILTIN_CHART_TYPE_VALUES:
+        if not (custom_chart_types and custom_chart_types.is_registered(_chart_type)):
+            valid = sorted(
+                (BUILTIN_CHART_TYPE_VALUES | CHART_TYPE_ALIASES.keys()) - {"auto"}
+            )
+            msg = (
+                f"Chart '{chart_id}' has unknown type '{_chart_type}'. "
+                f"Valid built-in types: {valid}"
+            )
+            if custom_chart_types and (
+                custom := sorted(custom_chart_types.registered_names())
+            ):
+                msg += f". Registered custom types: {custom}"
+            raise CompilationError(msg)
+
+    # Validate layered chart syntax
+    if _chart_type == "layered":
+        raw_layers = chart_dict.get("layers")
+        if not raw_layers:
+            raise CompilationError(
+                f"Chart '{chart_id}' has type 'layered' but no `layers` list. "
+                "Layered charts require at least one layer, each with a `type`."
+            )
+        validated_layers: list[Layer] = []
+        layer_aliases = {
+            k for k, v in CHART_TYPE_ALIASES.items() if v in VALID_LAYER_TYPES
+        }
+        for i, layer in enumerate(raw_layers):
+            if not isinstance(layer, dict):
+                raise CompilationError(
+                    f"Chart '{chart_id}', layer {i}: expected a dict, got {type(layer).__name__}"
+                )
+            layer_type = layer.get("type")
+            if not layer_type:
+                raise CompilationError(
+                    f"Chart '{chart_id}', layer {i}: each layer must specify a `type` "
+                    f"(e.g. bar, line, area). Valid layer types: {sorted(VALID_LAYER_TYPES | layer_aliases)}"
+                )
+            # Apply chart type aliases to layers too (point→scatter, etc.)
+            if layer_type in CHART_TYPE_ALIASES:
+                layer_type = CHART_TYPE_ALIASES[layer_type]
+                layer["type"] = layer_type
+            if layer_type == "layered":
+                raise CompilationError(
+                    f"Chart '{chart_id}', layer {i}: nested `type: layered` is not allowed. "
+                    "Use primitive mark types (bar, line, area, etc.) inside layers."
+                )
+            if layer_type not in VALID_LAYER_TYPES:
+                raise CompilationError(
+                    f"Chart '{chart_id}', layer {i}: invalid layer type '{layer_type}'. "
+                    f"Valid layer types: {sorted(VALID_LAYER_TYPES | layer_aliases)}"
+                )
+            # Validate per-layer query reference exists in query registry
+            layer_query = layer.get("query")
+            if layer_query is not None:
+                if not isinstance(layer_query, str):
+                    raise CompilationError(
+                        f"Chart '{chart_id}', layer {i}: layer `query` must be a "
+                        "string reference to a named query or inline SQL, got "
+                        f"{type(layer_query).__name__}"
+                    )
+                if layer_query not in query_registry:
+                    if looks_like_sql(layer_query):
+                        layer_inline_name = f"_inline_query_{chart_id}_layer{i}"
+                        query_registry[layer_inline_name] = normalize_query(
+                            layer_inline_name,
+                            {"sql": layer_query},
+                            default_source=default_source,
+                        )
+                        layer["query"] = layer_inline_name
+                    else:
+                        raise ReferenceError(
+                            layer_query, f"chart '{chart_id}', layer {i}"
+                        )
+            validated_layers.append(Layer(**layer))
+        chart_dict["layers"] = validated_layers
+
+    # Donut → pie alias: donut is just a pie with style.inner_radius > 0.
+    # After the per-family authored style move, PieChartStylePatch.inner_radius
+    # is a flat field (no "pie" sub-key). Inject directly into the style dict.
+    # 0.6 = ~64% ring area / ~36% hole area, putting the data ring as the
+    # dominant element while leaving room for a center total — the editorial-
+    # donut sweet spot. Authors override via explicit style.inner_radius.
+    if chart_dict.get("type") == "donut":
+        chart_dict["type"] = "pie"
+        style = chart_dict.setdefault("style", {})
+        if style.get("inner_radius") is None:
+            style["inner_radius"] = 0.6
+
+    # Promote sizing sentinels from per-chart style to chart root.
+    # aspect_ratio, min_height, max_height are authored in style: but stored as
+    # flat fields on compiled Chart for sizing.py to read directly (no style walk).
+    # For single-family charts the fields live in the per-family patch (e.g.
+    # BarChartStylePatch); for layered charts they live at the top of
+    # ChartStylePatch. In both cases the raw dict has them at style[key] at
+    # this point, so the pop loop is uniform.
+    # kpi and table patch models exclude these fields, so they can never reach
+    # this step for those types; the promotion runs unconditionally for all others.
+    if isinstance(chart_dict.get("style"), dict):
+        flat_style = chart_dict["style"]
+        for _sizing_key in ("aspect_ratio", "min_height", "max_height"):
+            if _sizing_key in flat_style:
+                chart_dict[_sizing_key] = flat_style.pop(_sizing_key)
+
+    # Resolve query reference
+    query_value = chart_dict.get("query")
+    query_is_inline = False
+
+    # Convert the per-family flat authored style dict to the monolithic
+    # ChartStylePatch shape expected by compiled Chart.style, then validate.
+    if chart_dict.get("style") and isinstance(chart_dict["style"], dict):
+        monolithic = _wrap_authored_style_for_compiled(
+            chart_dict.get("type"), chart_dict["style"]
+        )
+        chart_dict["style"] = ChartStylePatch.model_validate(monolithic)
+
+    # Handle inline query definitions (query is a dict, not a string reference)
+    if query_value is None:
+        # Blank chart with no query - this is valid for placeholder charts
+        chart_dict["query"] = None
+        chart_dict["query_name"] = None
+        chart_dict["source_path"] = source_path
+        chart_dict["query_is_inline"] = False
+        return Chart(**chart_dict)
+    elif isinstance(query_value, dict):
+        # Create an inline query in the registry
+        inline_query_name = f"_inline_query_{chart_id}"
+        normalized_query = normalize_query(
+            inline_query_name, query_value, default_source=default_source
+        )
+        query_registry[inline_query_name] = normalized_query
+        query_ref = inline_query_name
+        query_is_inline = True
+    elif isinstance(query_value, str):
+        # String reference to existing query
+        query_ref = query_value
+        if query_ref.startswith("queries."):
+            query_ref = query_ref[8:]
+
+        # Check for external file reference (file.yml#query_name)
+        if "#" in query_ref:
+            query_ref, query_registry = resolve_external_query(
+                query_ref, query_registry, base_path
+            )
+
+        if query_ref not in query_registry:
+            if looks_like_sql(query_ref):
+                # Promote bare SQL string to an inline query (same as query: {sql: ...})
+                inline_query_name = f"_inline_query_{chart_id}"
+                normalized_query = normalize_query(
+                    inline_query_name, {"sql": query_ref}, default_source=default_source
+                )
+                query_registry[inline_query_name] = normalized_query
+                query_ref = inline_query_name
+                query_is_inline = True
+            else:
+                raise ReferenceError(query_ref, f"chart '{chart_id}'")
+    else:
+        raise CompilationError(
+            f"Chart '{chart_id}' has invalid query field. Expected string reference or dict, got {type(query_value)}"
+        )
+
+    # Set query fields
+    resolved_query = query_registry[query_ref]
+    chart_dict["query"] = resolved_query
+    chart_dict["query_name"] = query_ref
+
+    # Compute variable dependencies (union of own deps + query deps)
+    chart_deps: set[str] = set()
+
+    # Own deps from title/subtitle
+    if chart_dict.get("title"):
+        chart_deps |= extract_variable_dependencies(chart_dict["title"])
+    if chart_dict.get("subtitle"):
+        chart_deps |= extract_variable_dependencies(chart_dict["subtitle"])
+
+    # Inherit query deps
+    if resolved_query.variable_dependencies:
+        chart_deps |= resolved_query.variable_dependencies
+
+    chart_dict["variable_dependencies"] = chart_deps
+
+    # Track source location and query type for edit-back support
+    chart_dict["source_path"] = source_path
+    chart_dict["query_is_inline"] = query_is_inline
+
+    chart = Chart(**chart_dict)
+
+    # Extract filter deps from typed FilterDef instances — single source of shape truth.
+    # Done post-validation so we read .var/.template directly without re-parsing raw forms.
+    if chart.filters:
+        filter_deps: set[str] = set()
+        for fd in chart.filters.values():
+            if fd.var is not None:
+                filter_deps.add(fd.var)
+            elif fd.template is not None:
+                filter_deps |= extract_variable_dependencies(fd.template)
+        if filter_deps:
+            chart = chart.model_copy(
+                update={"variable_dependencies": chart_deps | filter_deps}
+            )
+
+    # Resolve palette tokens in conditional_formatting color fields.
+    # ConditionalRule.background and .font.color are authored color literals
+    # that live outside the style tree, so they don't go through resolve_style().
+    # Walk them here, at compile time, so the renderer receives hex values.
+    if chart.conditional_formatting:
+        from dataface.core.compile.models.style.merged import _resolve_color_tokens
+
+        resolved_cf = {
+            col: _resolve_color_tokens(entry)
+            for col, entry in chart.conditional_formatting.items()
+        }
+        chart = chart.model_copy(update={"conditional_formatting": resolved_cf})
+
+    return chart
+
+
+def _generate_inline_chart_id(chart_def: Any, fallback_id: str, used_ids: set) -> str:
+    """Generate a unique chart ID for an inline chart definition.
+
+    Uses the chart's authored display text (``label`` for KPI, ``title``
+    otherwise) if available; falls back to the provided ID. Handles
+    duplicates by appending a number.
+
+    Args:
+        chart_def: The chart definition dict
+        fallback_id: Fallback ID to use if no title (e.g., "row0")
+        used_ids: Set of already-used chart IDs (will be mutated)
+
+    Returns:
+        A unique chart ID
+    """
+
+    # KPI charts carry their authored display text in ``label`` (the slot
+    # rendered above the headline value); every other chart type uses
+    # ``title``. The auto-derive in _normalize_chart runs *after* this
+    # helper picks the inline ID, so we must read the right slot here too.
+    if isinstance(chart_def, dict):
+        raw_type = chart_def.get("type")
+        chart_type = (
+            CHART_TYPE_ALIASES.get(raw_type, raw_type)
+            if isinstance(raw_type, str)
+            else raw_type
+        )
+        title = (
+            chart_def.get("label") if chart_type == "kpi" else chart_def.get("title")
+        )
+    else:
+        chart_type = getattr(chart_def, "type", None)
+        if chart_type == "kpi":
+            title = getattr(chart_def, "label", None)
+        else:
+            title = getattr(chart_def, "title", None)
+
+    # Generate base slug
+    base_slug = _title_to_slug(title) if title else fallback_id
+
+    # Ensure uniqueness
+    if base_slug not in used_ids:
+        used_ids.add(base_slug)
+        return base_slug
+
+    # Append number to make unique
+    counter = 2
+    while f"{base_slug}-{counter}" in used_ids:
+        counter += 1
+    unique_id = f"{base_slug}-{counter}"
+    used_ids.add(unique_id)
+    return unique_id
+
+
+def _title_to_slug(title: str) -> str:
+    """Convert a title to a URL-friendly slug.
+
+    Args:
+        title: Display title (e.g., "My First Chart")
+
+    Returns:
+        Slug string (e.g., "my-first-chart")
+    """
+    # Convert to lowercase
+    slug = title.lower()
+    # Replace spaces and underscores with hyphens
+    slug = slug.replace(" ", "-").replace("_", "-")
+    # Remove any characters that aren't alphanumeric or hyphens
+    slug = re.sub(r"[^a-z0-9-]", "", slug)
+    # Collapse multiple hyphens
+    slug = re.sub(r"-+", "-", slug)
+    # Strip leading/trailing hyphens
+    slug = slug.strip("-")
+    return slug or "chart"