dataface 0.1.2__py3-none-any.whl

dataface/core/render/warnings/y_encoding_mostly_null.py

@@ -0,0 +1,76 @@
+"""Detector: Y_ENCODING_MOSTLY_NULL.
+
+Fires on any chart where the y-encoding field is >50% NULL in the query
+result rows. Mostly-empty visual marks with no explanation usually indicate
+a broken join or a nullable source column.
+
+Detection rule:
+  NULL count / total rows > 0.5  (strictly greater-than)
+
+Multi-y charts: the rule is evaluated per column; one warning is emitted per
+offending column so the consumer can identify exactly which series is broken.
+
+Missing-key semantics: a row that does not contain the y column key at all is
+counted as NULL. This matches the rendering behavior — Vega-Lite treats missing
+values the same as explicit null.
+
+NULL-only (not NULL+zero): zero is a valid measurement (e.g. count of refunds).
+"""
+
+from __future__ import annotations
+
+from dataface.core.render.warnings.base import RenderWarning, WarningContext
+
+CODE = "Y_ENCODING_MOSTLY_NULL"
+
+# Strictly-greater-than threshold: null_count / row_count must exceed this.
+_NULL_FRACTION_THRESHOLD = 0.5
+
+
+def detect(ctx: WarningContext) -> list[RenderWarning]:
+    """Return one RenderWarning per y field that is >50% NULL across result rows."""
+    warnings: list[RenderWarning] = []
+
+    for chart_id, chart in ctx.face_spec.charts.items():
+        # Skip charts with no y encoding (kpi, text, markdown, callout, etc.).
+        if not chart.y:
+            continue
+        # A chart absent from chart_results failed execution; nothing to analyse.
+        if chart_id not in ctx.chart_results:
+            continue
+
+        rows = ctx.chart_results[chart_id]
+        # Zero-row results are handled by the QUERY_RETURNED_ZERO_ROWS detector.
+        if not rows:
+            continue
+
+        # Normalise single-string y to a list so the loop below handles both cases.
+        # chart.y is str | list[str] here (None guarded above).
+        raw_y = chart.y
+        y_fields: list[str] = raw_y if isinstance(raw_y, list) else [raw_y]
+        total = len(rows)
+
+        for y_field in y_fields:
+            # Rows missing the key entirely are counted as NULL.
+            null_count = sum(1 for row in rows if row.get(y_field) is None)
+            if null_count / total > _NULL_FRACTION_THRESHOLD:
+                warnings.append(
+                    RenderWarning(
+                        code=CODE,
+                        chart=chart_id,
+                        field=y_field,
+                        message=(
+                            f"Y-encoding field '{y_field}' is NULL in "
+                            f"{null_count}/{total} rows ({null_count / total:.0%}). "
+                            "Mostly-empty visual encoding usually indicates a broken "
+                            "join or a nullable source column."
+                        ),
+                        fix=(
+                            "Check the join condition producing this field and verify "
+                            "source field nullability. Consider a COALESCE or filter "
+                            "to remove rows where the metric is undefined."
+                        ),
+                    )
+                )
+
+    return warnings

dataface/core/render/yaml_format.py

@@ -0,0 +1,167 @@
+"""YAML render output format — resolved dataface YAML.
+
+Walks the layout tree via face_to_dict (shared with JSON/text formats),
+then maps the resolved structure back to valid dataface YAML schema.
+The output can be re-compiled and rendered without a database connection.
+"""
+
+from collections.abc import Mapping
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+
+from dataface.core.compile.models.face.compiled import Face, VariableValues
+from dataface.core.errors import StructuredError
+from dataface.core.execute.executor import Executor
+from dataface.core.render.json_format import face_to_dict
+
+# Fields to emit on chart definitions (order matters for readability)
+_CHART_FIELDS = (
+    "type",
+    "query",
+    "title",
+    # ``label`` is the KPI-only label slot (sibling of ``value``); the
+    # chart-type-aware gate keeps it off other chart types.
+    "label",
+    "description",
+    "x",
+    "y",
+    "color",
+    "size",
+    "shape",
+    "theta",
+    "format",
+    "x_label",
+    "y_label",
+    "geo",
+    "geo_source",
+    "lookup",
+    "value",
+    # KPI quantitative-text-object fields (siblings of `value`).
+    "support",
+    "projection",
+    "latitude",
+    "longitude",
+    "basemap",
+    "sort",
+    "style",
+)
+
+
+def _clean_value(v: Any) -> Any:
+    """Convert non-YAML-native types to serializable equivalents."""
+    if isinstance(v, BaseModel):
+        return _clean_value(v.model_dump(exclude_none=True))
+    if isinstance(v, Mapping):
+        return {k: _clean_value(val) for k, val in v.items()}
+    if isinstance(v, list):
+        return [_clean_value(item) for item in v]
+    if isinstance(v, Decimal):
+        # Use int if lossless, else float
+        if v == v.to_integral_value():
+            return int(v)
+        return float(v)
+    if isinstance(v, datetime):
+        return v.isoformat()
+    if isinstance(v, date):
+        return v.isoformat()
+    if isinstance(v, set):
+        return sorted(v)
+    return v
+
+
+def _chart_to_yaml_dict(item: dict[str, Any], query_name: str) -> dict[str, Any]:
+    """Convert a resolved chart item to a dataface YAML chart definition."""
+    chart = item["chart"]
+    d: dict[str, Any] = {}
+
+    # Map chart_type → type
+    if chart.get("chart_type"):
+        d["type"] = chart["chart_type"]
+
+    # Reference the query by name
+    d["query"] = query_name
+
+    # Copy known chart fields. ``title`` and ``label`` default to ``""`` on
+    # Chart; an empty string would emit `title: ''` / `label: ''`
+    # next to the real authored slot, so treat empty strings as unauthored.
+    for field in _CHART_FIELDS:
+        if field in ("type", "query"):
+            continue  # already handled
+        val = chart.get(field)
+        if val is None:
+            continue
+        if field in ("title", "label") and val == "":
+            continue
+        d[field] = _clean_value(val)
+
+    return d
+
+
+def _face_to_yaml_dict(
+    face_dict: dict[str, Any],
+) -> dict[str, Any]:
+    """Convert a face_to_dict result into valid dataface YAML schema."""
+    result: dict[str, Any] = {}
+
+    if face_dict.get("title"):
+        result["title"] = face_dict["title"]
+
+    # Collect queries and charts from items
+    queries: dict[str, Any] = {}
+    charts: dict[str, Any] = {}
+    layout_refs: list[Any] = []
+
+    for item in face_dict.get("items", []):
+        if item["type"] == "chart":
+            chart = item["chart"]
+            chart_id = chart.get("id", "chart")
+            query_name = chart.get("query_name") or chart_id
+
+            # Query: inline data via rows
+            data = item.get("data", [])
+            queries[query_name] = {"type": "values", "rows": _clean_value(data)}
+
+            # Chart definition
+            charts[chart_id] = _chart_to_yaml_dict(item, query_name)
+            layout_refs.append(chart_id)
+
+        elif item["type"] == "face":
+            # Nested face — inline as a sub-face in layout
+            nested = _face_to_yaml_dict(item["face"])
+            layout_refs.append(nested)
+
+    if queries:
+        result["queries"] = queries
+    if charts:
+        result["charts"] = charts
+    if layout_refs:
+        result["rows"] = layout_refs
+
+    # Variables
+    if face_dict.get("variables"):
+        result["variables"] = _clean_value(face_dict["variables"])
+
+    return result
+
+
+def render_face_yaml(
+    face: Face,
+    executor: Executor,
+    variables: VariableValues,
+    error_collector: list[StructuredError] | None = None,
+) -> str:
+    """Render a compiled face to resolved dataface YAML.
+
+    Walks the layout tree, executes queries, resolves charts, and produces
+    valid dataface YAML where queries use `values:` with inline data rows.
+    The output can be fed back into `compile()` as valid input.
+    """
+    face_dict = face_to_dict(face, executor, variables, error_collector)
+    yaml_dict = _face_to_yaml_dict(face_dict)
+    return yaml.dump(
+        yaml_dict, default_flow_style=False, sort_keys=False, allow_unicode=True
+    )

dataface/core/resolve_face.py

@@ -0,0 +1,195 @@
+"""Resolution layer: Face + Config → MergedFace.
+
+Stage: RESOLVE (between COMPILE and RENDER)
+
+Architecture (ADR-008):
+  Compile → Resolve → Render (executor, data-aware)
+
+resolve_face() reads face.resolved_style and bakes it into the returned
+MergedFace. Call sync_face_resolved_style(face) before resolve_face()
+when the face has been mutated after compile. It does not access
+databases or pre-render charts.
+
+Import boundary:
+  Allowed: compile/* (types, config, sizing, style)
+  Allowed: render/chart/presentation.py for compile_effective_vega_config
+           (pure function — no executor, no data — lives in render/ only for
+            historical reasons)
+  Forbidden: execute/* (no Executor here)
+  Forbidden: render/layout_sizing.py (data-aware sizing is render-phase)
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from dataface.core.compile.models.face.compiled import (
+    Face,
+    MergedFace,
+    ResolvedLayout,
+    ResolvedLayoutItem,
+)
+
+if TYPE_CHECKING:
+    from dataface.core.compile.models.config import Config
+    from dataface.core.compile.models.face.compiled import Layout
+
+
+def _build_resolved_layout(layout: Layout, gap: float = 0.0) -> ResolvedLayout:
+    """Wrap a compile-time Layout in the frozen ResolvedLayout type.
+
+    Dimensions come from whatever static sizing the compile/normalizer phase
+    produced.  If a field is falsy (no static estimate), 0.0 is used as a
+    sentinel so the render phase knows to compute the real value.
+
+    Args:
+        layout: Compiled layout tree.
+        gap: Config-driven gap between items (baked from resolve_face).
+    """
+    items = tuple(
+        ResolvedLayoutItem(
+            type=item.type,
+            chart=item.chart,
+            face=_build_resolved_nested_face(item.face) if item.face else None,
+            x=item.x or 0.0,
+            y=item.y or 0.0,
+            width=item.width or 0.0,
+            height=item.height or 0.0,
+            details_variable=getattr(item, "details_variable", None),
+            details_summary=getattr(item, "details_summary", None),
+            details_expanded_summary=getattr(item, "details_expanded_summary", None),
+            description=getattr(item, "description", None),
+            visible=item.visible,
+        )
+        for item in (layout.items or [])
+    )
+    return ResolvedLayout(
+        type=layout.type,
+        items=items,
+        width=layout.width or 0.0,
+        height=layout.height or 0.0,
+        content_width=layout.content_width or 0.0,
+        content_height=layout.content_height or 0.0,
+        columns=getattr(layout, "columns", None),
+        gap=gap,
+        tab_titles=tuple(getattr(layout, "tab_titles", None) or ()),
+        tab_slugs=tuple(getattr(layout, "tab_slugs", None) or ()),
+        tab_variable=getattr(layout, "tab_variable", None),
+        default_tab=getattr(layout, "default_tab", None) or 0,
+        tab_position=getattr(layout, "tab_position", None),
+    )
+
+
+def resolve_nested_face(face: Face) -> MergedFace:
+    """Resolve a nested Face to MergedFace without global config.
+
+    For nested faces, board-level config (page_padding, card_padding, card_gap)
+    is None — those apply only to the root renderable face.
+    """
+    return _build_resolved_nested_face(face)
+
+
+def _build_resolved_nested_face(face: Face) -> MergedFace:
+    """Resolve a nested face using only static/compile-time information.
+
+    Board-level config (page_padding, card_padding, card_gap) are None —
+    those apply only to the root renderable face.
+    vega_config is compiled from face.theme so the render pass and sizing
+    pass use the same Vega-Lite config for nested charts.
+    Uses face.resolved_style as set by the compiler's propagation walk,
+    which correctly carries the parent→child semantic color cascade.
+    """
+    from dataface.core.compile.vega_config import compile_effective_vega_config
+
+    return MergedFace(
+        id=face.id,
+        title=face.title,
+        description=face.description,
+        tags=tuple(face.tags),
+        text=face.text,
+        level=face.level,
+        style=face.resolved_style,
+        page_padding=None,
+        card_padding=None,
+        card_gap=None,
+        width=face.layout.width or 0.0,
+        height=face.layout.height or 0.0,
+        vega_config=compile_effective_vega_config(face.theme),
+        layout=_build_resolved_layout(face.layout),
+        charts=face.charts,
+        variables=face.variables,
+        queries=face.queries,
+        variable_defaults=face.variable_defaults,
+    )
+
+
+def resolve_face(
+    face: Face,
+    config: Config,
+) -> MergedFace:
+    """Resolve a compiled face into a render-ready object.
+
+    Reads face.resolved_style as-is; callers are responsible for calling
+    sync_face_resolved_style(face) first if the face was mutated after
+    compile.  No database access.  No chart pre-rendering.  The render
+    phase applies data-aware sizing on top of the static estimates this
+    layer provides.
+
+    Board-level config fields (page_padding, card_padding, card_gap) are baked
+    for the root face.  Nested faces carry None for those fields — they are not
+    independently renderable and do not own board-level layout config.
+
+    Args:
+        face: Compiled face from the YAML → compile pipeline.
+        config: Global runtime config (provides board dimensions etc.).
+        variables: Active variable values (reserved for future static height
+            estimates that depend on variable-driven content; currently unused).
+
+    Returns:
+        MergedFace with baked config constants, merged style, resolved Vega
+        config, and static layout estimates.  Width/height are static estimates;
+        the render phase will refine them with data-aware sizing.
+    """
+    from dataface.core.compile.sizing import get_face_gap
+    from dataface.core.compile.vega_config import compile_effective_vega_config
+
+    # 1. Bake board config constants (root face only)
+    page_padding = float(config.style.board.margin)
+    card_padding = float(config.style.board.card_padding)
+    card_gap = float(config.style.board.card_gap) if face.card_gap else 0.0
+    layout_gap = get_face_gap(face)  # reads config internally; canonical implementation
+
+    # 2. Static dimension estimates from the compile-time layout tree.
+    #    face.layout.width/height are 0.0 when the normalizer has not yet run
+    #    calculate_layout_height.  Fall back to config defaults; render refines.
+    width = float(face.layout.width or config.style.board.width)
+    height = float(face.layout.height or config.style.board.default_height)
+
+    # 3. Resolve Vega-Lite config from theme name (pure, no data access)
+    vega_config = compile_effective_vega_config(face.theme)
+
+    # 4. Wrap the layout tree in the frozen ResolvedLayout type (with baked gap)
+    resolved_layout = _build_resolved_layout(face.layout, gap=layout_gap)
+
+    # 5. Use the face's resolved style. renderer.py calls sync_face_resolved_style
+    #    before this point so face.resolved_style is up-to-date.
+    return MergedFace(
+        id=face.id,
+        title=face.title,
+        description=face.description,
+        tags=tuple(face.tags),
+        text=face.text,
+        level=face.level,
+        style=face.resolved_style,
+        page_padding=page_padding,
+        card_padding=card_padding,
+        card_gap=card_gap,
+        width=width,
+        height=height,
+        vega_config=vega_config,
+        layout=resolved_layout,
+        charts=face.charts,
+        variables=face.variables,
+        queries=face.queries,
+        variable_defaults=face.variable_defaults,
+    )

dataface/core/schema/guidance.py

@@ -0,0 +1,151 @@
+"""Empty-table-list discoverability hints for the schema verb.
+
+Pure helpers — no agent-api or Pydantic coupling. Called by
+``dataface.agent_api.schema:schema()`` when a level-3 query returns no
+tables so the agent can understand why and what to try next.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Literal
+
+from dataface.core.execute.adapters import AdapterRegistry
+
+
+@dataclass
+class EmptyTableGuidance:
+    """Typed guidance payload for an empty level-3 table list."""
+
+    message: str
+    warnings: list[str] = field(default_factory=list)
+    hints: list[str] = field(default_factory=list)
+
+
+def empty_table_list_guidance(
+    registry: AdapterRegistry,
+    source: str | None,
+    *,
+    uses_direct_file_queries: Callable[[], bool],
+    surface: Literal["cli", "mcp"] = "mcp",
+) -> EmptyTableGuidance:
+    """Build discoverability guidance when no tables were found.
+
+    The schema verb errors on missing-source upstream, so ``source`` is always
+    a real string by the time callers reach this helper. ``uses_direct_file_queries``
+    is a callable so the (I/O-heavy) dashboard-walk is deferred until we
+    actually need its answer — it carries no dependency on ``agent_api`` here.
+
+    ``surface`` controls whether hints use CLI verbs (``dft inspect``,
+    ``dft search``, ``dft query``) or MCP tool names (``search_dashboards``,
+    ``execute_query``). Default is ``"mcp"`` so existing MCP callers need no
+    change.
+    """
+    configured_sources = registry.list_sql_sources()
+    selected_source = next(
+        (item for item in configured_sources if item["name"] == source),
+        None,
+    )
+    is_in_memory = False
+    # Connection path for the inspect hint — must be a DB path, not the source name.
+    # Absent when the source is in-memory (profiling :memory: across processes is useless).
+    connection_path: str | None = None
+    if selected_source is not None:
+        is_in_memory = bool(selected_source.get("in_memory"))
+        if not is_in_memory:
+            connection_path = selected_source.get("path")
+
+    warnings: list[str] = []
+    hints: list[str] = []
+    message = "No registered tables were found in the schema cache for this project."
+
+    if is_in_memory and selected_source is not None:
+        warnings.append(
+            f"Source '{selected_source['name']}' uses in-memory DuckDB, so schema() "
+            "can only see tables that have been created in the current session."
+        )
+        message = (
+            f"Source '{selected_source['name']}' is an in-memory DuckDB with no "
+            "registered tables loaded into the schema cache."
+        )
+
+    if uses_direct_file_queries():
+        warnings.append(
+            "This project appears to query files directly in dashboard SQL "
+            "(for example via read_csv(...)) rather than relying on persistent tables."
+        )
+        if surface == "cli":
+            hints.extend(
+                [
+                    "Use `dft search` to find existing dashboards with relevant SQL patterns.",
+                    "Use `dft query` with the discovered read_csv(...) pattern to inspect the underlying data directly.",
+                ]
+            )
+            if connection_path:
+                hints.append(
+                    f"Run `dft inspect --connection {connection_path}` to populate the schema cache."
+                )
+        else:
+            hints.extend(
+                [
+                    "Use search_dashboards to find existing dashboards with relevant SQL patterns.",
+                    "Use render_dashboard(path=<source_path>, format='text' or 'json') to inspect the dashboard you found.",
+                    "Use execute_query with the discovered read_csv(...) pattern to inspect the underlying data directly.",
+                ]
+            )
+
+    if not hints:
+        if surface == "cli":
+            hints.append(
+                "Use `dft search` to inspect existing dashboards or `dft query` to probe data sources directly."
+            )
+            if connection_path:
+                hints.append(
+                    f"Run `dft inspect --connection {connection_path}` to populate the schema cache."
+                )
+        else:
+            hints.append(
+                "Use search_dashboards to inspect existing dashboards or execute_query "
+                "to probe data sources directly."
+            )
+
+    return EmptyTableGuidance(message=message, warnings=warnings, hints=hints)
+
+
+def attach_empty_table_list_hints(
+    response: object,
+    registry: AdapterRegistry,
+    source: str,
+    *,
+    uses_direct_file_queries: Callable[[], bool],
+    surface: Literal["cli", "mcp"] = "mcp",
+) -> None:
+    """Attach level-3 discoverability hints to response when no tables matched.
+
+    ``response`` must expose mutable ``.message``, ``.warnings``, and
+    ``.hints`` attributes (the ``SchemaResponse`` Pydantic model). Typed
+    loosely so ``core/`` carries no dependency on ``agent_api`` Pydantic models.
+    ``uses_direct_file_queries`` is a callable so the (I/O-heavy) dashboard-walk
+    is skipped on the common path where the response already has tables.
+    ``surface`` controls CLI vs MCP verb vocabulary in the hints.
+    """
+    source_branch = getattr(response, "sources", {}).get(source, {})
+    schemas_branch = source_branch.get("schemas", {}) or {}
+    any_tables = any(
+        (s_entry.get("tables") or {}) for s_entry in schemas_branch.values()
+    )
+    if any_tables:
+        return
+
+    guidance = empty_table_list_guidance(
+        registry,
+        source,
+        uses_direct_file_queries=uses_direct_file_queries,
+        surface=surface,
+    )
+    response.message = guidance.message  # type: ignore[attr-defined]
+    if guidance.warnings:
+        response.warnings = guidance.warnings  # type: ignore[attr-defined]
+    if guidance.hints:
+        response.hints = guidance.hints  # type: ignore[attr-defined]

dataface/core/scoped_paths.py

@@ -0,0 +1,59 @@
+"""Path-scoping helpers used by the dashboard render orchestrator.
+
+These resolve user-supplied face paths against a Dataface project root, with a
+containment check so absolute paths cannot escape the project. They sit on top
+of `core.project_roots` discovery and have no agent_api-shape concerns, so they
+live in core where both `core.render.dashboard` and the agent_api verbs can
+reach them without inverting the layer stack.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from dataface.core.project_roots import discover_project_root
+
+
+def project_root_for(project_dir: Path | None) -> Path:
+    """Resolve project root: explicit project_dir or walk up from cwd."""
+    if project_dir is not None:
+        return project_dir.resolve()
+    return discover_project_root()
+
+
+def _resolve_against_project(path: Path, project_dir: Path | None) -> Path:
+    """Resolve a non-absolute path: '..' from cwd, bare from project root."""
+    if ".." in path.parts:
+        return (Path.cwd() / path).resolve()
+    return (project_root_for(project_dir) / path).resolve()
+
+
+def resolve_scoped_path(path: Path, project_dir: Path | None = None) -> Path:
+    """Resolve a path against a scoped base directory.
+
+    Absolute paths must remain inside `project_dir` when one is provided;
+    otherwise they are resolved as-is. When `project_dir` is set, the resolved
+    path must remain inside it (containment check).
+
+    Bare relative paths resolve from the project root (walking up from cwd when
+    no project_dir is given). Paths containing '..' resolve from cwd so shell
+    navigation feels natural.
+    """
+    raw_path = path
+    if raw_path.is_absolute():
+        if project_dir is None:
+            return raw_path.resolve()
+        resolved = raw_path.resolve()
+        try:
+            resolved.relative_to(project_dir.resolve())
+        except ValueError as exc:
+            raise ValueError(f"Absolute path escapes project_dir: {path}") from exc
+        return resolved
+
+    resolved_base = project_root_for(project_dir)
+    resolved = _resolve_against_project(raw_path, project_dir)
+    try:
+        resolved.relative_to(resolved_base)
+    except ValueError as exc:
+        raise ValueError(f"Path escapes base directory: {path}") from exc
+    return resolved

dataface/core/serve/__init__.py

@@ -0,0 +1,14 @@
+"""HTTP server for Dataface.
+
+Stage: SERVE
+Purpose: Provide HTTP endpoints for dashboard rendering.
+
+Routes:
+- /health - Health check
+- /inspect/{template}/?model=X - Inspect templates
+- /{path}/?var=value - Any face file (path.yml → /path/)
+"""
+
+from dataface.core.serve.server import create_server
+
+__all__ = ["create_server"]