dataface 0.1.2__py3-none-any.whl

dataface/agent_api/skill_render.py

@@ -0,0 +1,90 @@
+"""Expand ``{{ s_<key> }}`` macros in wheel skill bodies.
+
+Skill markdown is authored once with ``{{ s_<key> }}`` tokens. On the MCP
+surface the ``mcp`` side of the alias renders; on the CLI surface the ``dft``
+side renders. The alias table lives in ``surface_aliases.yaml`` next to this
+module so every wheel skill and every surface points at the same source of
+truth.
+
+Skills that should never appear on one surface (e.g. ``dataface-mcp-setup`` is
+CLI-only) declare ``surfaces:`` in their frontmatter; the registry hides them
+from the wrong surface entirely. Skills shown on both surfaces use ``{{ s_key }}``
+wherever they reference a tool/command so the prose reads naturally to both
+audiences.
+
+Face-YAML template tokens such as ``{{ region }}`` (no ``s_`` prefix) pass
+through untouched — the regex only matches the ``s_`` prefix.
+"""
+
+from __future__ import annotations
+
+import re
+from functools import cache
+from pathlib import Path
+from typing import Literal
+
+import yaml
+
+SkillSurface = Literal["mcp", "cli"]
+
+_ALIASES_PATH = Path(__file__).resolve().parent / "surface_aliases.yaml"
+
+# Matches `{{ s_<key> }}`. Keys are lowercase ASCII letters, digits,
+# underscores; must start with a letter.  Plain `{{ variable }}` tokens
+# (face-YAML template tokens) do NOT match because they lack the `s_` prefix.
+_MACRO_RE = re.compile(r"\{\{\s*s_([a-z][a-z0-9_]*)\s*\}\}")
+
+
+@cache
+def _aliases() -> dict[str, dict[str, str]]:
+    """Load and validate ``surface_aliases.yaml`` once per process."""
+    raw = yaml.safe_load(_ALIASES_PATH.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise ValueError(f"{_ALIASES_PATH}: top level must be a mapping")
+    out: dict[str, dict[str, str]] = {}
+    for key, entry in raw.items():
+        if not isinstance(entry, dict):
+            raise ValueError(f"{_ALIASES_PATH}: entry {key!r} must be a mapping")
+        unknown = set(entry) - {"mcp", "dft"}
+        if unknown:
+            raise ValueError(
+                f"{_ALIASES_PATH}: entry {key!r} has unknown keys {sorted(unknown)!r}"
+            )
+        mcp = entry.get("mcp", "")
+        dft = entry.get("dft", "")
+        if not isinstance(mcp, str) or not isinstance(dft, str):
+            raise ValueError(
+                f"{_ALIASES_PATH}: entry {key!r} must have string `mcp` and `dft` values"
+            )
+        out[key] = {"mcp": mcp, "dft": dft}
+    return out
+
+
+class MissingSurfaceAlias(KeyError):
+    """Raised when a SKILL.md references a macro key that has no alias entry."""
+
+
+def render_skill_body(body: str, *, surface: SkillSurface) -> str:
+    """Return ``body`` with ``{{ s_key }}`` macros expanded for ``surface``.
+
+    Each ``{{ s_key }}`` resolves to the ``mcp`` or ``dft`` side of the alias
+    table based on the active surface. Unknown keys raise ``MissingSurfaceAlias``
+    so a typo fails the surface fan-out test rather than shipping an unexpanded
+    token into an agent's context window.
+
+    Face-YAML template tokens (``{{ variable }}`` without ``s_`` prefix) pass
+    through untouched.
+    """
+    aliases = _aliases()
+    side = "mcp" if surface == "mcp" else "dft"
+
+    def repl(match: re.Match[str]) -> str:
+        key = match.group(1)
+        if key not in aliases:
+            raise MissingSurfaceAlias(
+                f"surface_aliases.yaml has no entry for {key!r} "
+                f"(referenced as {{{{ s_{key} }}}})"
+            )
+        return aliases[key][side]
+
+    return _MACRO_RE.sub(repl, body)

dataface/agent_api/skills.py

@@ -0,0 +1,293 @@
+"""dataface.agent_api.skills — registry of agent recipes packaged with the wheel.
+
+A skill is a directory under ``dataface/ai/skills/<name>/`` containing a
+``SKILL.md`` file with frontmatter (``name``, ``description``, ``kind``, optional
+``metadata``, optional ``surfaces``) and an optional ``examples/`` directory of
+face YAML files exercised by the snapshot test in
+``dataface/tests/agent_api/test_skills.py``.
+
+``kind`` is ``"workflow"`` (end-to-end agent playbooks like ``dashboard-build``)
+or ``"pattern"`` (layout recipes like ``kpi-row``). Required: the loader
+rejects a SKILL.md that omits it so the genre stays self-documenting as new
+skills land.
+
+``surfaces`` is an optional list of ``"mcp"`` / ``"cli"`` declaring where the
+skill should be exposed. Default is both. ``dataface-mcp-setup`` is CLI-only
+because shipping setup instructions to an already-connected MCP agent is
+worse than useless.
+
+Skill bodies are authored once with ``{{ s_<key> }}`` macros
+(see ``skill_render.py`` and ``surface_aliases.yaml``); the registry
+renders them on the requested surface before returning.
+"""
+
+from __future__ import annotations
+
+from functools import cache, cached_property
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+from dataface.agent_api.skill_render import SkillSurface, render_skill_body
+
+_SKILLS_DIR = Path(__file__).resolve().parent.parent / "ai" / "skills"
+
+SkillKind = Literal["workflow", "pattern"]
+_VALID_KINDS: tuple[SkillKind, ...] = ("workflow", "pattern")
+_VALID_SURFACES: tuple[SkillSurface, ...] = ("mcp", "cli")
+_ALL_SURFACES: frozenset[SkillSurface] = frozenset(_VALID_SURFACES)
+
+
+class SkillExample(BaseModel):
+    path: Path
+    name: str  # filename without extension
+
+
+class SkillMetadata(BaseModel):
+    author: str | None = None
+
+
+class Skill(BaseModel):
+    name: str
+    description: str
+    kind: SkillKind
+    directory: Path
+    body: str
+    surfaces: frozenset[SkillSurface] = Field(
+        default=_ALL_SURFACES,
+        description="Surfaces (`mcp`, `cli`) on which this skill is exposed.",
+    )
+    rendered_for: SkillSurface | None = Field(
+        default=None,
+        description=(
+            "Surface this body was rendered for. ``None`` on the raw parsed "
+            "skill (macros still present); set by ``list_skills`` / "
+            "``get_skill`` / ``search_skills`` to the surface they rendered."
+        ),
+    )
+    examples: list[SkillExample] = Field(
+        default_factory=list, description="Example face YAML entries for this skill."
+    )
+    metadata: SkillMetadata = Field(
+        default_factory=SkillMetadata,
+        description="Parsed SKILL.md frontmatter metadata.",
+    )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def has_examples(self) -> bool:
+        return bool(self.examples)
+
+
+class SkillList(BaseModel):
+    success: bool = True
+    skills: list[Skill] = Field(
+        default_factory=list, description="Available skills parsed from disk."
+    )
+    errors: list[str] = Field(
+        default_factory=list, description="Errors encountered while loading skills."
+    )
+
+
+class SkillSearchHit(BaseModel):
+    name: str
+    description: str
+    kind: SkillKind
+    has_examples: bool
+    score: float
+
+
+class SkillSearchResult(BaseModel):
+    success: bool = True
+    query: str
+    hits: list[SkillSearchHit] = Field(default_factory=list)
+
+
+class SkillNotFound(Exception): ...
+
+
+class GetSkillArgs(BaseModel):
+    """Input to get_skill: fetch one skill by name."""
+
+    name: str = Field(..., description="Skill directory name (kebab-case)")
+
+
+class SearchSkillsArgs(BaseModel):
+    """Input to search_skills: substring search across names, descriptions, and bodies."""
+
+    query: str = Field(..., description="Substring query (case-insensitive)")
+    limit: int = Field(10, ge=1, le=25, description="Max hits to return")
+
+    model_config = ConfigDict(extra="forbid")
+
+
+def _parse_surfaces(value: object, skill_md: Path) -> frozenset[SkillSurface]:
+    """Validate the optional ``surfaces:`` frontmatter list."""
+    if value is None:
+        return _ALL_SURFACES
+    if not isinstance(value, list) or not value:
+        raise ValueError(
+            f"{skill_md}: `surfaces` must be a non-empty list (got {value!r})"
+        )
+    out: set[SkillSurface] = set()
+    for item in value:
+        if item not in _VALID_SURFACES:
+            raise ValueError(
+                f"{skill_md}: `surfaces` entry {item!r} not in {_VALID_SURFACES}"
+            )
+        out.add(item)
+    return frozenset(out)
+
+
+def _parse_skill_dir(directory: Path) -> Skill:
+    """Parse a single skill directory into a Skill. Raises ValueError on bad data."""
+    skill_md = directory / "SKILL.md"
+    raw = skill_md.read_text(encoding="utf-8")
+
+    # Split frontmatter: expect leading --- block
+    if not raw.startswith("---"):
+        raise ValueError(f"{skill_md}: missing frontmatter (no leading ---)")
+    parts = raw.split("---", 2)
+    if len(parts) < 3:
+        raise ValueError(f"{skill_md}: malformed frontmatter")
+    frontmatter = yaml.safe_load(parts[1]) or {}
+    body = parts[2].lstrip("\n")
+
+    name = frontmatter.get("name")
+    if not name:
+        raise ValueError(f"{skill_md}: frontmatter missing 'name'")
+    if name != directory.name:
+        raise ValueError(
+            f"{skill_md}: frontmatter name {name!r} does not match directory name {directory.name!r}"
+        )
+
+    description = frontmatter.get("description", "")
+    if not description:
+        raise ValueError(f"{skill_md}: frontmatter missing 'description'")
+
+    kind = frontmatter.get("kind")
+    if not kind:
+        raise ValueError(
+            f"{skill_md}: frontmatter missing 'kind' (must be one of {_VALID_KINDS})"
+        )
+    if kind not in _VALID_KINDS:
+        raise ValueError(
+            f"{skill_md}: invalid kind {kind!r}; must be one of {_VALID_KINDS}"
+        )
+
+    meta_raw = frontmatter.get("metadata") or {}
+    metadata = SkillMetadata(author=meta_raw.get("author"))
+
+    surfaces = _parse_surfaces(frontmatter.get("surfaces"), skill_md)
+
+    examples: list[SkillExample] = []
+    examples_dir = directory / "examples"
+    if examples_dir.is_dir():
+        for yml_path in sorted(examples_dir.glob("*.yml")):
+            examples.append(SkillExample(path=yml_path, name=yml_path.stem))
+
+    return Skill(
+        name=name,
+        description=description.strip(),
+        kind=kind,
+        directory=directory,
+        body=body,
+        surfaces=surfaces,
+        examples=examples,
+        metadata=metadata,
+    )
+
+
+def all_skill_names() -> frozenset[str]:
+    """Every skill directory name shipped in the wheel."""
+    return frozenset(_load_all())
+
+
+@cache
+def _load_all() -> dict[str, Skill]:
+    """Walk _SKILLS_DIR and parse every <name>/SKILL.md. Cached for process lifetime."""
+    skills: dict[str, Skill] = {}
+    for skill_md in sorted(_SKILLS_DIR.glob("*/SKILL.md")):
+        directory = skill_md.parent
+        skill = _parse_skill_dir(directory)
+        skills[skill.name] = skill
+    return skills
+
+
+def _render_for_surface(skill: Skill, surface: SkillSurface) -> Skill:
+    """Return a shallow copy of ``skill`` with body + ``rendered_for`` set."""
+    return skill.model_copy(
+        update={
+            "body": render_skill_body(skill.body, surface=surface),
+            "rendered_for": surface,
+        }
+    )
+
+
+def list_skills(*, surface: SkillSurface = "mcp") -> SkillList:
+    """Return every skill exposed on ``surface``, with bodies rendered for it."""
+    skills = [
+        _render_for_surface(s, surface)
+        for s in _load_all().values()
+        if surface in s.surfaces
+    ]
+    skills.sort(key=lambda s: s.name)
+    return SkillList(skills=skills)
+
+
+def get_skill(name: str, *, surface: SkillSurface = "mcp") -> Skill:
+    """Return one skill by directory name, rendered for ``surface``.
+
+    Raises ``SkillNotFound`` when the skill does not exist *or* is not exposed
+    on the requested surface (e.g. ``dataface-mcp-setup`` is CLI-only and is
+    invisible to MCP callers).
+    """
+    skills = _load_all()
+    skill = skills.get(name)
+    if skill is None or surface not in skill.surfaces:
+        raise SkillNotFound(
+            f"Unknown skill: {name!r}. Run `dft skills` to list available."
+        )
+    return _render_for_surface(skill, surface)
+
+
+def search_skills(
+    query: str, *, limit: int = 10, surface: SkillSurface = "mcp"
+) -> SkillSearchResult:
+    """Substring search across name (1.0), description (0.8), and rendered body (0.5).
+
+    Honors the per-skill ``surfaces`` frontmatter — a skill that isn't exposed
+    on ``surface`` is invisible to search on that surface. Body matches use the
+    surface-rendered body so macro expansion doesn't leak macro suffixes into
+    relevance ranking.
+    """
+    q = query.strip().lower()
+    if not q:
+        raise ValueError("query must be a non-empty string")
+
+    hits: list[SkillSearchHit] = []
+    for skill in _load_all().values():
+        if surface not in skill.surfaces:
+            continue
+        if q in skill.name.lower():
+            score = 1.0
+        elif q in skill.description.lower():
+            score = 0.8
+        elif q in render_skill_body(skill.body, surface=surface).lower():
+            score = 0.5
+        else:
+            continue
+        hits.append(
+            SkillSearchHit(
+                name=skill.name,
+                description=skill.description,
+                kind=skill.kind,
+                has_examples=skill.has_examples,
+                score=score,
+            )
+        )
+
+    hits.sort(key=lambda h: (-h.score, h.name))
+    return SkillSearchResult(query=query, hits=hits[:limit])

dataface/agent_api/surface_aliases.yaml

@@ -0,0 +1,128 @@
+# Surface aliases for skill macro expansion.
+#
+# Authoring convention:
+#
+#   {{ s_<key> }}  renders as the `mcp` side on MCP surface, `dft` side on CLI
+#
+# Common patterns inside a SKILL.md:
+#
+#   "Use `{{ s_validate_dashboard }}` after every edit."
+#     → MCP agent reads: `validate_dashboard`
+#     → CLI agent reads: `dft validate`
+#
+#   "{{ s_render_png_example }}"
+#     → Whole code block flips between surfaces.
+#
+# Each top-level key here maps to an `mcp:` and `dft:` value. Either side may
+# be an empty string when the content is one-sided.
+#
+# Add a new key here, then reference it as `{{ s_<key> }}` in any wheel
+# SKILL.md. Face-YAML template tokens (`{{ region }}` etc.) are not macros —
+# they have no `s_` prefix and pass through untouched. Test coverage lives in
+# `dataface/tests/agent_api/test_skill_render.py` and the surface fan-out test
+# in `dataface/tests/agent_api/test_skills.py`.
+
+# ---- Inline tool references (one-liners inside prose) ----------------------
+
+validate_dashboard:
+  mcp: "validate_dashboard"
+  dft: "dft validate"
+
+execute_query:
+  mcp: "execute_query"
+  dft: "dft query"
+
+render_dashboard:
+  mcp: "render_dashboard"
+  dft: "dft render"
+
+query_face:
+  mcp: "query_face"
+  dft: "dft query --face"
+
+describe_query:
+  mcp: "describe_query"
+  dft: "dft query --describe"
+
+schema:
+  mcp: "schema"
+  dft: "dft schema"
+
+schema_search:
+  mcp: "schema_search"
+  dft: "dft schema -s"
+
+# ---- Inline skill-pointer references --------------------------------------
+
+skill_build:
+  mcp: "get_skill('dashboard-build')"
+  dft: "dft skills dashboard-build"
+
+skill_design_dashboard:
+  mcp: "get_skill('dashboard-design')"
+  dft: "dft skills dashboard-design"
+
+skill_design_report:
+  mcp: "get_skill('report-design')"
+  dft: "dft skills report-design"
+
+skill_review:
+  mcp: "get_skill('dashboard-review')"
+  dft: "dft skills dashboard-review"
+
+skill_troubleshooting:
+  mcp: "get_skill('dataface-troubleshooting')"
+  dft: "dft skills dataface-troubleshooting"
+
+# ---- Sentence-level shared phrases (same intent, different vocabulary) ----
+
+render_to_verify:
+  mcp: "Call `render_dashboard` to see the visual result — queries are cached after first execution, so re-rendering is nearly instant."
+  dft: "Run `dft render <path>` to see the visual result — queries are cached after first execution, so re-rendering is nearly instant."
+
+explore_schema_first:
+  mcp: "Use `schema` to drill source → schema → table → column, and `schema_search` for keyword search across the corpus."
+  dft: "Use `dft schema` to drill source → schema → table → column, and `dft schema -s <kw>` for keyword search across the corpus."
+
+# ---- Discovery / footer blocks (whole-paragraph, both sides populated) ----
+
+docs_discovery:
+  mcp: |
+    YAML field reference lives behind the `docs` tool (overview),
+    `docs(topic=…)` (one section in depth), and `docs(search=…)` (full-text
+    search).
+  dft: |
+    YAML field reference is `dft docs` (overview), `dft docs <topic>` (one
+    section in depth), and `dft docs -s "<query>"` (full-text search).
+
+# ---- Code-block examples (whole fenced block, surface-specific) -----------
+
+render_png_example:
+  mcp: |
+    ```text
+    render_dashboard(path="faces/finance/revenue-overview.yml", format="png")
+    ```
+  dft: |
+    ```bash
+    dft render faces/finance/revenue-overview.yml --format png
+    ```
+
+validate_example:
+  mcp: |
+    ```text
+    validate_dashboard(path="faces/finance/revenue-overview.yml")
+    ```
+  dft: |
+    ```bash
+    dft validate faces/finance/revenue-overview.yml
+    ```
+
+# ---- Cross-skill pointers (single-line footers) ---------------------------
+
+yaml_reference_footer:
+  mcp: "Full YAML reference is the `docs` tool — call `docs` for the overview, `docs(topic=…)` for one section, `docs(search=…)` for full-text search."
+  dft: "Full YAML reference is `dft docs` — `dft docs` for the overview, `dft docs <topic>` for one section, `dft docs -s \"<query>\"` for full-text search."
+
+review_pointer:
+  mcp: "Run `get_skill('dashboard-review')` before declaring a face done."
+  dft: "Run `dft skills dashboard-review` before declaring a face done."

dataface/agent_api/validate.py

@@ -0,0 +1,175 @@
+"""Typed validate verb — fast YAML schema + cross-reference validation.
+
+No warehouse connection, no query execution.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from dataface.agent_api._paths import (
+    no_project_hint,
+    project_root_for,
+    resolve_scoped_path,
+)
+from dataface.core.compile.errors import DatafaceError
+from dataface.core.errors import DF_UNKNOWN_INTERNAL, StructuredError
+from dataface.core.render.warnings.base import RenderWarning
+
+
+class ValidateDashboardArgs(BaseModel):
+    """Validate a face YAML file without executing queries.
+
+    Runs YAML parse, Pydantic schema validation, and cross-reference checks
+    (chart → query resolution, variable resolution, partial-include expansion).
+    Fast — no warehouse connection, no query execution. Use this as the inner
+    edit-loop verb: edit, validate, fix, repeat. Prefer render_dashboard for a
+    full compile + execute validation.
+    """
+
+    path: Path = Field(
+        ..., description="Path to face YAML file (relative to project_dir or cwd)"
+    )
+    project_dir: Path | None = Field(
+        None, description="Project root for resolving relative paths"
+    )
+
+
+class ValidateResult(BaseModel):
+    success: bool
+    path: Path
+    errors: list[StructuredError] = Field(
+        default_factory=list, description="Validation errors found in the face file."
+    )
+    warnings: list[RenderWarning] = Field(
+        default_factory=list,
+        description=(
+            "Non-fatal warnings as RenderWarning objects with stable codes. "
+            "Each warning carries code, message, and an optional fix hint."
+        ),
+    )
+
+
+def validate_paths(
+    paths: list[Path] | None,
+    project_dir: Path | None = None,
+) -> list[ValidateResult]:
+    """Validate N face files / directories, or default ``faces/`` when empty.
+
+    Each argv path expands like the single-path verb did: a file validates
+    directly, a directory walks recursively (skipping ``_*.yml`` partials and
+    ejected inspect-template directories). Empty / ``None`` defaults to
+    ``<project_root>/faces``. Results are concatenated in argv order.
+    """
+    if not paths:
+        return _validate_one_path(None, project_dir=project_dir)
+    out: list[ValidateResult] = []
+    for p in paths:
+        out.extend(_validate_one_path(p, project_dir=project_dir))
+    return out
+
+
+def _validate_one_path(
+    path: Path | None,
+    project_dir: Path | None = None,
+) -> list[ValidateResult]:
+    """Per-argv expansion: ``None`` → faces/, file → [one], dir → walk."""
+    # WHY: dataface.core.inspect.manifest_utils triggers the inspect package
+    # __init__, which eagerly imports TableInspector + grain/quality/semantic
+    # detectors. Keep this lazy so `dft --help` doesn't pay that startup cost.
+    from dataface.core.inspect.manifest_utils import INSPECT_TEMPLATE_MANIFEST
+
+    raw_path = path if path is not None else project_root_for(project_dir) / "faces"
+
+    try:
+        resolved = resolve_scoped_path(raw_path, project_dir)
+    except ValueError as exc:
+        return [
+            ValidateResult(
+                success=False,
+                path=raw_path,
+                errors=[
+                    DatafaceError.from_code(
+                        DF_UNKNOWN_INTERNAL, message=str(exc)
+                    ).to_structured()
+                ],
+            )
+        ]
+
+    if not resolved.is_dir():
+        return [validate(resolved, project_dir=project_dir)]
+
+    template_dirs = {m.parent for m in resolved.glob(f"**/{INSPECT_TEMPLATE_MANIFEST}")}
+    yaml_files = sorted(
+        f
+        for f in (list(resolved.glob("**/*.yml")) + list(resolved.glob("**/*.yaml")))
+        if not f.name.startswith("_") and f.parent not in template_dirs
+    )
+    if not yaml_files:
+        return [
+            ValidateResult(
+                success=False,
+                path=resolved,
+                errors=[
+                    DatafaceError.from_code(
+                        DF_UNKNOWN_INTERNAL,
+                        message=f"No face files found in {resolved}",
+                    ).to_structured()
+                ],
+            )
+        ]
+    return [validate(f, project_dir=project_dir) for f in yaml_files]
+
+
+def validate(path: Path, project_dir: Path | None = None) -> ValidateResult:
+    """Fast YAML schema + cross-reference validation. No warehouse, no execute."""
+    from dataface.core.compile.compiler import compile_file
+
+    try:
+        resolved = resolve_scoped_path(path, project_dir)
+    except ValueError as exc:
+        return ValidateResult(
+            success=False,
+            path=path,
+            errors=[
+                DatafaceError.from_code(
+                    DF_UNKNOWN_INTERNAL, message=str(exc)
+                ).to_structured()
+            ],
+        )
+
+    if not resolved.exists():
+        hint = no_project_hint(project_dir)
+        return ValidateResult(
+            success=False,
+            path=path,
+            errors=[
+                DatafaceError.from_code(
+                    DF_UNKNOWN_INTERNAL,
+                    message=f"File not found: {resolved}{hint}",
+                ).to_structured()
+            ],
+        )
+
+    try:
+        result = compile_file(resolved)
+    except OSError as exc:
+        return ValidateResult(
+            success=False,
+            path=resolved,
+            errors=[
+                DatafaceError.from_code(
+                    DF_UNKNOWN_INTERNAL, message=str(exc)
+                ).to_structured(file=str(resolved))
+            ],
+        )
+
+    errors = list(result.errors)
+    return ValidateResult(
+        success=len(errors) == 0,
+        path=resolved,
+        errors=errors,
+        warnings=list(result.warnings),
+    )