dataface 0.1.2__py3-none-any.whl

dataface/ai/mcp/server.py

@@ -0,0 +1,289 @@
+"""MCP Server — thin shim over dataface.agent_api.
+
+Resources: dataface://dashboards, dataface://dashboard/{path},
+           dataface://docs/all, dataface://docs/{topic}, dataface://guide/*
+
+Tools: validate_dashboard, render_dashboard, execute_query, describe_query,
+       schema, schema_search, search_dashboards, query_face, docs,
+       describe_dashboard, list_skills, get_skill
+
+Usage: dft mcp serve
+"""
+
+import asyncio
+import json
+import logging
+from pathlib import Path
+from typing import Any, cast
+
+from pydantic import AnyUrl, TypeAdapter
+
+from dataface._install_hint import install_hint
+from dataface.ai.context import DatafaceAIContext
+from dataface.ai.tool_schemas import (
+    DESCRIBE_DASHBOARD,
+    DESCRIBE_QUERY,
+    DOCS,
+    EXECUTE_QUERY,
+    GET_SKILL,
+    LIST_SKILLS,
+    QUERY_FACE,
+    RENDER_DASHBOARD,
+    SCHEMA,
+    SCHEMA_SEARCH,
+    SEARCH_DASHBOARDS,
+    VALIDATE_DASHBOARD,
+)
+from dataface.ai.tools import dispatch_tool_call
+from dataface.core.execute.adapters import build_adapter_registry
+
+logger = logging.getLogger(__name__)
+
+_BASE_RESOURCES = [
+    (
+        "dataface://dashboards",
+        "application/json",
+        "Dashboard List",
+        "List of all dashboards in the project with metadata",
+    ),
+    (
+        "dataface://docs/all",
+        "text/markdown",
+        "Dataface YAML Reference (full)",
+        "Complete YAML syntax reference — same content as `dft docs all`",
+    ),
+    (
+        "dataface://docs/reference",
+        "text/markdown",
+        "Dataface YAML Field Reference (generated)",
+        "Auto-generated field-level spec for every YAML field. Run `just gen-yaml-reference` to refresh.",
+    ),
+    (
+        "dataface://guide/dashboard-design",
+        "text/markdown",
+        "Dashboard Design Guide",
+        "Design principles for creating effective dashboards. Read this when creating a new dashboard, choosing chart types, or designing layouts for at-a-glance monitoring.",
+    ),
+    (
+        "dataface://guide/report-design",
+        "text/markdown",
+        "Report Design Guide",
+        "Design principles for creating data-driven reports. Read this when creating narrative analyses, answering specific business questions, or writing investigative reports.",
+    ),
+    (
+        "dataface://guide/dashboard-build",
+        "text/markdown",
+        "Dashboard Build Workflow",
+        "Best practices for building dashboards incrementally. Read this before starting work — covers the recommended build-test-iterate workflow, caching, and tool usage patterns.",
+    ),
+    (
+        "dataface://guide/dashboard-review",
+        "text/markdown",
+        "Dashboard Review",
+        "Primary review skill for a Dataface face — runs structural review (YAML + dft validate) and visual review (PNG + vision-model evaluation), synthesizes a ranked findings list.",
+    ),
+]
+
+
+def _docs_topic_resources() -> list[tuple[str, str, str, str]]:
+    """One static resource per H2 in DATAFACE_SYNTAX.md, derived from the live topic index."""
+    from dataface.agent_api.docs import docs as docs_verb
+
+    return [
+        (
+            f"dataface://docs/{entry.id}",
+            "text/markdown",
+            f"Dataface YAML Reference — {entry.title}",
+            entry.description or f"`dft docs {entry.id}` section",
+        )
+        for entry in docs_verb().topics
+    ]
+
+
+def _read_resource_content(uri: str, context: DatafaceAIContext) -> str:
+    from dataface.agent_api.dashboards import get_dashboard, list_dashboards
+    from dataface.agent_api.docs import docs as docs_verb, read_full_text
+    from dataface.ai.prompts import load_shared_prompt
+
+    if uri == "dataface://dashboards":
+        return list_dashboards(
+            directory=context.dashboards_directory or Path("."),
+            recursive=True,
+        ).model_dump_json(indent=2, exclude_none=True)
+
+    if uri == "dataface://docs/all":
+        return read_full_text()
+
+    if uri.startswith("dataface://docs/"):
+        slug = uri.replace("dataface://docs/", "")
+        result = docs_verb(topic=slug)
+        if result.topic is None:
+            return json.dumps({"error": f"Unknown topic: {slug}"})
+        return result.topic.content
+
+    if uri.startswith("dataface://guide/"):
+        slug = uri.removeprefix("dataface://guide/")
+        return load_shared_prompt(slug) or json.dumps(
+            {"error": f"Unknown resource: {uri}"}
+        )
+
+    if uri.startswith("dataface://dashboard/"):
+        path = uri.replace("dataface://dashboard/", "")
+        try:
+            resolved_path = context.resolve_dashboard_path(Path(path))
+        except ValueError as exc:
+            return json.dumps({"error": str(exc)})
+        return get_dashboard(path=resolved_path, include_raw=True).model_dump_json(
+            indent=2, exclude_none=True
+        )
+
+    return json.dumps({"error": f"Unknown resource: {uri}"})
+
+
+def _is_domain_error(result: dict[str, Any]) -> bool:
+    # Two failure shapes flow through dispatch: handlers emit
+    # {"success": false, ...}; dispatch fallbacks emit {"error": str}.
+    # The "success is not True" guard keeps a future {"success": True,
+    # "error": "warning..."} happy path from being misclassified.
+    if result.get("success") is False:
+        return True
+    if result.get("success") is True:
+        return False
+    err = result.get("error")
+    return isinstance(err, str) and bool(err)
+
+
+def create_server(context: DatafaceAIContext) -> Any:
+    try:
+        from mcp.server import Server
+        from mcp.types import (
+            CallToolResult,
+            ContentBlock,
+            Resource,
+            ResourceTemplate,
+            TextContent,
+            Tool,
+        )
+    except ImportError as e:
+        raise ImportError(
+            "MCP server requires the 'mcp' package. "
+            f"Install with: {install_hint('mcp')}"
+        ) from e
+
+    server = Server("dataface")
+
+    def _uri(value: str) -> AnyUrl:
+        return TypeAdapter(AnyUrl).validate_python(value)
+
+    @server.list_resources()
+    async def handle_list_resources() -> list[Resource]:
+        return [
+            Resource(uri=_uri(u), mimeType=m, name=n, description=d)
+            for u, m, n, d in (*_BASE_RESOURCES, *_docs_topic_resources())
+        ]
+
+    @server.list_resource_templates()
+    async def handle_list_resource_templates() -> list[ResourceTemplate]:
+        return [
+            ResourceTemplate(
+                uriTemplate="dataface://dashboard/{path}",
+                name="Dashboard Content",
+                description="YAML content and compiled structure of a specific dashboard",
+                mimeType="application/json",
+            ),
+            ResourceTemplate(
+                uriTemplate="dataface://docs/{topic}",
+                name="Dataface YAML Reference Section",
+                description="One H2 section of DATAFACE_SYNTAX.md (e.g. face, charts, cheatsheet, all)",
+                mimeType="text/markdown",
+            ),
+        ]
+
+    @server.read_resource()
+    async def handle_read_resource(uri: str) -> str:
+        return _read_resource_content(str(uri), context=context)
+
+    @server.list_tools()
+    async def handle_list_tools() -> list[Tool]:
+        return [
+            Tool(
+                name=cast(str, t["name"]),
+                description=cast(str | None, t["description"]),
+                inputSchema=cast(dict[str, Any], t["input_schema"]),
+            )
+            for t in [
+                VALIDATE_DASHBOARD,
+                RENDER_DASHBOARD,
+                EXECUTE_QUERY,
+                DESCRIBE_QUERY,
+                SCHEMA,
+                SCHEMA_SEARCH,
+                SEARCH_DASHBOARDS,
+                QUERY_FACE,
+                DOCS,
+                DESCRIBE_DASHBOARD,
+                LIST_SKILLS,
+                GET_SKILL,
+            ]
+        ]
+
+    @server.call_tool()
+    async def handle_call_tool(
+        name: str, arguments: dict[str, Any] | None
+    ) -> CallToolResult:
+        arguments = arguments or {}
+        result = dispatch_tool_call(name, arguments, context=context)
+        content: list[ContentBlock] = [
+            TextContent(type="text", text=json.dumps(result, indent=2))
+        ]
+        return CallToolResult(content=content, isError=_is_domain_error(result))
+
+    return server
+
+
+async def run_server() -> None:
+    """Run the MCP server in stdio mode with embedded HTTP server."""
+    try:
+        from mcp.server.stdio import stdio_server
+    except ImportError as e:
+        raise ImportError(
+            "MCP server requires the 'mcp' package. "
+            f"Install with: {install_hint('mcp')}"
+        ) from e
+
+    from dataface.core.execute.adapters import LOCAL_AUTHORING_REGISTRY_KWARGS
+    from dataface.core.serve.embedded import build_embedded_server
+
+    http_server, resolved_http_port = build_embedded_server()
+    server = create_server(
+        context=DatafaceAIContext(
+            adapter_registry=build_adapter_registry(
+                Path.cwd(),
+                **LOCAL_AUTHORING_REGISTRY_KWARGS,
+            ),
+            server_port=resolved_http_port,
+        )
+    )
+
+    logger.info("Starting Dataface MCP server (stdio mode)")
+    http_task = asyncio.create_task(http_server.serve())
+
+    try:
+        async with stdio_server() as (read_stream, write_stream):
+            await server.run(
+                read_stream,
+                write_stream,
+                server.create_initialization_options(),
+            )
+    finally:
+        http_server.should_exit = True
+        await http_task
+
+
+def main() -> None:
+    """Entry point for running the MCP server."""
+    asyncio.run(run_server())
+
+
+if __name__ == "__main__":
+    main()

dataface/ai/memories.py

@@ -0,0 +1,85 @@
+"""Project-scoped analyst memory loading and prompt formatting."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+DEFAULT_MEMORY_FILENAME = "memories.yml"
+MAX_PROMPT_MEMORIES = 20
+
+
+def load_memories(base_dir: Path | None = None) -> list[dict[str, Any]]:
+    root = base_dir if base_dir is not None else Path.cwd()
+    path = root / DEFAULT_MEMORY_FILENAME
+    if not path.exists():
+        return []
+
+    payload = yaml.safe_load(path.read_text()) or {}
+    rows = payload.get("memories") if isinstance(payload, dict) else None
+    if not isinstance(rows, list):
+        return []
+
+    memories: list[dict[str, Any]] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        status = str(row.get("status", "active")).strip().lower()
+        if status not in {"active", ""}:
+            continue
+        note = str(row.get("note", "")).strip()
+        if not note:
+            continue
+        memories.append(row)
+    memories.sort(
+        key=lambda row: (
+            str(row.get("updated", "")),
+            str(row.get("created", "")),
+            str(row.get("type", "")),
+        ),
+        reverse=True,
+    )
+    return memories
+
+
+def _scope_label(memory: dict[str, Any]) -> str:
+    table = str(memory.get("table", "")).strip()
+    column = str(memory.get("column", "")).strip()
+    tables = memory.get("tables")
+    if table and column:
+        return f"{table}.{column}"
+    if table:
+        return table
+    if isinstance(tables, list):
+        joined = ", ".join(str(item).strip() for item in tables if str(item).strip())
+        if joined:
+            return joined
+    scope = str(memory.get("scope", "")).strip()
+    return scope
+
+
+def format_memories_prompt(
+    memories: list[dict[str, Any]], *, limit: int = MAX_PROMPT_MEMORIES
+) -> str:
+    if not memories:
+        return ""
+
+    lines = ["## Learned Context"]
+    for memory in memories[:limit]:
+        memory_type = str(memory.get("type", "memory")).strip() or "memory"
+        label = _scope_label(memory)
+        note = str(memory.get("note", "")).strip()
+        parts = [memory_type]
+        if label:
+            parts.append(label)
+        prefix = " | ".join(parts)
+        source = str(memory.get("source", "")).strip()
+        confidence = str(memory.get("confidence", "")).strip()
+        meta = ", ".join(part for part in [source, confidence] if part)
+        if meta:
+            lines.append(f"- {prefix}: {note} [{meta}]")
+        else:
+            lines.append(f"- {prefix}: {note}")
+    return "\n".join(lines)

dataface/ai/prompts.py

@@ -0,0 +1,177 @@
+"""Prompt loading and building utilities for AI services.
+
+This module provides shared utilities for loading prompts from markdown files
+and building system prompts with context.
+
+Shared guides (design principles, workflow, etc.) live in dataface/ai/skills/
+as SKILL.md files — the single source of truth for all AI consumers.
+App-specific prompts (tool instructions, validation loops) live in each app's
+own prompts/ directory.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from dataface.agent_api.skill_render import SkillSurface, render_skill_body
+
+SKILLS_DIR = Path(__file__).parent / "skills"
+
+_FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL)
+
+
+def _strip_frontmatter(text: str) -> str:
+    """Strip YAML frontmatter (--- ... ---) from a skill file."""
+    return _FRONTMATTER_RE.sub("", text, count=1).lstrip("\n")
+
+
+def load_prompt(prompt_name: str, prompts_dir: Path) -> str:
+    """Load a prompt template from markdown file.
+
+    Args:
+        prompt_name: Name of prompt file (without .md extension)
+        prompts_dir: Directory containing prompt files
+
+    Returns:
+        Prompt content as string, empty string if not found
+
+    Example:
+        >>> prompt = load_prompt("yaml_generation", Path("/prompts"))
+    """
+    prompt_file = prompts_dir / f"{prompt_name}.md"
+    if prompt_file.exists():
+        return prompt_file.read_text()
+    return ""
+
+
+class PromptLoader:
+    """Prompt loader with a fixed prompts directory.
+
+    This class provides a convenient way to load prompts from a specific directory,
+    useful when you need to load multiple prompts from the same location.
+
+    Attributes:
+        prompts_dir: The directory containing prompt markdown files
+
+    Example:
+        >>> loader = PromptLoader(Path("/prompts"))
+        >>> yaml_prompt = loader.load("yaml_generation")
+        >>> dashboard_prompt = loader.load("dashboard_design")
+    """
+
+    def __init__(self, prompts_dir: Path) -> None:
+        """Initialize the prompt loader.
+
+        Args:
+            prompts_dir: Directory containing prompt markdown files
+        """
+        self.prompts_dir = prompts_dir
+
+    def load(self, prompt_name: str) -> str:
+        """Load a prompt template.
+
+        Args:
+            prompt_name: Name of prompt file (without .md extension)
+
+        Returns:
+            Prompt content as string, empty string if not found
+        """
+        return load_prompt(prompt_name, self.prompts_dir)
+
+    def load_required(self, prompt_name: str) -> str:
+        """Load a prompt template, raising error if not found.
+
+        Args:
+            prompt_name: Name of prompt file (without .md extension)
+
+        Returns:
+            Prompt content as string
+
+        Raises:
+            FileNotFoundError: If prompt file doesn't exist
+        """
+        prompt_file = self.prompts_dir / f"{prompt_name}.md"
+        if not prompt_file.exists():
+            raise FileNotFoundError(f"Prompt file not found: {prompt_file}")
+        return prompt_file.read_text()
+
+    def exists(self, prompt_name: str) -> bool:
+        """Check if a prompt file exists.
+
+        Args:
+            prompt_name: Name of prompt file (without .md extension)
+
+        Returns:
+            True if prompt file exists
+        """
+        prompt_file = self.prompts_dir / f"{prompt_name}.md"
+        return prompt_file.exists()
+
+
+def load_shared_prompt(skill_name: str, *, surface: SkillSurface = "mcp") -> str:
+    """Load a shared guide from dataface/ai/skills/, rendered for ``surface``.
+
+    Skills are the single source of truth for design guides and reference
+    materials, shared across MCP, playground, cloud, and any other consumer.
+    Both call sites today are MCP-vocab (the terminal agent's tool loop and
+    the MCP ``dataface://guide/<slug>`` resource handler), so the default
+    surface is ``"mcp"``.
+
+    Args:
+        skill_name: The skill directory name under dataface/ai/skills/
+            (kebab-case, matches the on-disk directory exactly — e.g.
+            "dashboard-build", "dashboard-design", "dashboard-review").
+        surface: Which surface's macro vocabulary to render. Default ``"mcp"``.
+
+    Returns:
+        Guide content as string (frontmatter stripped, surface macros
+        expanded), empty string if not found.
+    """
+    skill_file = SKILLS_DIR / skill_name / "SKILL.md"
+    if skill_file.exists():
+        body = _strip_frontmatter(skill_file.read_text())
+        return render_skill_body(body, surface=surface)
+    return ""
+
+
+def build_context_section(
+    database_context: str | None = None,
+    yaml_context: str | None = None,
+    chart_context: str | None = None,
+) -> str:
+    """Build a context section for system prompts.
+
+    This function creates a formatted context section that can be prepended
+    to system prompts. Database context is given highest priority.
+
+    Args:
+        database_context: Optional database schema information
+        yaml_context: Optional selected YAML code
+        chart_context: Optional chart information
+
+    Returns:
+        Formatted context string, empty if no context provided
+
+    Example:
+        >>> context = build_context_section(
+        ...     database_context="Tables: users, orders",
+        ...     yaml_context="title: My Dashboard"
+        ... )
+    """
+    context_parts = []
+
+    # Database context is critical and should be first
+    if database_context:
+        context_parts.append(f"## ⚠️ CRITICAL: Database Context\n\n{database_context}\n")
+
+    if yaml_context:
+        context_parts.append(f"## Current YAML Code\n\n```yaml\n{yaml_context}\n```\n")
+
+    if chart_context:
+        context_parts.append(f"## Chart Context\n\n{chart_context}\n")
+
+    if context_parts:
+        return "\n\n".join(context_parts)
+
+    return ""

dataface/ai/schema_context.py

@@ -0,0 +1,138 @@
+"""Schema context helper — dispatches to the private package when installed.
+
+When ``dataface-super-schema`` is installed, its enriched ``get_schema_context``
+is used (cache-first with warm-cache profiler data). Without the private
+package, a bare table list is produced from the dbt adapter + manifest.
+
+Import contracts: callers at ``dataface.ai.schema_context.get_schema_context``
+are stable. The private package supplies enrichment; the OSS fallback ensures
+basic functionality without it.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from dataface.core.execute.adapters import AdapterRegistry
+
+logger = logging.getLogger(__name__)
+
+_SUPER_SCHEMA_AVAILABLE = importlib.util.find_spec("dataface_super_schema") is not None
+
+
+def get_schema_context(
+    *,
+    adapter_registry: AdapterRegistry,
+    source: str | None = None,
+    cache_path: Path | None = None,
+) -> str:
+    """Build a schema context string for AI consumption.
+
+    Delegates to the private package's enriched version when installed.
+    Falls back to a bare dbt-only table list otherwise.
+    """
+    if _SUPER_SCHEMA_AVAILABLE:
+        from dataface_super_schema.ai_schema_context import (  # noqa: PLC0415
+            get_schema_context as _enriched,
+        )
+
+        return _enriched(
+            adapter_registry=adapter_registry,
+            source=source,
+            cache_path=cache_path,
+        )
+
+    # OSS fallback: dbt-only, no profiler cache.
+    return _oss_schema_context(
+        adapter_registry=adapter_registry,
+        source=source,
+    )
+
+
+def _oss_schema_context(
+    *,
+    adapter_registry: AdapterRegistry,
+    source: str | None = None,
+) -> str:
+    """Bare table list from the dbt adapter + manifest (no profiler cache)."""
+    from dataface.core.compile.errors import DatafaceError
+    from dataface.core.inspect.resolver import LayeredSchemaResolver
+
+    resolved_source = (
+        source if source is not None else _default_source_name(adapter_registry)
+    )
+    if resolved_source is None:
+        return "No data source configured."
+
+    source_cfg: dict[str, Any] | None = None
+    dialect = "unknown"
+    try:
+        source_cfg = adapter_registry.resolve_source_config(resolved_source)
+        dialect = source_cfg.get("type", "unknown")
+    except DatafaceError as e:
+        logger.warning("No live DB connection for schema context: %s", e)
+        return "No database connection available."
+
+    project_root = Path(adapter_registry.project_root or ".").resolve()
+    resolver = LayeredSchemaResolver(
+        cache=None,
+        adapter_registry=adapter_registry,
+        project_root=project_root,
+    )
+
+    try:
+        schemas_resp = resolver.list_schemas(source=resolved_source)
+    except Exception as e:  # noqa: BLE001 — context provider degrades gracefully
+        logger.warning("Failed to list schemas for context: %s", e)
+        return f"Error listing schemas: {e}"
+
+    schemas_branch = (
+        schemas_resp.get("sources", {}).get(resolved_source, {}).get("schemas", {})
+    )
+    lines: list[str] = [f"## Schema: {resolved_source} ({dialect})"]
+    for schema_name in schemas_branch:
+        try:
+            tables_resp = resolver.list_tables(
+                source=resolved_source, schema=schema_name
+            )
+        except Exception as e:  # noqa: BLE001 — per-schema best effort
+            logger.warning("list_tables failed for %s: %s", schema_name, e)
+            continue
+        tables_branch = (
+            tables_resp.get("sources", {})
+            .get(resolved_source, {})
+            .get("schemas", {})
+            .get(schema_name, {})
+            .get("tables", {})
+        )
+        for table_name, summary in tables_branch.items():
+            desc = summary.get("description", "")
+            row_count = summary.get("row_count")
+            meta_parts = []
+            if row_count is not None:
+                meta_parts.append(f"{row_count:,} rows")
+            if desc:
+                meta_parts.append(desc)
+            suffix = f" — {'; '.join(meta_parts)}" if meta_parts else ""
+            lines.append(f"- {schema_name}.{table_name}{suffix}")
+
+    if len(lines) == 1:
+        return "No tables found in the configured data source."
+
+    return "\n".join(lines)
+
+
+def _default_source_name(registry: AdapterRegistry) -> str | None:
+    sources = registry.list_sql_sources()
+    if not sources:
+        return None
+    if len(sources) > 1:
+        names = ", ".join(item["name"] for item in sources)
+        raise ValueError(
+            "source is required when multiple SQL sources are configured: " f"{names}"
+        )
+    return sources[0]["name"]