dataface 0.1.2__py3-none-any.whl

dataface/agent_api/schema_search.py

@@ -0,0 +1,284 @@
+"""Typed schema_search verb — full-text + filter over the metadata corpus.
+
+Companion to ``dft schema``: the drill-down verb answers "I know what I'm
+looking for" (navigation), this verb answers "I have a property, find what
+matches" (find/filter). The two output shapes are intentionally different:
+``schema`` returns a hierarchical named-dict tree wrapped in ``sources``;
+``schema_search`` returns a flat list of fully-qualified hits.
+
+Provenance flows through unchanged from the resolver: a hit on a name that
+came from the adapter carries ``source='dbt_adapter'``; a hit on a
+description that came from the cache carries ``source='super_schema'``;
+a hit on a tag/test/owner from the manifest carries
+``source='dbt_manifest'``.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+
+from dataface.core.execute.adapters import AdapterRegistry
+from dataface.core.inspect.cache_factory import build_resolver
+from dataface.core.inspect.search import InvalidRegex, SchemaSearch, SchemaSearchHit
+
+
+class SchemaSearchArgs(BaseModel):
+    """Search the company's data vocabulary by keyword + structured filters.
+
+    Returns a flat list of fully-qualified hits across the metadata corpus:
+    schema/table/column names, descriptions (column + table), tags, listed
+    tests, and meta values. Use this when you want to find vocabulary or
+    cross-cutting matches ("which tables have a customer_id column?",
+    "find columns tagged pii", "find primary keys in analytics") — not for
+    drilling into a specific known table (use ``schema`` for that).
+
+    Keyword is a case-insensitive substring match by default; pass
+    ``regex=True`` for a Python regex. Scope narrows which text-bearing
+    fields are searched (default: all of name, description, tag, tests,
+    meta, owner). Structured filters (``role``, ``tag``, ``has_test``,
+    ``missing``, ``column_name``, ``table_name``, ``meta``, ``fk_to``) layer
+    as additional predicates.
+
+    Hits are returned in stable iteration order — no ranking, no top-N.
+    Each hit reports the path keys via ``location`` ("source.schema.table"
+    or "source.schema.table.column"), which text-bearing field matched
+    (``matched_field``), an excerpt (``snippet``), and the layer that
+    contributed the field (``source`` — "super_schema", "dbt_manifest",
+    or "dbt_adapter").
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    keyword: str = Field(
+        "",
+        description=(
+            "Substring to match (case-insensitive). Empty string disables "
+            "keyword matching — use with structured filters (role, tag, "
+            "has_test, missing, ...) to enumerate qualifying rows."
+        ),
+    )
+    scope: list[str] | None = Field(
+        None,
+        description=(
+            "Limit which text-bearing fields are searched. "
+            "Choices: name, description, tag, tests, meta, owner. "
+            "Default: all."
+        ),
+    )
+    regex: bool = Field(
+        False,
+        description="Treat keyword as a Python regex (case-insensitive).",
+    )
+    role: str | None = Field(
+        None,
+        description=(
+            "Filter columns by profiled role "
+            "(primary_key, foreign_key, dimension, measure, ...)."
+        ),
+    )
+    tag: str | None = Field(
+        None,
+        description="Filter to rows with this tag in their tags array.",
+    )
+    has_test: str | None = Field(
+        None,
+        description=(
+            "Filter columns whose tests include this name "
+            "(unique, not_null, accepted_values, relationships, ...)."
+        ),
+    )
+    missing: str | None = Field(
+        None,
+        description=(
+            "Emit synthetic hits for rows that LACK this field "
+            "(description, tag, owner, ...). matched_field on each hit is "
+            "'_missing:<field>'."
+        ),
+    )
+    meta: dict[str, str] | None = Field(
+        None,
+        description="Filter by exact-match meta key=value pairs.",
+    )
+    column_name: str | None = Field(
+        None,
+        description=(
+            "Filter to columns whose name matches this fnmatch glob "
+            "(e.g. *_id, customer_*)."
+        ),
+    )
+    table_name: str | None = Field(
+        None,
+        description=(
+            "Filter to tables whose name matches this fnmatch glob "
+            "(e.g. stg_*, fct_*)."
+        ),
+    )
+    fk_to: str | None = Field(
+        None,
+        description=(
+            "Filter to columns with an explicit relationships: test "
+            "pointing at this table. No naming-heuristic inference."
+        ),
+    )
+    fields: list[str] | None = Field(
+        None,
+        description=(
+            "Project each hit to a subset of fields "
+            "(location, matched_field, snippet, source, value, attrs)."
+        ),
+    )
+    limit: int | None = Field(
+        None,
+        description=(
+            "Truncate results to N hits. ``total`` and ``truncated`` "
+            "report the pre-truncation count."
+        ),
+    )
+
+
+class _SearchMeta(BaseModel):
+    """Single ``_meta`` footer per response."""
+
+    retrieved_at: str
+
+
+class SchemaSearchResult(BaseModel):
+    """Flat list of hits across the metadata corpus."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    success: bool = True
+    hits: list[SchemaSearchHit] = Field(
+        default_factory=list,
+        description="Matching schema entries, ranked by relevance.",
+    )
+    total: int = 0
+    truncated: bool = False
+    # Per-source schema-walk failures (unreachable adapter, etc.). Surfaced
+    # so callers can tell "no hits" apart from "search couldn't reach a source"
+    # — the silent-fallback antipattern AGENTS.md flags.
+    warnings: list[str] = Field(
+        default_factory=list,
+        description="Per-source warnings (e.g. unreachable adapter).",
+    )
+    meta: _SearchMeta | None = Field(
+        default=None, alias="_meta", description="Pagination and timing metadata."
+    )
+    errors: list[str] = Field(
+        default_factory=list, description="Errors encountered during search."
+    )
+
+    # Projection is applied at dump time — not part of the schema. CLI/MCP
+    # callers that pass `fields=[...]` get a lean wire shape; in-process
+    # consumers iterate the typed `hits` list with full fields.
+    _project_fields: list[str] | None = PrivateAttr(default=None)
+
+    def model_dump(self, **kwargs: Any) -> dict[str, Any]:
+        out = super().model_dump(**kwargs)
+        if self._project_fields is not None:
+            allowed = set(self._project_fields)
+            out["hits"] = [
+                {k: v for k, v in hit.items() if k in allowed} for hit in out["hits"]
+            ]
+        return out
+
+    def model_dump_json(self, **kwargs: Any) -> str:
+        # model_dump_json bypasses model_dump — re-route through dump+json.dumps
+        # so projection is honored. Force ``mode='json'`` so any future field
+        # carrying a non-JSON-native value (datetime, Decimal, Path) is
+        # converted by Pydantic, not crashed by stdlib json.dumps. ``indent``
+        # is a json.dumps-only kwarg; pop it out of the model_dump call.
+        import json as _json
+
+        indent = kwargs.pop("indent", None)
+        kwargs.setdefault("mode", "json")
+        return _json.dumps(self.model_dump(**kwargs), indent=indent)
+
+
+def schema_search(
+    keyword: str = "",
+    *,
+    scope: list[str] | None = None,
+    regex: bool = False,
+    role: str | None = None,
+    tag: str | None = None,
+    has_test: str | None = None,
+    missing: str | None = None,
+    meta: dict[str, str] | None = None,
+    column_name: str | None = None,
+    table_name: str | None = None,
+    fk_to: str | None = None,
+    fields: list[str] | None = None,
+    limit: int | None = None,
+    cache_path: Path | None = None,
+    adapter_registry: AdapterRegistry,
+) -> SchemaSearchResult:
+    """Full-text + filter search across the resolver's materialized schema search index."""
+    if (
+        not keyword
+        and missing is None
+        and not _any_filter(role, tag, has_test, meta, column_name, table_name, fk_to)
+    ):
+        return SchemaSearchResult(
+            success=False,
+            errors=[
+                "schema_search requires a keyword or at least one filter "
+                "(role, tag, has_test, missing, meta, column_name, table_name, fk_to)"
+            ],
+        )
+    try:
+        resolver = build_resolver(adapter_registry, cache_path)
+        searcher = SchemaSearch(resolver)
+        hits, warnings = searcher.search(
+            keyword,
+            scope=scope,
+            regex=regex,
+            role=role,
+            tag=tag,
+            has_test=has_test,
+            missing=missing,
+            meta=meta,
+            column_name=column_name,
+            table_name=table_name,
+            fk_to=fk_to,
+        )
+    except InvalidRegex as exc:
+        return SchemaSearchResult(success=False, errors=[str(exc)])
+    except Exception as exc:  # noqa: BLE001 — verb returns errors via response envelope
+        return SchemaSearchResult(success=False, errors=[str(exc)])
+
+    total = len(hits)
+    truncated = False
+    if limit is not None and total > limit:
+        hits = hits[:limit]
+        truncated = True
+
+    result = SchemaSearchResult(
+        hits=hits,
+        total=total,
+        truncated=truncated,
+        warnings=warnings,
+        _meta=_SearchMeta(retrieved_at=datetime.now(timezone.utc).isoformat()),
+    )
+    if fields is not None:
+        result._project_fields = list(fields)
+    return result
+
+
+def _any_filter(
+    role: str | None,
+    tag: str | None,
+    has_test: str | None,
+    meta: dict[str, str] | None,
+    column_name: str | None,
+    table_name: str | None,
+    fk_to: str | None,
+) -> bool:
+    return any(
+        f is not None
+        for f in (role, tag, has_test, meta, column_name, table_name, fk_to)
+    )

dataface/agent_api/search.py

@@ -0,0 +1,270 @@
+"""Typed search verb for the agent API."""
+
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+MAX_SEARCH_LIMIT = 50
+DEFAULT_SEARCH_LIMIT = 10
+
+# Maintainer paths that must not appear in agent-facing search summaries.
+_INTERNAL_REF_RE = re.compile(r"\s*;?\s*see\s+ai_notes/\S+", re.IGNORECASE)
+
+
+def _agent_safe_summary(description: str) -> str:
+    """Strip maintainer-only references from face descriptions."""
+    return _INTERNAL_REF_RE.sub("", description).strip()
+
+
+class SearchDashboardsArgs(BaseModel):
+    """Search existing dashboards by keyword. Returns ranked results with match scores and reasons. Use to discover relevant dashboards and reuse validated query patterns before creating new ones. The returned source_path can be passed to render_dashboard(path=source_path, ...) to inspect that dashboard. Results may also include sample SQL and literal file paths from matching queries; reuse those exact paths instead of inventing new file globs."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    query: str = Field(
+        ...,
+        description="Search query text (keywords to match against dashboard metadata)",
+    )
+    project_dir: Path | None = Field(
+        None,
+        description="Project directory to search (defaults to the dispatch context's dashboards directory, then CWD)",
+    )
+    tags: list[str] | None = Field(
+        None, description="Filter results to dashboards with ALL of these tags"
+    )
+    limit: int | None = Field(
+        None, description="Maximum results to return (default 10, max 50)"
+    )
+
+
+_index_cache: list[dict[str, Any]] | None = None
+_index_cache_dir: Path | None = None
+_index_cache_file_set: set[Path] | None = None
+
+
+class DashboardSearchHit(BaseModel):
+    title: str
+    summary: str
+    match_score: float
+    match_reasons: list[str]
+    source_path: Path
+    query_names: list[str]
+    chart_names: list[str]
+    sample_sql: str | None = None
+    file_paths: list[str]
+
+
+class SearchResult(BaseModel):
+    success: bool
+    errors: list[str]
+    results: list[DashboardSearchHit]
+
+
+def search_dashboards(
+    query: str,
+    project_dir: Path | None = None,
+    tags: list[str] | None = None,
+    limit: int = DEFAULT_SEARCH_LIMIT,
+) -> SearchResult:
+    """Search dashboards by keyword with ranked results."""
+    limit = max(0, min(limit, MAX_SEARCH_LIMIT))
+    query_stripped = query.strip()
+
+    if not query_stripped or limit == 0:
+        return SearchResult(success=True, errors=[], results=[])
+
+    query_tokens = list(_tokenize(query_stripped))
+    if not query_tokens:
+        return SearchResult(success=True, errors=[], results=[])
+
+    index = _get_index(project_dir or Path("."))
+
+    if tags:
+        required_tags = {t.lower() for t in tags}
+        index = [e for e in index if required_tags.issubset(set(e["tags"]))]
+
+    scored: list[tuple[float, Path, dict[str, Any], list[str]]] = []
+    for entry in index:
+        score, reasons = _score_entry(entry, query_tokens)
+        if score > 0:
+            scored.append((score, entry["source_path"], entry, reasons))
+
+    scored.sort(key=lambda x: (-x[0], str(x[1])))
+
+    results = []
+    for score, _path, entry, reasons in scored[:limit]:
+        results.append(
+            DashboardSearchHit(
+                title=entry["title"],
+                summary=entry["description"],
+                match_score=round(score, 2),
+                match_reasons=reasons,
+                source_path=entry["source_path"],
+                query_names=entry["query_names"],
+                chart_names=entry["chart_names"],
+                sample_sql=(
+                    entry["sql_snippets"][0] if entry["sql_snippets"] else None
+                ),
+                file_paths=_extract_file_paths(entry["sql_snippets"]),
+            )
+        )
+
+    return SearchResult(success=True, errors=[], results=results)
+
+
+def _build_index(directory: Path) -> list[dict[str, Any]]:
+    from dataface.agent_api.dashboards import list_dashboards
+
+    listing = list_dashboards(directory, recursive=True)
+    entries: list[dict[str, Any]] = []
+
+    for dash in listing.dashboards:
+        abs_path = dash.absolute_path
+        try:
+            content = yaml.safe_load(abs_path.read_text())
+        except (yaml.YAMLError, OSError):
+            continue
+
+        if not isinstance(content, dict):
+            continue
+
+        tags = content.get("tags", [])
+        queries = content.get("queries", {})
+        charts = content.get("charts", {})
+
+        sql_snippets: list[str] = []
+        query_names: list[str] = []
+        if isinstance(queries, dict):
+            for qname, qdef in queries.items():
+                query_names.append(qname)
+                if isinstance(qdef, dict) and qdef.get("sql"):
+                    sql_snippets.append(qdef["sql"])
+
+        chart_names: list[str] = []
+        if isinstance(charts, dict):
+            chart_names = list(charts.keys())
+
+        entries.append(
+            {
+                "source_path": dash.path,
+                "title": dash.title,
+                "description": _agent_safe_summary(dash.description),
+                "tags": [t.lower() for t in tags] if isinstance(tags, list) else [],
+                "query_names": query_names,
+                "chart_names": chart_names,
+                "sql_snippets": sql_snippets,
+                "mtime": abs_path.stat().st_mtime,
+            }
+        )
+
+    return entries
+
+
+def _get_index(directory: Path) -> list[dict[str, Any]]:
+    global _index_cache, _index_cache_dir, _index_cache_file_set
+
+    dir_path = directory.resolve()
+
+    if _index_cache is not None and _index_cache_dir == dir_path:
+        current_files = {
+            f.relative_to(dir_path)
+            for f in dir_path.glob("**/*.yml")
+            if not f.name.startswith("_")
+        }
+        current_files |= {
+            f.relative_to(dir_path)
+            for f in dir_path.glob("**/*.yaml")
+            if not f.name.startswith("_")
+        }
+
+        if current_files == _index_cache_file_set:
+            still_valid = True
+            for entry in _index_cache:
+                fpath = dir_path / entry["source_path"]
+                try:
+                    if fpath.stat().st_mtime != entry["mtime"]:
+                        still_valid = False
+                        break
+                except OSError:
+                    still_valid = False
+                    break
+            if still_valid:
+                return _index_cache
+
+    snapshot = {
+        f.relative_to(dir_path)
+        for f in dir_path.glob("**/*.yml")
+        if not f.name.startswith("_")
+    }
+    snapshot |= {
+        f.relative_to(dir_path)
+        for f in dir_path.glob("**/*.yaml")
+        if not f.name.startswith("_")
+    }
+
+    _index_cache = _build_index(directory)
+    _index_cache_dir = dir_path
+    _index_cache_file_set = snapshot
+    return _index_cache
+
+
+def _tokenize(text: str) -> set[str]:
+    return set(re.findall(r"[a-z0-9]+", text.lower()))
+
+
+def _extract_file_paths(sql_snippets: list[str]) -> list[str]:
+    paths: list[str] = []
+    pattern = re.compile(
+        r"""read_(?:csv|csv_auto|parquet|json(?:_auto)?)\(\s*['"]([^'"]+)['"]""",
+        re.IGNORECASE,
+    )
+    for snippet in sql_snippets:
+        for match in pattern.findall(snippet):
+            if match not in paths:
+                paths.append(match)
+    return paths
+
+
+def _score_entry(
+    entry: dict[str, Any], query_tokens: list[str]
+) -> tuple[float, list[str]]:
+    score = 0.0
+    reasons: list[str] = []
+
+    title_tokens = _tokenize(entry["title"])
+    desc_tokens = _tokenize(entry["description"])
+    tag_set = set(entry["tags"])
+    query_name_tokens = _tokenize(" ".join(entry["query_names"]))
+    chart_name_tokens = _tokenize(" ".join(entry["chart_names"]))
+    sql_tokens = _tokenize(" ".join(entry["sql_snippets"]))
+
+    for qt in query_tokens:
+        if qt in title_tokens:
+            score += 3.0
+            if "title_match" not in reasons:
+                reasons.append("title_match")
+
+        if qt in tag_set:
+            score += 2.5
+            if "tag_match" not in reasons:
+                reasons.append("tag_match")
+
+        if qt in desc_tokens:
+            score += 1.5
+            if "description_match" not in reasons:
+                reasons.append("description_match")
+
+        if qt in query_name_tokens or qt in chart_name_tokens:
+            score += 1.0
+            if "metric_overlap" not in reasons:
+                reasons.append("metric_overlap")
+
+        if qt in sql_tokens:
+            score += 0.5
+            if "sql_match" not in reasons:
+                reasons.append("sql_match")
+
+    return score, reasons

dataface/agent_api/skill_install.py

@@ -0,0 +1,141 @@
+"""File-based install of wheel workflow skills into agent skill directories."""
+
+from __future__ import annotations
+
+import shutil
+from collections.abc import Set
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from dataface.agent_api.skill_render import render_skill_body
+from dataface.agent_api.skills import Skill, all_skill_names, list_skills
+
+# Pattern skills are not file-installed; tombstone retired dirs on re-run.
+RETIRED_SKILL_NAMES: tuple[str, ...] = (
+    "before-after-comparison",
+    "drill-down-link",
+    "faceted-small-multiples",
+    "filter-bar-with-variables",
+    "kpi-row",
+    "single-metric-bignum",
+    "table-heavy-ops-dashboard",
+    "time-series-trend",
+    "top-n-with-detail",
+    "two-by-two-grid-overview",
+)
+
+_LEGACY_SKILL_ROOTS: tuple[Path, ...] = (
+    Path(".cursor/skills"),
+    Path(".codex/skills"),
+)
+
+_SKILL_TARGET_DIRS: dict[str, Path] = {
+    "agents": Path(".agents/skills"),
+    "codex": Path(".agents/skills"),
+    "claude": Path(".claude/skills"),
+}
+
+SKILL_INSTALL_TARGETS: frozenset[str] = frozenset(_SKILL_TARGET_DIRS)
+
+
+class InstallSkillsResult(BaseModel):
+    installed: list[str] = Field(default_factory=list)
+    retired_removed: list[str] = Field(default_factory=list)
+    skipped_existing: list[str] = Field(default_factory=list)
+    legacy_dirs_detected: list[Path] = Field(default_factory=list)
+
+
+def skills_for_file_install() -> list[Skill]:
+    """Workflow skills exposed on the CLI surface."""
+    return sorted(
+        (s for s in list_skills(surface="cli").skills if s.kind == "workflow"),
+        key=lambda s: s.name,
+    )
+
+
+def target_dir_for(target: str, *, project_root: Path) -> Path:
+    rel = _SKILL_TARGET_DIRS[target]
+    return project_root / rel
+
+
+def detect_skill_targets(project_root: Path) -> list[str]:
+    """Return install target keys detected under ``project_root``."""
+    targets: list[str] = []
+    if (project_root / ".cursor").is_dir() or (project_root / "AGENTS.md").is_file():
+        targets.append("agents")
+    if (project_root / "CLAUDE.md").is_file():
+        targets.append("claude")
+    return targets
+
+
+def detect_legacy_skill_dirs(
+    project_root: Path, wheel_skill_names: Set[str]
+) -> list[Path]:
+    """Legacy triplicate dirs that still contain a wheel skill name."""
+    found: list[Path] = []
+    for rel_root in _LEGACY_SKILL_ROOTS:
+        root = project_root / rel_root
+        if not root.is_dir():
+            continue
+        for child in root.iterdir():
+            if child.is_dir() and child.name in wheel_skill_names:
+                found.append(child)
+    return found
+
+
+def _rendered_skill_md(skill: Skill) -> str:
+    raw = (skill.directory / "SKILL.md").read_text(encoding="utf-8")
+    if not raw.startswith("---"):
+        raise ValueError(f"{skill.directory / 'SKILL.md'}: missing frontmatter")
+    parts = raw.split("---", 2)
+    if len(parts) < 3:
+        raise ValueError(f"{skill.directory / 'SKILL.md'}: malformed frontmatter")
+    rendered_body = render_skill_body(skill.body, surface="cli")
+    return f"---{parts[1]}---\n{rendered_body}"
+
+
+def install_skills(
+    *,
+    target_dir: Path,
+    project_root: Path,
+    force: bool = False,
+    check: bool = False,
+) -> InstallSkillsResult:
+    """Install CLI-rendered workflow skills into ``target_dir``."""
+    wheel_names = all_skill_names()
+    legacy = detect_legacy_skill_dirs(project_root, wheel_names)
+    retired_removed: list[str] = []
+    installed: list[str] = []
+    skipped_existing: list[str] = []
+
+    if not check:
+        target_dir.mkdir(parents=True, exist_ok=True)
+        for name in RETIRED_SKILL_NAMES:
+            retired_path = target_dir / name
+            if retired_path.is_dir():
+                shutil.rmtree(retired_path)
+                retired_removed.append(name)
+
+    for skill in skills_for_file_install():
+        dest_dir = target_dir / skill.name
+        dest_md = dest_dir / "SKILL.md"
+        content = _rendered_skill_md(skill)
+        if check:
+            installed.append(skill.name)
+            continue
+        if dest_md.is_file() and not force:
+            skipped_existing.append(skill.name)
+            continue
+        if dest_dir.is_dir():
+            shutil.rmtree(dest_dir)
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        dest_md.write_text(content, encoding="utf-8")
+        installed.append(skill.name)
+
+    return InstallSkillsResult(
+        installed=installed,
+        retired_removed=retired_removed,
+        skipped_existing=skipped_existing,
+        legacy_dirs_detected=legacy,
+    )