modelspec-dev 0.1.0__py3-none-any.whl

cli/modelspec/snapshot_build_cmd.py

@@ -0,0 +1,49 @@
+"""``modelspec snapshot build``: compile the decision snapshot (MODEL-138).
+
+Collects cards, offerings, sources and the verification log, keeps only
+verified values from registered, non-excluded sources, runs the premier-set
+completeness gate, and writes a gzipped snapshot. Signed with
+``MODELSPEC_SNAPSHOT_KEY`` when it is set. Exits 1 on any failure, naming the
+model, facet and source for each gate gap.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from decision import snapshot as snap
+
+EXIT_ERROR = 1
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def build(
+    out: Path = typer.Option(..., "--out", help="Where to write the snapshot (.json.gz)."),
+    root: Path = typer.Option(REPO_ROOT, "--root", help="Repository root."),
+    premier: Optional[Path] = typer.Option(  # noqa: UP045 - Typer reads the annotation
+        None, "--premier", help="Premier list; default premier/slice-1.yaml under --root."),
+    as_of: Optional[str] = typer.Option(  # noqa: UP045
+        None, "--as-of", help="The snapshot's date, YYYY-MM-DD; default today."),
+) -> None:
+    """Build the decision snapshot. Fails when a premier model lacks a guaranteed fact."""
+    premier = premier or root / "premier" / "slice-1.yaml"
+    try:
+        when = date.fromisoformat(as_of) if as_of else date.today()
+        built = snap.build_from_repo(root, premier=premier, as_of=when,
+                                     registry=snap.default_registry())
+    except ValueError as exc:  # SnapshotError, CompletenessError, a bad --as-of
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(EXIT_ERROR) from exc
+    signed = snap.env_key() is not None
+    built.write(out)
+    lineup = len(built.content["lineup"]["candidates"])
+    typer.echo(f"{built.snapshot_id} {built.content_hash}")
+    typer.echo(f"{lineup} candidates in the lineup, "
+               f"{len(built.content['archive']['candidates'])} in the archive, "
+               f"{built.content['out_of_lineup']} models outside the premier lineup; "
+               f"{'signed' if signed else f'unsigned ({snap.KEY_ENV} not set)'}")
+    typer.echo(f"written to {out}")

cli/modelspec/verify_cmd.py

@@ -0,0 +1,125 @@
+"""``modelspec verify``: verification and decision-engine accuracy.
+
+Reads the queue in ``verification/queue/events.jsonl``, re-reads each claim from
+its retained source copy with the deterministic extractors, appends every
+outcome to ``verification/log.jsonl`` and prints a summary. Pass
+``--llm-reader claude`` (Claude Sonnet) or ``--llm-reader mistral`` (Mistral
+Large on the local ollama host) to add a prose reader after the deterministic
+readers. A reader is only asked about values collected by another model family.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+
+import typer
+
+from decision import verify as v
+from decision.sources import CopyStore
+
+#: ``--llm-reader`` name -> the ``decision.verify`` factory that builds it.
+READERS = {"claude": "claude_extractor", "mistral": "mistral_extractor"}
+
+app = typer.Typer(
+    help="Verify queued values or run decision-engine accuracy checks.",
+    invoke_without_command=True,
+    no_args_is_help=False,
+)
+
+
+@app.callback()
+def verify(
+    ctx: typer.Context,
+    changed_only: bool = typer.Option(
+        False, "--changed-only", help="Only values re-queued by source change detection."
+    ),
+    root: Path = typer.Option(v.REPO_ROOT, "--root", help="Repository root."),
+    llm_reader: str | None = typer.Option(
+        None, "--llm-reader", help="Independent prose reader: claude or mistral."
+    ),
+    as_json: bool = typer.Option(False, "--json", help="Machine-readable output."),
+) -> None:
+    """Verify queued facts and evidence against their sources (two keys)."""
+    if ctx.invoked_subcommand is not None:
+        return
+    directory = root / "verification"
+    extractors = v.deterministic_extractors()
+    if llm_reader is not None:
+        if llm_reader not in READERS:
+            raise typer.BadParameter(
+                f"supported readers: {', '.join(READERS)}", param_hint="--llm-reader"
+            )
+        extractors.append(getattr(v, READERS[llm_reader])())
+    report = v.run(
+        v.Queue(directory),
+        v.VerificationLog(directory),
+        v.StoredRegions(CopyStore(), v.load_sources(root / "registry" / "sources.yaml")),
+        extractors,
+        today=date.today(),
+        changed_only=changed_only,
+    )
+    if as_json:
+        typer.echo(json.dumps({"command": "verify", **report.to_dict()}, indent=2))
+        return
+    scope = "re-queued by change detection" if changed_only else "queued"
+    typer.echo(f"Verified what was {scope}: {len(report.results)} value(s).")
+    for outcome, n in report.counts.items():
+        typer.echo(f"  {outcome}: {n}")
+    for r in report.results:
+        if r.outcome == "verified":
+            continue
+        detail = (
+            "; ".join(f"{d.field}: expected {d.expected}, found {d.found}" for d in r.diffs)
+            or r.reason
+            or ""
+        )
+        typer.echo(f"  {r.outcome:<11} {v.ref_str(r.target)}  {detail}")
+    for t in report.unknown:
+        typer.echo(f"  no claim filed for re-queued {v.ref_str(t)}")
+
+
+def accuracy(
+    profile: str = typer.Option("pr", "--profile", help="pr or nightly."),
+    output_dir: Path = typer.Option(Path("accuracy-report"), "--output-dir"),
+    root: Path = typer.Option(v.REPO_ROOT, "--root", help="Repository root."),
+    config: Path | None = typer.Option(None, "--config", help="Accuracy config YAML."),
+    snapshot_file: Path | None = typer.Option(None, "--snapshot-file"),
+    report_date: str | None = typer.Option(None, "--date", help="Report date, YYYY-MM-DD."),
+    llm_reader: str | None = typer.Option(None, "--llm-reader"),
+    sample_size: int | None = typer.Option(None, "--sample-size"),
+    seed: str | None = typer.Option(None, "--seed"),
+) -> None:
+    """Run the decision-engine accuracy harness and print its Markdown report."""
+    from scripts import accuracy as harness
+
+    if profile not in harness.PROFILES:
+        raise typer.BadParameter("supported profiles: pr, nightly", param_hint="--profile")
+    try:
+        day = date.fromisoformat(report_date) if report_date else date.today()
+    except ValueError as exc:
+        raise typer.BadParameter("expected YYYY-MM-DD", param_hint="--date") from exc
+    report = harness.run_profile(
+        root=root,
+        config_path=config or root / "accuracy.yaml",
+        profile=profile,
+        output_dir=output_dir,
+        snapshot_file=snapshot_file,
+        as_of=day,
+        llm_reader=llm_reader,
+        sample_size=sample_size,
+        random_seed=seed,
+    )
+    markdown, payload = harness.write_report(report, output_dir)
+    typer.echo(markdown.read_text(encoding="utf-8"), nl=False)
+    typer.echo(f"JSON: {payload}")
+    if report.status == "fail":
+        raise typer.Exit(1)
+
+
+# The accuracy harness runs repository tests and reads repository-only recall
+# material. Keep it available in a source checkout, but do not advertise a
+# command that an installed wheel cannot execute.
+if (v.REPO_ROOT / "scripts" / "accuracy.py").is_file():
+    app.command("accuracy")(accuracy)

cli/modelspec/vocab_cmd.py

@@ -0,0 +1,204 @@
+"""Inspect the decision vocabulary cached by ``snapshot fetch``."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any, Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from . import offline
+from . import snapshot as snap
+from .snapshot import decision_vocabulary_path
+from .vocabulary_cache import (
+    VocabularyInvalidError,
+    VocabularyMissingError,
+)
+from .vocabulary_cache import (
+    load_cached_vocabulary as _load_cached_vocabulary,
+)
+
+SECTIONS = (
+    "facets", "benchmarks", "domains", "providers", "task-types", "coverage", "templates"
+)
+
+
+def load_cached_vocabulary(*, as_json: bool = False) -> dict[str, Any]:
+    """Read the current cached vocabulary, or exit like other offline commands."""
+    try:
+        return _load_cached_vocabulary()
+    except VocabularyMissingError as exc:
+        offline._emit_error("vocab", str(exc), as_json)
+        raise typer.Exit(offline.EXIT_NO_SNAPSHOT) from exc
+    except VocabularyInvalidError as exc:
+        offline._emit_error("vocab", str(exc), as_json)
+        raise typer.Exit(offline.EXIT_ERROR) from exc
+
+
+def _freshness(vocabulary: dict[str, Any]) -> dict[str, Any]:
+    """Use rank-cache metadata when present, with an offline decision fallback."""
+    try:
+        return snap.load().freshness()
+    except (snap.SnapshotMissing, snap.SnapshotInvalid):
+        path = decision_vocabulary_path()
+        fetched_at = datetime.fromtimestamp(path.stat().st_mtime, tz=UTC)
+        age_days = (datetime.now(UTC) - fetched_at).total_seconds() / 86400
+        return {
+            "fetched_at": fetched_at.isoformat(),
+            "age_days": round(age_days, 2),
+            "stale": age_days > snap.STALE_AFTER_DAYS,
+            "stale_after_days": snap.STALE_AFTER_DAYS,
+            "origin": "local decision cache",
+            "build_commit": vocabulary.get("snapshot"),
+            "built_at": vocabulary.get("coverage", {}).get("as_of"),
+        }
+
+
+def _search(rows: list[dict[str, Any]], text: str | None) -> list[dict[str, Any]]:
+    if not text:
+        return rows
+    needle = text.casefold()
+    return [row for row in rows if needle in str(row.get("id", "")).casefold()
+            or needle in str(row.get("label") or row.get("name") or "").casefold()]
+
+
+def _filtered(vocabulary: dict[str, Any], section: str, class_id: str | None,
+              domain_id: str | None, search: str | None) -> Any:
+    key = "task_types" if section == "task-types" else section
+    value = vocabulary.get(key, [] if key != "providers" and key != "coverage" else {})
+    if section == "providers":
+        rows = [{"id": id_, "name": name} for id_, name in value.items()]
+        return _search(rows, search)
+    if section == "task-types":
+        if not search:
+            return value
+        needle = search.casefold()
+        return [item for item in value if needle in str(item).casefold()]
+    if section == "coverage":
+        return value
+
+    rows = list(value)
+    if section == "benchmarks" and domain_id:
+        rows = [row for row in rows if any(tag.get("id") == domain_id
+                                           for tag in row.get("domains", []))]
+    if section == "benchmarks" and class_id:
+        classes = vocabulary.get("coverage", {}).get("classes", [])
+        selected = next((row for row in classes if row.get("id") == class_id), None)
+        covered_domains = ({row["id"] for row in selected.get("domains", [])}
+                           if selected else set())
+        rows = [row for row in rows if any(tag.get("id") in covered_domains
+                                           for tag in row.get("domains", []))]
+    return _search(rows, search)
+
+
+def _table(section: str, value: Any) -> Table:
+    table = Table(show_header=True, header_style="bold")
+    if section == "facets":
+        for column in ("id", "label", "type", "operators", "coverage"):
+            table.add_column(column)
+        for row in value:
+            table.add_row(row["id"], row.get("label", ""), row.get("value_type", ""),
+                          ", ".join(row.get("operators", [])),
+                          f"{row.get('known', 0)}/{row.get('of', 0)}")
+    elif section == "benchmarks":
+        for column in ("id", "name", "domains"):
+            table.add_column(column)
+        for row in value:
+            domains = ", ".join(f"{tag['id']} ({tag['directness']})"
+                                for tag in row.get("domains", []))
+            table.add_row(row["id"], row.get("name", ""), domains)
+    elif section == "domains":
+        for column in ("id", "name", "proxy_only", "estimate models"):
+            table.add_column(column)
+        for row in value:
+            table.add_row(row["id"], row.get("name", ""), str(row.get("proxy_only", False)),
+                          str(row.get("estimate_models", 0)))
+    elif section == "providers":
+        table.add_column("id")
+        table.add_column("name")
+        for row in value:
+            table.add_row(row["id"], row["name"])
+    elif section == "task-types":
+        table.add_column("task type")
+        for item in value:
+            table.add_row(str(item))
+    elif section == "templates":
+        for column in ("id", "name", "available", "reason", "purpose"):
+            table.add_column(column)
+        for row in value:
+            table.add_row(
+                row["id"], row.get("name", ""), str(row.get("available", True)),
+                row.get("unavailable_reason") or "",
+                row.get("purpose", ""),
+            )
+    else:
+        table.add_column("coverage")
+        table.add_column("value")
+        for key, item in value.items():
+            rendered = (
+                json.dumps(item, separators=(",", ":"))
+                if isinstance(item, list | dict)
+                else str(item)
+            )
+            table.add_row(key, rendered)
+    return table
+
+
+def vocab(
+    section: Optional[str] = typer.Argument(  # noqa: UP045
+        None, help="facets, benchmarks, domains, providers, task-types, coverage or templates"
+    ),
+    class_id: Optional[str] = typer.Option(  # noqa: UP045
+        None, "--class", help="Limit entries to a model class where coverage permits."
+    ),
+    domain: Optional[str] = typer.Option(  # noqa: UP045
+        None, "--domain", help="Limit benchmarks to a capability domain."
+    ),
+    search: Optional[str] = typer.Option(  # noqa: UP045
+        None, "--search", help="Substring match on ID and label or name."
+    ),
+    as_json: bool = typer.Option(
+        False, "--json", help="Print the raw vocabulary or selected section."
+    ),
+) -> None:
+    """Show the cached decision vocabulary. Never uses the network."""
+    vocabulary = load_cached_vocabulary(as_json=as_json)
+    if section is None:
+        if as_json:
+            typer.echo(json.dumps({
+                "schema_version": offline.SCHEMA_VERSION,
+                "command": "vocab",
+                "freshness": _freshness(vocabulary),
+                "result": vocabulary,
+            }, indent=2))
+            return
+        providers = vocabulary.get("providers", {})
+        typer.echo(f"snapshot: {vocabulary.get('snapshot', 'unknown')}")
+        typer.echo("  " + "  ".join(
+            f"{name}: {len(vocabulary.get(key, providers if key == 'providers' else []))}"
+            for name, key in (("facets", "facets"), ("benchmarks", "benchmarks"),
+                              ("domains", "domains"), ("providers", "providers"),
+                              ("task-types", "task_types"), ("templates", "templates"))
+        ))
+        typer.echo("next: modelspec vocab facets")
+        return
+    if section not in SECTIONS:
+        offline._emit_error(
+            "vocab", f"unknown section {section!r}; choose one of {', '.join(SECTIONS)}", as_json
+        )
+        raise typer.Exit(offline.EXIT_ERROR)
+    value = _filtered(vocabulary, section, class_id, domain, search)
+    if as_json:
+        if section == "providers":
+            value = {row["id"]: row["name"] for row in value}
+        typer.echo(json.dumps({
+            "schema_version": offline.SCHEMA_VERSION,
+            "command": "vocab",
+            "freshness": _freshness(vocabulary),
+            "result": value,
+        }, indent=2))
+        return
+    Console(width=160).print(_table(section, value))

cli/modelspec/vocabulary_cache.py

@@ -0,0 +1,54 @@
+"""Read the decision vocabulary selected by the local cache."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from . import snapshot
+
+
+class VocabularyMissingError(ValueError):
+    """No decision vocabulary has been cached."""
+
+
+class VocabularyInvalidError(ValueError):
+    """The selected decision vocabulary cannot be read."""
+
+
+def load_cached_vocabulary() -> dict[str, Any]:
+    """Return the selected vocabulary, distinguishing absence from corruption."""
+    try:
+        current = (
+            snapshot.cache_dir() / snapshot.DECISION_DIRECTORY / "current"
+        ).read_text(encoding="utf-8").strip()
+    except FileNotFoundError as exc:
+        raise VocabularyMissingError(
+            "no cached decision vocabulary. Run `modelspec snapshot fetch`."
+        ) from exc
+    except (OSError, UnicodeError) as exc:
+        raise VocabularyInvalidError(
+            f"cannot read the cached decision vocabulary: {exc}"
+        ) from exc
+    if snapshot.DECISION_SNAPSHOT_ID.fullmatch(current) is None:
+        raise VocabularyInvalidError(
+            "cannot read the cached decision vocabulary: "
+            "decision/current contains an invalid snapshot ID"
+        )
+    path = (
+        snapshot.cache_dir() / snapshot.DECISION_DIRECTORY / current
+        / snapshot.DECISION_VOCABULARY_FILENAME
+    )
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise VocabularyMissingError(
+            "no cached decision vocabulary. Run `modelspec snapshot fetch`."
+        ) from exc
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise VocabularyInvalidError(
+            f"cannot read the cached decision vocabulary: {exc}"
+        ) from exc
+    if not isinstance(value, dict):
+        raise VocabularyInvalidError("cached decision vocabulary is not a JSON object")
+    return value

decision/__init__.py

@@ -0,0 +1,13 @@
+"""The decision engine (slice 1; design: `docs/design/decision-engine.md`). The contract is ``decision.contract``.
+
+A new package beside v1. Nothing here is imported by `api/ranking/` or
+`pipeline/ranking.py`, and nothing here changes `/v1/rank`.
+"""
+
+from decision.contract import CONTRACT_VERSION, Decision, Spec, SpecError, parse_spec, spec_hash
+from decision.engine import decide
+from decision.filter import apply
+from decision.resolve import resolve
+
+__all__ = ["CONTRACT_VERSION", "Decision", "Spec", "SpecError", "apply", "decide", "parse_spec",
+           "resolve", "spec_hash"]