touchstone-eval 0.1.0__py3-none-any.whl

touchstone/__init__.py

@@ -0,0 +1,3 @@
+"""touchstone: a personal model/harness eval benchmark."""
+
+__version__ = "0.1.0"

touchstone/artifacts.py

@@ -0,0 +1,51 @@
+"""Resolve a case's AI artifacts into concrete paths.
+
+Artifacts are *materialized* into a sandbox by the harness adapter (different agents
+expect different layouts), so this module only resolves and validates paths. See
+`harness/claude_code.py` for the Claude Code layout (`.claude/skills`, `.mcp.json`, ...).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .config import Case
+
+
+@dataclass
+class ArtifactSet:
+    """Resolved, existing artifact paths for one case."""
+
+    skills: list[Path] = field(default_factory=list)
+    commands: list[Path] = field(default_factory=list)
+    plugins: list[Path] = field(default_factory=list)
+    mcp: Path | None = None
+
+    def is_empty(self) -> bool:
+        return not (self.skills or self.commands or self.plugins or self.mcp)
+
+
+def _resolve_all(case: Case, rels: list[str]) -> list[Path]:
+    out: list[Path] = []
+    for rel in rels:
+        p = case.resolve(rel)
+        if not p.exists():
+            raise FileNotFoundError(f"artifact not found: {rel} (resolved {p})")
+        out.append(p)
+    return out
+
+
+def build_artifact_set(case: Case) -> ArtifactSet:
+    spec = case.artifacts
+    mcp = None
+    if spec.mcp:
+        mcp = case.resolve(spec.mcp)
+        if not mcp.exists():
+            raise FileNotFoundError(f"mcp config not found: {spec.mcp} (resolved {mcp})")
+    return ArtifactSet(
+        skills=_resolve_all(case, spec.skills),
+        commands=_resolve_all(case, spec.commands),
+        plugins=_resolve_all(case, spec.plugins),
+        mcp=mcp,
+    )

touchstone/cli.py

@@ -0,0 +1,212 @@
+"""Command-line interface: validate / run / report / list."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .config import discover_cases, load_case
+from .harness.registry import available_harnesses
+from .reachability import UnavailableError, scan_cases
+from .report import generate
+from .runner import build_cells, run
+from .store import RunStore
+
+DEFAULT_EVALS = "evals"
+DEFAULT_RUNS = "runs"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="touchstone", description=__doc__)
+    parser.add_argument("--version", action="version", version=f"touchstone {__version__}")
+    parser.add_argument("--evals-dir", default=DEFAULT_EVALS)
+    parser.add_argument("--runs-dir", default=DEFAULT_RUNS)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("list", help="list cases and past runs")
+    p_validate = sub.add_parser("validate", help="schema-check case.yaml files")
+    p_validate.add_argument("--check-access", action="store_true",
+                            help="also probe each case's external repos (network) and report "
+                                 "reachability; exit non-zero if a required case is unreachable")
+
+    p_run = sub.add_parser("run", help="run the matrix and produce a report")
+    p_run.add_argument("--eval", action="append", help="case id (repeatable); default all")
+    p_run.add_argument("--harness", action="append", help="restrict to these matrix harnesses")
+    p_run.add_argument("--model", action="append",
+                       help="restrict to these declared models (filter)")
+    p_run.add_argument("--with-model", action="append", metavar="[HARNESS=]MODEL",
+                       help="run this model even if a case did not declare it, replacing the "
+                            "case's models for the harness (repeatable). Use to compare "
+                            "models on the same harness, e.g. --harness droid "
+                            "--with-model A --with-model B. Prefix HARNESS= to scope to one "
+                            "harness when a run spans several.")
+    p_run.add_argument("--trials", type=int, help="override trial count")
+    p_run.add_argument("--resume", help="continue an interrupted run id")
+    p_run.add_argument("--workers", type=int, default=1,
+                       help="run cells in parallel (default 1; manual interaction forces 1)")
+    p_run.add_argument("--keep-sandboxes", action="store_true",
+                       help="do not tear down worktree sandboxes after each cell")
+    p_run.add_argument("--llm-concurrency", type=int,
+                       help="max concurrent auxiliary LLM calls (judge + responder)")
+    p_run.add_argument("--on-unavailable", choices=["fail", "skip"], default="fail",
+                       help="when a case's external repo is unreachable: 'fail' the whole run "
+                            "(default) or 'skip' those cases and run the rest")
+
+    p_report = sub.add_parser("report", help="(re)generate a run's report.md")
+    p_report.add_argument("run_id")
+
+    p_export = sub.add_parser("export", help="export a run's traces to LangFuse JSON")
+    p_export.add_argument("run_id")
+    p_export.add_argument("--push", action="store_true",
+                          help="also push via the langfuse SDK (needs langfuse + keys)")
+
+    args = parser.parse_args(argv)
+    dispatch = {
+        "list": _cmd_list, "validate": _cmd_validate,
+        "run": _cmd_run, "report": _cmd_report, "export": _cmd_export,
+    }
+    return dispatch[args.cmd](args)
+
+
+def _cmd_list(args) -> int:
+    cases = discover_cases(args.evals_dir)
+    print(f"Cases ({len(cases)}) in {args.evals_dir}:")
+    for c in cases:
+        obs = "observe" if c.observe else "output-only"
+        print(f"  {c.id:30} harnesses={c.matrix.all_harnesses} "
+              f"models={c.matrix.all_models} trials={c.matrix.trials} [{obs}]")
+    print(f"\nHarnesses available: {available_harnesses()}")
+    runs = RunStore.list_runs(args.runs_dir)
+    print(f"\nRuns ({len(runs)}) in {args.runs_dir}:")
+    for r in runs:
+        print(f"  {r}")
+    return 0
+
+
+def _cmd_validate(args) -> int:
+    evals = Path(args.evals_dir)
+    dirs = [d for d in sorted(evals.iterdir()) if d.is_dir()] if evals.is_dir() else []
+    if not dirs:
+        print(f"No case directories under {evals}", file=sys.stderr)
+        return 1
+    ok = True
+    cases = []
+    for d in dirs:
+        if not (d / "case.yaml").is_file():
+            continue
+        try:
+            case = load_case(d)
+            cells = build_cells([case])
+            print(f"OK   {case.id}: {len(cells)} cells "
+                  f"({len(case.graders)} graders)")
+            cases.append(case)
+        except Exception as e:
+            ok = False
+            print(f"FAIL {d.name}: {type(e).__name__}: {e}", file=sys.stderr)
+    if args.check_access and cases:
+        ok = _report_access(cases) and ok
+    return 0 if ok else 1
+
+
+def _report_access(cases) -> bool:
+    """Probe each case's external repos and print reachability. Returns False if any REQUIRED
+    case is unreachable (optional-only unreachables don't fail validation)."""
+    unreachable = scan_cases(cases)
+    if not unreachable:
+        print(f"\nAccess: all {len(cases)} cases' external repos reachable.")
+        return True
+    print("\nAccess check:", file=sys.stderr)
+    required_blocked = False
+    for cid in sorted(unreachable):
+        case = next(c for c in cases if c.id == cid)
+        sev = "REQUIRED" if case.availability == "required" else "optional"
+        required_blocked = required_blocked or case.availability == "required"
+        for u in unreachable[cid]:
+            print(f"  UNREACHABLE [{sev}] {cid}: {u.external.kind} {u.external.repo} "
+                  f"({u.result.reason}) — {u.result.detail}", file=sys.stderr)
+    reachable = len(cases) - len(unreachable)
+    print(f"\n{reachable}/{len(cases)} cases reachable; {len(unreachable)} unreachable "
+          f"({'required cases blocked' if required_blocked else 'all optional — ok'}).",
+          file=sys.stderr)
+    return not required_blocked
+
+
+def _parse_model_override(values: list[str] | None) -> dict[str | None, list[str]] | None:
+    """Parse `--with-model` values into {harness|None: [models]}.
+
+    `MODEL` -> applies to any harness (key None); `HARNESS=MODEL` -> scoped to that
+    harness. Repeats accumulate; later values append.
+    """
+    if not values:
+        return None
+    override: dict[str | None, list[str]] = {}
+    for v in values:
+        harness, sep, model = v.partition("=")
+        key, name = (harness, model) if sep else (None, harness)
+        name = name.strip()
+        if not name:
+            raise ValueError(f"--with-model {v!r}: empty model name")
+        override.setdefault(key, [])
+        if name not in override[key]:
+            override[key].append(name)
+    return override
+
+
+def _cmd_run(args) -> int:
+    try:
+        model_override = _parse_model_override(args.with_model)
+        run_id = run(
+            args.evals_dir, args.runs_dir,
+            eval_filter=args.eval, harnesses=args.harness,
+            models=args.model, trials=args.trials, model_override=model_override,
+            resume=args.resume, workers=args.workers,
+            keep_sandboxes=args.keep_sandboxes, llm_concurrency=args.llm_concurrency,
+            on_unavailable=args.on_unavailable,
+        )
+    except UnavailableError as e:
+        print(str(e), file=sys.stderr)
+        print("\nFix git access for these repos, mark the cases `availability: optional`, or "
+              "re-run with `--on-unavailable skip` to degrade them.", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"Run failed: {type(e).__name__}: {e}", file=sys.stderr)
+        return 1
+    report_path = generate(args.runs_dir, run_id)
+    print(f"\nDone. Report: {report_path}")
+    return 0
+
+
+def _cmd_report(args) -> int:
+    try:
+        path = generate(args.runs_dir, args.run_id)
+    except FileNotFoundError as e:
+        print(str(e), file=sys.stderr)
+        return 1
+    print(f"Report: {path}")
+    return 0
+
+
+def _cmd_export(args) -> int:
+    from .export import push_langfuse, to_langfuse
+
+    try:
+        data = to_langfuse(args.runs_dir, args.run_id)
+    except FileNotFoundError as e:
+        print(str(e), file=sys.stderr)
+        return 1
+    out = Path(args.runs_dir) / args.run_id / "langfuse.json"
+    print(f"Exported {len(data['traces'])} traces -> {out}")
+    if args.push:
+        try:
+            n = push_langfuse(data)
+            print(f"Pushed {n} traces to LangFuse")
+        except Exception as e:
+            print(f"Push failed: {e}", file=sys.stderr)
+            return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

touchstone/concurrency.py

@@ -0,0 +1,34 @@
+"""Shared concurrency limits for parallel runs.
+
+When many Cells run in parallel, the auxiliary LLMs (Judge + Responder) can fan out wide
+enough to trip provider rate limits. They acquire a shared `llm_slot()` so total in-flight
+auxiliary LLM calls stay bounded regardless of how many Cells are executing.
+"""
+
+from __future__ import annotations
+
+import threading
+from contextlib import contextmanager
+
+_DEFAULT = 8
+_sem = threading.Semaphore(_DEFAULT)
+_limit = _DEFAULT
+
+
+def set_llm_concurrency(n: int) -> None:
+    global _sem, _limit
+    _limit = max(1, int(n))
+    _sem = threading.Semaphore(_limit)
+
+
+def llm_concurrency() -> int:
+    return _limit
+
+
+@contextmanager
+def llm_slot():
+    _sem.acquire()
+    try:
+        yield
+    finally:
+        _sem.release()