skillproof 0.1.0__py3-none-any.whl

skillproof/config.py

@@ -0,0 +1,188 @@
+"""The evals.yaml schema and loader.
+
+Paths in evals.yaml are relative to the evals.yaml file. `load_config()`
+returns a config with every path resolved to an absolute one.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError, model_validator
+
+from skillproof.assertions import ASSERTION_TYPES, Assertion
+from skillproof.errors import ConfigError
+from skillproof.runners import RUNNERS, get_runner_class
+
+CONFIG_NAMES = ("evals.yaml", "evals.yml")
+
+
+class JudgeConfig(BaseModel):
+    """Which model grades `llm_judge` assertions.
+
+    `auto` picks a vendor different from the agent under test (Claude agents
+    are judged by an OpenAI model and vice versa) to avoid self-grading.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    provider: Literal["auto", "anthropic", "openai", "mock"] = "auto"
+    model: str | None = None  # None = the provider's default judge model
+    base_url: str | None = None  # for OpenAI-compatible judge endpoints
+    api_key_env: str | None = None  # env var holding the key; None = provider default
+
+
+class Case(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    name: str = Field(min_length=1)
+    prompt: str = Field(min_length=1)
+    fixtures: Path | None = None  # folder copied into each run's sandbox
+    trials: int | None = Field(default=None, ge=1, le=100)  # overrides the top-level value
+    timeout: float | None = Field(default=None, gt=0)  # seconds; overrides top-level
+    # false = a negative case: the skill should NOT activate for this prompt.
+    should_trigger: bool = True
+    assertions: list[Assertion] = Field(default_factory=list, alias="assert")
+
+
+class EvalConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    skill: Path = Path(".")
+    runner: str = "claude-code"
+    model: str | None = None  # passed to the runner; None = the agent's default
+    runner_options: dict[str, Any] = Field(default_factory=dict)
+    trials: int = Field(default=5, ge=1, le=100)
+    timeout: float = Field(default=300, gt=0)  # seconds per run
+    judge: JudgeConfig = Field(default_factory=JudgeConfig)
+    cases: list[Case] = Field(min_length=1)
+
+    _path: Path | None = PrivateAttr(default=None)
+
+    @model_validator(mode="after")
+    def _check(self) -> EvalConfig:
+        if self.runner not in RUNNERS:
+            valid = ", ".join(RUNNERS)
+            raise ValueError(f"unknown runner {self.runner!r}. Valid runners: {valid}")
+        if self.model is None and get_runner_class(self.runner).requires_model:
+            raise ValueError(f"the {self.runner} runner needs `model:` (e.g. model: gpt-6.1-sol)")
+        seen: set[str] = set()
+        for case in self.cases:
+            if case.name in seen:
+                raise ValueError(f"duplicate case name {case.name!r}; case names must be unique")
+            seen.add(case.name)
+        return self
+
+    @property
+    def path(self) -> Path | None:
+        """The evals.yaml this config was loaded from."""
+        return self._path
+
+    def trials_for(self, case: Case) -> int:
+        return case.trials or self.trials
+
+    def timeout_for(self, case: Case) -> float:
+        return case.timeout or self.timeout
+
+    def uses_judge(self) -> bool:
+        return any(a.type == "llm_judge" for c in self.cases for a in c.assertions)
+
+
+def find_config(path: Path) -> Path:
+    """Accept either an evals.yaml file or a folder containing one."""
+    if path.is_file():
+        return path
+    if path.is_dir():
+        for name in CONFIG_NAMES:
+            if (path / name).is_file():
+                return path / name
+        raise ConfigError(
+            f"No evals.yaml in {path}",
+            hint=f"Create one with: skillproof init {path}",
+        )
+    raise ConfigError(f"Path not found: {path}", hint="Pass a skill folder or an evals.yaml file.")
+
+
+def format_validation_error(exc: ValidationError, source: Path | str) -> str:
+    """Turn pydantic errors into lines like `cases[0].assert[1]: unknown field 'vale'`."""
+    lines = []
+    for err in exc.errors():
+        loc = ""
+        for part in err["loc"]:
+            if isinstance(part, int):
+                loc += f"[{part}]"
+            elif part in ASSERTION_TYPES:
+                continue  # pydantic adds the union tag to the path; it's noise for users
+            else:
+                loc += f".{part}" if loc else str(part)
+        msg = err["msg"]
+        if err["type"] == "extra_forbidden":
+            msg = f"unknown field {err['loc'][-1]!r} (typo?)"
+        elif err["type"] == "missing":
+            msg = "missing required field"
+        elif err["type"] == "union_tag_invalid":
+            msg = f"unknown assertion type {err['input'].get('type')!r}. " + (
+                "Valid types: " + ", ".join(ASSERTION_TYPES)
+            )
+        elif msg.startswith("Value error, "):
+            msg = msg.removeprefix("Value error, ")
+        lines.append(f"  {loc or '(top level)'}: {msg}")
+    return f"{source} is invalid:\n" + "\n".join(lines)
+
+
+def parse_config(data: Any, base_dir: Path, source: Path | str = "evals.yaml") -> EvalConfig:
+    """Validate raw YAML data and resolve paths relative to `base_dir`."""
+    if not isinstance(data, dict):
+        raise ConfigError(
+            f"{source} must be a YAML mapping (key: value pairs).",
+            hint="Run `skillproof init` to see an example evals.yaml.",
+        )
+    try:
+        config = EvalConfig.model_validate(data)
+    except ValidationError as exc:
+        raise ConfigError(
+            format_validation_error(exc, source),
+            hint="Fix the fields above. See `skillproof init` output for a working example.",
+        ) from exc
+
+    base_dir = base_dir.resolve()
+    config.skill = (base_dir / config.skill).resolve()
+    for case in config.cases:
+        if case.fixtures is None:
+            continue
+        case.fixtures = (base_dir / case.fixtures).resolve()
+        if not case.fixtures.is_dir():
+            raise ConfigError(
+                f"Case {case.name!r}: fixtures folder not found: {case.fixtures}",
+                hint="`fixtures:` is a folder, relative to evals.yaml, copied into each sandbox.",
+            )
+
+    _validate_runner_options(config, source)
+    return config
+
+
+def _validate_runner_options(config: EvalConfig, source: Path | str) -> None:
+    options_model = get_runner_class(config.runner).Options
+    try:
+        options_model.model_validate(config.runner_options)
+    except ValidationError as exc:
+        message = format_validation_error(exc, f"{source} (runner_options for {config.runner})")
+        raise ConfigError(
+            message, hint=f"Check the `runner_options` supported by the {config.runner} runner."
+        ) from exc
+
+
+def load_config(path: Path) -> EvalConfig:
+    config_path = find_config(path).resolve()
+    try:
+        data = yaml.safe_load(config_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise ConfigError(
+            f"{config_path} is not valid YAML: {exc}",
+            hint="Check indentation and quoting near the line mentioned above.",
+        ) from exc
+    config = parse_config(data, config_path.parent, source=config_path.name)
+    config._path = config_path
+    return config

skillproof/engine.py

@@ -0,0 +1,268 @@
+"""Runs an eval suite: every case, N trials, with and without the skill.
+
+Each run gets a fresh sandbox (fixtures copied in), the runner does its thing,
+assertions are checked against the sandbox, and the transcript is saved under
+`<out_dir>/runs/<timestamp>/` with secrets redacted.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import shutil
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from skillproof.assertions import CheckContext, check_all
+from skillproof.config import Case, EvalConfig
+from skillproof.errors import ConfigError, RunAborted, hint_for_error
+from skillproof.judges import Judge, make_judge
+from skillproof.judges.base import JudgeUnavailable
+from skillproof.results import CaseResult, SuiteResult, TrialResult, Variant
+from skillproof.runners import AgentRun, Runner, RunRequest, get_runner
+from skillproof.sandbox import create_sandbox, redact, remove_sandbox
+from skillproof.skill import Skill, load_skill
+
+VARIANTS: tuple[Variant, ...] = ("with", "without")
+# Stop early when this many runs have finished and every one of them errored.
+FAIL_FAST_AFTER = 3
+
+
+@dataclass
+class RunOptions:
+    parallel: int = 4
+    out_dir: Path = Path(".skillproof")
+    keep_sandboxes: bool = False
+    trials: int | None = None  # overrides evals.yaml for every case
+    only_cases: list[str] | None = None  # run just these case names
+
+
+@dataclass
+class Job:
+    case: Case
+    variant: Variant
+    trial: int
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "case"
+
+
+def new_run_dir(out_dir: Path) -> Path:
+    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    run_dir = out_dir / "runs" / stamp
+    suffix = 1
+    while run_dir.exists():
+        suffix += 1
+        run_dir = out_dir / "runs" / f"{stamp}-{suffix}"
+    run_dir.mkdir(parents=True)
+    return run_dir
+
+
+def selected_cases(config: EvalConfig, only: list[str] | None) -> list[Case]:
+    if not only:
+        return list(config.cases)
+    names = {c.name for c in config.cases}
+    unknown = [n for n in only if n not in names]
+    if unknown:
+        raise ConfigError(
+            f"No case named {unknown[0]!r}",
+            hint="Available cases: " + ", ".join(repr(n) for n in names),
+        )
+    return [c for c in config.cases if c.name in only]
+
+
+class Engine:
+    def __init__(
+        self,
+        config: EvalConfig,
+        options: RunOptions | None = None,
+        runner: Runner | None = None,
+        judge: Judge | None = None,
+        on_trial: Callable[[TrialResult], None] | None = None,
+    ) -> None:
+        self.config = config
+        self.options = options or RunOptions()
+        self.runner = runner or get_runner(config.runner, config.runner_options)
+        self.judge = judge
+        self.on_trial = on_trial
+        self.cases = selected_cases(config, self.options.only_cases)
+
+    def jobs(self) -> list[Job]:
+        """Trials are interleaved (with/without, case by case) so drift over time
+        in API latency or model behaviour hits both variants equally."""
+        jobs = []
+        max_trials = max(self.trials_for(c) for c in self.cases)
+        for trial in range(max_trials):
+            for case in self.cases:
+                if trial < self.trials_for(case):
+                    jobs.extend(Job(case, variant, trial) for variant in VARIANTS)
+        return jobs
+
+    def trials_for(self, case: Case) -> int:
+        return self.options.trials or self.config.trials_for(case)
+
+    def skill_exclude(self, skill: Skill) -> list[Path]:
+        """Eval files inside the skill folder that the agent must never see."""
+        paths = [c.fixtures for c in self.config.cases if c.fixtures is not None]
+        if self.config.path is not None:
+            paths.append(self.config.path)
+        return [p for p in paths if p.is_relative_to(skill.path)]
+
+    async def run(self) -> SuiteResult:
+        skill = load_skill(self.config.skill)
+        self.runner.check_ready()
+        if self.judge is None and self.config.uses_judge():
+            self.judge = make_judge(self.config.judge, self.runner.vendor)
+
+        run_dir = new_run_dir(self.options.out_dir)
+        suite = SuiteResult(
+            skill_name=skill.name,
+            skill_path=str(skill.path),
+            runner=self.runner.name,
+            approximate=self.runner.approximate,
+            model=self.config.model,
+            judge=self.judge.label if self.judge else None,
+            started_at=datetime.now(timezone.utc),
+            run_dir=str(run_dir),
+        )
+        exclude = self.skill_exclude(skill)
+        semaphore = asyncio.Semaphore(max(1, self.options.parallel))
+
+        trials: list[TrialResult] = []
+        stopped = False
+
+        async def limited(job: Job) -> None:
+            nonlocal stopped
+            async with semaphore:
+                if stopped:  # don't start new runs once we've decided to stop
+                    return
+                trials.append(await self.run_trial(job, skill, exclude, run_dir))
+                try:
+                    self.stop_if_everything_fails(trials, run_dir)
+                except RunAborted:
+                    stopped = True
+                    raise
+
+        tasks = [asyncio.create_task(limited(job)) for job in self.jobs()]
+        try:
+            await asyncio.gather(*tasks)
+        except BaseException:  # fail-fast, Ctrl-C (cancellation) or a crash
+            for task in tasks:
+                task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            self.finish(suite, trials, complete=False)
+            raise
+        self.finish(suite, trials, complete=True)
+        return suite
+
+    def stop_if_everything_fails(self, trials: list[TrialResult], run_dir: Path) -> None:
+        """If the first few runs all errored, the rest will too (bad key, wrong model...).
+        Stop instead of burning time and money, and say what to fix."""
+        if len(trials) != FAIL_FAST_AFTER or not all(t.errored for t in trials):
+            return
+        error = trials[-1].run.error or "unknown error"
+        raise RunAborted(
+            f"The first {FAIL_FAST_AFTER} runs all failed, so skillproof stopped early.\n"
+            f"Last error: {error.strip()[:500]}",
+            hint=f"{hint_for_error(error)} Transcripts: {run_dir}",
+        )
+
+    def finish(self, suite: SuiteResult, trials: list[TrialResult], complete: bool) -> None:
+        """Group trials into cases and save results.json (also for partial runs)."""
+        suite.cases = []
+        for case in self.cases:
+            case_trials = sorted(
+                (t for t in trials if t.case == case.name), key=lambda t: (t.variant, t.trial)
+            )
+            suite.cases.append(
+                CaseResult(
+                    name=case.name,
+                    should_trigger=case.should_trigger,
+                    has_assertions=bool(case.assertions),
+                    trials=case_trials,
+                )
+            )
+        suite.complete = complete
+        suite.finished_at = datetime.now(timezone.utc)
+        (Path(suite.run_dir) / "results.json").write_text(
+            redact(suite.model_dump_json(indent=2)), encoding="utf-8"
+        )
+
+    async def run_trial(
+        self, job: Job, skill: Skill, exclude: list[Path], run_dir: Path
+    ) -> TrialResult:
+        case = job.case
+        timeout = self.config.timeout_for(case)
+        workdir = create_sandbox(case.fixtures)
+        try:
+            request = RunRequest(
+                prompt=case.prompt,
+                workdir=workdir,
+                skill=skill if job.variant == "with" else None,
+                skill_exclude=exclude,
+                model=self.config.model,
+                timeout_s=timeout,
+                case_name=case.name,
+                trial=job.trial,
+            )
+            agent_run = await self.call_runner(request, timeout)
+            assertions = []
+            if agent_run.error is None:  # don't grade (or pay a judge for) a broken run
+                ctx = CheckContext(run=agent_run, workdir=workdir, judge=self.judge)
+                try:
+                    assertions = await check_all(case.assertions, ctx)
+                except JudgeUnavailable as exc:
+                    agent_run.error = f"llm_judge failed: {exc}"
+            result = TrialResult(
+                case=case.name,
+                variant=job.variant,
+                trial=job.trial,
+                run=agent_run,
+                assertions=assertions,
+            )
+            self.save_trial(result, workdir, run_dir)
+        finally:
+            remove_sandbox(workdir)
+        if self.on_trial is not None:
+            self.on_trial(result)
+        return result
+
+    async def call_runner(self, request: RunRequest, timeout: float) -> AgentRun:
+        """Call the runner, enforcing the timeout and turning crashes into errors."""
+        start = time.monotonic()
+        try:
+            agent_run = await asyncio.wait_for(self.runner.run(request), timeout=timeout)
+        except asyncio.TimeoutError:
+            agent_run = AgentRun(timed_out=True)
+        except Exception as exc:
+            agent_run = AgentRun(error=f"{type(exc).__name__}: {exc}")
+        if not agent_run.duration_s:
+            agent_run.duration_s = time.monotonic() - start
+        return agent_run
+
+    def save_trial(self, result: TrialResult, workdir: Path, run_dir: Path) -> None:
+        case_dir = run_dir / slugify(result.case)
+        case_dir.mkdir(parents=True, exist_ok=True)
+        name = f"{result.variant}-{result.trial + 1}"
+        result.artifact = f"{case_dir.name}/{name}.json"
+        (case_dir / f"{name}.json").write_text(
+            redact(result.model_dump_json(indent=2)), encoding="utf-8"
+        )
+        if self.options.keep_sandboxes:
+            shutil.copytree(workdir, case_dir / f"{name}.workdir", symlinks=True)
+
+
+def run_suite(
+    config: EvalConfig,
+    options: RunOptions | None = None,
+    runner: Runner | None = None,
+    judge: Judge | None = None,
+    on_trial: Callable[[TrialResult], None] | None = None,
+) -> SuiteResult:
+    """Synchronous entry point used by the CLI and tests."""
+    engine = Engine(config, options, runner=runner, judge=judge, on_trial=on_trial)
+    return asyncio.run(engine.run())

skillproof/errors.py

@@ -0,0 +1,72 @@
+"""Errors that are shown to the user.
+
+Every error carries a `hint`: one sentence telling the user exactly what to do
+next. The CLI prints the message and the hint and exits non-zero, without a
+traceback.
+"""
+
+from __future__ import annotations
+
+
+class SkillproofError(Exception):
+    def __init__(self, message: str, hint: str | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.hint = hint
+
+
+class ConfigError(SkillproofError):
+    """evals.yaml is missing, malformed, or points at things that don't exist."""
+
+
+class SkillError(SkillproofError):
+    """The skill folder or its SKILL.md is missing or malformed."""
+
+
+class RunnerNotReady(SkillproofError):
+    """A runner can't start: missing CLI, missing package, missing API key..."""
+
+
+class JudgeNotReady(SkillproofError):
+    """An llm_judge assertion is used but no judge can be built."""
+
+
+class RunAborted(SkillproofError):
+    """The run stopped early: the first runs all failed, or the user hit Ctrl-C.
+    Partial results are saved before this is raised."""
+
+
+# (pattern, what to do) for errors agents and APIs commonly report.
+ERROR_HINTS = [
+    (
+        r"401|unauthori[sz]ed|authenticat|invalid.{0,20}api.?key|x-api-key|not logged in|"
+        r"credentials",
+        "Your API key or login is missing or invalid. Check ANTHROPIC_API_KEY, OPENAI_API_KEY "
+        "or CODEX_API_KEY (or log in with `claude` / `codex login`).",
+    ),
+    (
+        r"credit|billing|quota|insufficient",
+        "Your API account is out of credits or quota. Check billing for the provider.",
+    ),
+    (
+        r"429|rate.?limit|overloaded|529|too many requests",
+        "The API is rate-limiting or overloaded. Lower --parallel (e.g. -p 1) and try again.",
+    ),
+    (
+        r"model.{0,40}(not.?found|does not exist|invalid|unknown|not supported)|404",
+        "The model name looks wrong. Check `model:` in evals.yaml.",
+    ),
+    (
+        r"enotfound|connection|network|name resolution|getaddrinfo|ssl",
+        "Network problem reaching the API. Check your connection, proxy and base_url.",
+    ),
+]
+
+
+def hint_for_error(text: str) -> str:
+    import re
+
+    for pattern, hint in ERROR_HINTS:
+        if re.search(pattern, text, re.IGNORECASE):
+            return hint
+    return "Open the saved transcripts for details."

skillproof/estimate.py

@@ -0,0 +1,83 @@
+"""Rough cost estimate before a run starts, so nobody burns $50 by accident.
+
+Per-run cost comes from your last run of the same skill with the same runner
+when there is one (agents like Claude Code report cost per run). Otherwise a
+deliberately rough default per runner is used; tasks vary 10x in cost, so the
+first estimate is a ballpark, and later ones learn from your real runs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from skillproof.engine import Engine
+from skillproof.results import RESULTS_FILE, SuiteResult
+
+# USD per agent run when there's no history: ballpark for a small coding task
+# (our example skills averaged ~$0.05 per Claude Code run on claude-opus-5-5).
+DEFAULT_COST_PER_RUN = {
+    "claude-code": 0.10,
+    "codex": 0.15,
+    "openai-compatible": 0.02,
+    "mock": 0.0,
+}
+# Ask before spending more than this (override with --confirm-above).
+CONFIRM_ABOVE_USD = 5.0
+
+
+@dataclass
+class Estimate:
+    runs: int
+    judge_calls: int
+    cost_per_run: float | None  # None: unknown runner and no history
+    source: str  # where cost_per_run came from, shown to the user
+
+    @property
+    def total_usd(self) -> float | None:
+        return None if self.cost_per_run is None else self.runs * self.cost_per_run
+
+    def describe(self) -> str:
+        text = f"{self.runs} agent runs"
+        if self.judge_calls:
+            text += f" + {self.judge_calls} judge calls"
+        if self.total_usd is None:
+            return text + " · cost unknown"
+        if self.total_usd == 0:
+            return text + " · no API cost"
+        return (
+            f"{text} · estimated ~${self.total_usd:.2f} "
+            f"(${self.cost_per_run:.3f}/run, {self.source})"
+        )
+
+
+def past_cost_per_run(out_dir: Path, skill_path: Path, runner: str) -> float | None:
+    """Mean reported cost per run in the newest saved run of this skill and runner."""
+    for results in sorted((out_dir / "runs").glob(f"*/{RESULTS_FILE}"), reverse=True):
+        try:
+            suite = SuiteResult.model_validate_json(results.read_text(encoding="utf-8"))
+        except ValueError:
+            continue  # unreadable or from an incompatible version
+        if Path(suite.skill_path) != skill_path or suite.runner != runner:
+            continue
+        costs = [
+            t.run.usage.cost_usd
+            for c in suite.cases
+            for t in c.trials
+            if t.run.usage.cost_usd is not None
+        ]
+        if costs:
+            return sum(costs) / len(costs)
+    return None
+
+
+def estimate(engine: Engine, out_dir: Path) -> Estimate:
+    jobs = engine.jobs()
+    judge_calls = sum(sum(a.type == "llm_judge" for a in job.case.assertions) for job in jobs)
+    runner = engine.runner.name
+    past = past_cost_per_run(out_dir, engine.config.skill, runner)
+    if past is not None:
+        return Estimate(len(jobs), judge_calls, past, "from your last run")
+    default = DEFAULT_COST_PER_RUN.get(runner)
+    source = f"rough default for {runner}" if default else "no estimate for this runner"
+    return Estimate(len(jobs), judge_calls, default, source)

skillproof/judges/__init__.py

@@ -0,0 +1,43 @@
+"""Building the judge for `llm_judge` assertions.
+
+By default the judge comes from a different vendor than the agent under test,
+so a model never grades its own family's homework.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from skillproof.errors import JudgeNotReady
+from skillproof.judges.base import Judge, Verdict
+from skillproof.judges.mock import MockJudge
+
+if TYPE_CHECKING:
+    from skillproof.config import JudgeConfig
+
+# agent vendor -> judge provider used when `judge.provider: auto`
+CROSS_VENDOR = {"anthropic": "openai", "openai": "anthropic"}
+
+
+def resolve_provider(config: JudgeConfig, agent_vendor: str) -> str:
+    if config.provider != "auto":
+        return config.provider
+    return CROSS_VENDOR.get(agent_vendor, "anthropic")
+
+
+def make_judge(config: JudgeConfig, agent_vendor: str) -> Judge:
+    provider = resolve_provider(config, agent_vendor)
+    if provider == "mock":
+        return MockJudge()
+    if provider == "anthropic":
+        from skillproof.judges.anthropic_judge import AnthropicJudge
+
+        return AnthropicJudge(config.model, config.api_key_env, config.base_url)
+    if provider == "openai":
+        from skillproof.judges.openai_judge import OpenAIJudge
+
+        return OpenAIJudge(config.model, config.api_key_env, config.base_url)
+    raise JudgeNotReady(f"Unknown judge provider {provider!r}.", hint="Use anthropic or openai.")
+
+
+__all__ = ["Judge", "MockJudge", "Verdict", "make_judge", "resolve_provider"]