skillproof 0.1.0__py3-none-any.whl

skillproof/results.py

@@ -0,0 +1,88 @@
+"""Result models. These are what gets saved to results.json and fed to reports."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from skillproof.assertions import AssertionResult
+from skillproof.errors import SkillproofError
+from skillproof.runners.base import AgentRun
+
+Variant = Literal["with", "without"]
+
+
+class TrialResult(BaseModel):
+    case: str
+    variant: Variant
+    trial: int  # 0-based
+    run: AgentRun
+    assertions: list[AssertionResult] = Field(default_factory=list)
+    artifact: str | None = None  # path of the saved transcript, relative to the run dir
+
+    @property
+    def errored(self) -> bool:
+        """Infrastructure failure: excluded from pass rates, reported separately."""
+        return self.run.error is not None
+
+    @property
+    def passed(self) -> bool:
+        return (
+            not self.errored and not self.run.timed_out and all(a.passed for a in self.assertions)
+        )
+
+
+class CaseResult(BaseModel):
+    name: str
+    should_trigger: bool = True
+    has_assertions: bool = True
+    trials: list[TrialResult] = Field(default_factory=list)
+
+    def variant(self, variant: Variant) -> list[TrialResult]:
+        return [t for t in self.trials if t.variant == variant]
+
+
+class SuiteResult(BaseModel):
+    skill_name: str
+    skill_path: str
+    runner: str
+    approximate: bool = False  # the runner only simulates skill loading
+    model: str | None = None
+    judge: str | None = None
+    started_at: datetime
+    finished_at: datetime | None = None
+    run_dir: str
+    complete: bool = True  # False when the run was interrupted or stopped early
+    cases: list[CaseResult] = Field(default_factory=list)
+
+
+RESULTS_FILE = "results.json"
+
+
+def load_suite(path: Path) -> SuiteResult:
+    """Load a saved run from its results.json or its run directory."""
+    file = path / RESULTS_FILE if path.is_dir() else path
+    if not file.is_file():
+        raise SkillproofError(
+            f"No {RESULTS_FILE} at {path}",
+            hint="Pass a run directory like .skillproof/runs/20261017-120000.",
+        )
+    return SuiteResult.model_validate_json(file.read_text(encoding="utf-8"))
+
+
+def find_latest_run(out_dir: Path, skill_path: Path | None = None) -> Path:
+    """The newest run directory under <out_dir>/runs, optionally for one skill."""
+    runs = sorted((out_dir / "runs").glob(f"*/{RESULTS_FILE}"), reverse=True)
+    for results in runs:
+        if skill_path is None:
+            return results.parent
+        if Path(load_suite(results).skill_path) == skill_path.resolve():
+            return results.parent
+    where = f" for {skill_path}" if skill_path else ""
+    raise SkillproofError(
+        f"No skillproof runs found in {out_dir / 'runs'}{where}.",
+        hint="Run `skillproof run <skill>` first, or pass --out if you saved runs elsewhere.",
+    )

skillproof/runners/__init__.py

@@ -0,0 +1,33 @@
+"""Runner registry.
+
+To add a runner: write a `Runner` subclass in this package and add one line to
+`RUNNERS` below. Runners are imported lazily so that an optional SDK for one
+agent is never required to use another.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+from skillproof.runners.base import AgentRun, Runner, RunRequest, ToolCall, Usage
+
+# runner name -> "module:ClassName"
+RUNNERS: dict[str, str] = {
+    "claude-code": "skillproof.runners.claude_code:ClaudeCodeRunner",
+    "codex": "skillproof.runners.codex:CodexRunner",
+    "openai-compatible": "skillproof.runners.openai_compat:OpenAICompatibleRunner",
+    "mock": "skillproof.runners.mock:MockRunner",
+}
+
+
+def get_runner_class(name: str) -> type[Runner]:
+    module_name, class_name = RUNNERS[name].split(":")
+    return getattr(importlib.import_module(module_name), class_name)
+
+
+def get_runner(name: str, options: dict[str, Any] | None = None) -> Runner:
+    return get_runner_class(name)(options)
+
+
+__all__ = ["RUNNERS", "AgentRun", "RunRequest", "Runner", "ToolCall", "Usage", "get_runner"]

skillproof/runners/base.py

@@ -0,0 +1,102 @@
+"""The Runner interface. Adding a new agent = one subclass of `Runner`.
+
+A runner gets a `RunRequest` (prompt, a sandbox directory with fixtures already
+copied in, and the skill or None for the baseline) and returns an `AgentRun`.
+
+Rules every runner must follow:
+  * Only touch `request.workdir` (and temp dirs you create yourself).
+  * When `request.skill` is None the agent must NOT be able to see the skill,
+    including any copy installed in the user's home config.
+  * Fill `triggered` from the transcript if you can tell, else leave it None.
+  * Infrastructure failures (API down, CLI crashed) can be raised or put in
+    `error`; either way the run is reported as errored, not failed.
+  * The engine enforces the timeout by cancelling `run()`. If you start a
+    subprocess, kill it in a `finally:` block.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skillproof.skill import Skill
+
+
+class ToolCall(BaseModel):
+    name: str
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: str | None = None
+
+
+class Usage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_write_tokens: int = 0
+    cost_usd: float | None = None  # only when the agent reports it
+
+    @property
+    def total_tokens(self) -> int:
+        return (
+            self.input_tokens
+            + self.output_tokens
+            + self.cache_read_tokens
+            + self.cache_write_tokens
+        )
+
+
+class AgentRun(BaseModel):
+    """Everything we captured from one agent run."""
+
+    final_text: str = ""
+    transcript: list[dict[str, Any]] = Field(default_factory=list)  # raw, runner-specific
+    tool_calls: list[ToolCall] = Field(default_factory=list)
+    usage: Usage = Field(default_factory=Usage)
+    duration_s: float = 0.0
+    num_turns: int | None = None
+    model: str | None = None  # the model that actually ran, when the agent reports it
+    triggered: bool | None = None  # None = this runner can't tell
+    timed_out: bool = False
+    error: str | None = None  # infrastructure error; the run is excluded from pass rates
+    warnings: list[str] = Field(default_factory=list)  # e.g. "skill not listed at startup"
+
+
+@dataclass
+class RunRequest:
+    prompt: str
+    workdir: Path  # fresh sandbox, fixtures already copied in
+    skill: Skill | None  # None = baseline run
+    skill_exclude: list[Path]  # pass to copy_skill() when installing the skill
+    model: str | None
+    timeout_s: float
+    case_name: str
+    trial: int  # 0-based
+
+
+class Runner(ABC):
+    name: ClassVar[str]
+    # Who makes the model. Used to pick a judge from a *different* vendor.
+    vendor: ClassVar[str]
+    # True when the runner only simulates skill loading (e.g. stuffing SKILL.md
+    # into the system prompt). Reports label these results as approximate.
+    approximate: ClassVar[bool] = False
+    # True when there is no sensible default model, so evals.yaml must set `model:`.
+    requires_model: ClassVar[bool] = False
+
+    class Options(BaseModel):
+        """Runner-specific settings from `runner_options:` in evals.yaml."""
+
+        model_config = ConfigDict(extra="forbid")
+
+    def __init__(self, options: dict[str, Any] | None = None) -> None:
+        self.options = self.Options.model_validate(options or {})
+
+    def check_ready(self) -> None:  # noqa: B027 - optional hook, no-op by default
+        """Raise RunnerNotReady (with a hint) if the runner can't start."""
+
+    @abstractmethod
+    async def run(self, request: RunRequest) -> AgentRun: ...

skillproof/runners/claude_code.py

@@ -0,0 +1,117 @@
+"""Claude Code runner, via the Claude Agent SDK (`claude-agent-sdk` on PyPI).
+
+Isolation:
+  * The agent runs in the sandbox (`cwd`), with only *project* settings loaded
+    (`setting_sources=["project"]`). Your personal ~/.claude skills, CLAUDE.md,
+    settings and plugins are not loaded, for both variants.
+  * With the skill: it is copied to `<sandbox>/.claude/skills/<name>/` and is
+    the only skill the model may invoke (`skills=[name]`).
+  * Baseline: nothing is installed and no skill may be invoked (`skills=[]`).
+  * Sessions are not saved to ~/.claude (`--no-session-persistence`).
+
+Trigger detection: a `Skill` tool call naming the skill (see parsers/claude_code.py).
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from skillproof.errors import RunnerNotReady
+from skillproof.parsers.claude_code import parse_transcript, to_dict
+from skillproof.runners.base import AgentRun, Runner, RunRequest
+from skillproof.skill import copy_skill
+
+MAX_STDERR_LINES = 50
+
+
+def _hide_reaped_child_warning(record: logging.LogRecord) -> bool:
+    """CPython 3.10/3.11 race: when a finished subprocess object is garbage-collected,
+    a later `subprocess.Popen` (e.g. a `command` assertion) can reap it before
+    asyncio's child watcher does. asyncio then logs "Unknown child process pid ...,
+    will report returncode 255". It only affects Claude Code processes that already
+    delivered their result (which is what skillproof uses), so hide the noise."""
+    return "will report returncode 255" not in record.getMessage()
+
+
+logging.getLogger("asyncio").addFilter(_hide_reaped_child_warning)
+
+
+class ClaudeCodeRunner(Runner):
+    name = "claude-code"
+    vendor = "anthropic"
+
+    class Options(BaseModel):
+        model_config = ConfigDict(extra="forbid")
+
+        # The sandbox is a throwaway temp dir, so by default the agent may edit
+        # files and run commands without prompts, like a real unattended session.
+        permission_mode: Literal["default", "acceptEdits", "bypassPermissions", "dontAsk"] = (
+            "bypassPermissions"
+        )
+        max_turns: int | None = None
+        max_budget_usd: float | None = None  # per run; the SDK stops the run past this
+        effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
+        tools: list[str] | None = None  # restrict built-in tools, e.g. ["Read", "Bash"]
+        cli_path: str | None = None  # use this `claude` binary instead of the bundled one
+
+    options: Options
+
+    def check_ready(self) -> None:
+        try:
+            import claude_agent_sdk  # noqa: F401
+        except ImportError as exc:
+            raise RunnerNotReady(
+                "The claude-code runner needs the Claude Agent SDK.",
+                hint='Install it with: pip install "claude-agent-sdk"',
+            ) from exc
+        cli = self.options.cli_path
+        if cli and not (Path(cli).is_file() or shutil.which(cli)):
+            raise RunnerNotReady(
+                f"Claude Code CLI not found at {cli!r}.",
+                hint="Fix `runner_options.cli_path`, or remove it to use the bundled CLI.",
+            )
+
+    async def run(self, request: RunRequest) -> AgentRun:
+        from claude_agent_sdk import ClaudeAgentOptions, query
+
+        skill_name = request.skill.name if request.skill else None
+        if request.skill is not None:
+            dest = request.workdir / ".claude" / "skills" / request.skill.name
+            copy_skill(request.skill, dest, exclude=request.skill_exclude)
+
+        stderr: list[str] = []
+        opts = self.options
+        options = ClaudeAgentOptions(
+            cwd=request.workdir,
+            setting_sources=["project"],
+            skills=[skill_name] if skill_name else [],
+            permission_mode=opts.permission_mode,
+            model=request.model,
+            max_turns=opts.max_turns,
+            max_budget_usd=opts.max_budget_usd,
+            effort=opts.effort,
+            tools=opts.tools,
+            cli_path=opts.cli_path,
+            extra_args={"no-session-persistence": None},
+            stderr=lambda line: stderr.append(line) if len(stderr) < MAX_STDERR_LINES else None,
+        )
+
+        transcript: list[dict[str, Any]] = []
+        stream = query(prompt=request.prompt, options=options)
+        try:
+            async for message in stream:
+                transcript.append(to_dict(message))
+        except Exception as exc:
+            # The SDK raises after yielding an error result; the result says it all.
+            if not any(m.get("_type") == "ResultMessage" for m in transcript):
+                detail = "\n".join(stderr[-5:])
+                raise RuntimeError(f"{exc}\n{detail}".strip()) from exc
+        finally:
+            await stream.aclose()
+
+        return parse_transcript(transcript, skill_name)

skillproof/runners/codex.py

@@ -0,0 +1,133 @@
+"""Codex CLI runner, via `codex exec --json` (non-interactive mode).
+
+Isolation (each run):
+  * A fresh CODEX_HOME and HOME, so your ~/.codex config, AGENTS.md and personal
+    skills (~/.codex/skills, ~/.agents/skills) are not loaded for either variant.
+    Auth is carried over: CODEX_API_KEY / OPENAI_API_KEY from the environment,
+    else your `codex login` auth.json is copied into the temp CODEX_HOME.
+  * Codex's bundled system skills are disabled (`skills.bundled.enabled=false`).
+  * With the skill: it is copied to `<sandbox>/.agents/skills/<name>/`, where
+    Codex discovers repo skills. Baseline: nothing is installed.
+  * `--ephemeral`: no session files are written.
+
+Trigger detection: a shell command touching `skills/<name>/` (see parsers/codex.py).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skillproof.errors import RunnerNotReady
+from skillproof.parsers.codex import parse_events, parse_jsonl
+from skillproof.runners.base import AgentRun, Runner, RunRequest
+from skillproof.skill import copy_skill
+
+STREAM_LIMIT = 64 * 1024 * 1024  # JSONL lines carry whole command outputs
+
+
+def user_codex_home() -> Path:
+    return Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex")
+
+
+class CodexRunner(Runner):
+    name = "codex"
+    vendor = "openai"
+
+    class Options(BaseModel):
+        model_config = ConfigDict(extra="forbid")
+
+        sandbox: Literal["read-only", "workspace-write", "danger-full-access"] = "workspace-write"
+        cli_path: str = "codex"
+        # Extra `-c key=value` overrides, e.g. {"model_reasoning_effort": "low"}.
+        config: dict[str, str] = Field(default_factory=dict)
+
+    options: Options
+
+    def check_ready(self) -> None:
+        if shutil.which(self.options.cli_path) is None:
+            raise RunnerNotReady(
+                f"Codex CLI not found ({self.options.cli_path!r}).",
+                hint="Install it with: npm install -g @openai/codex",
+            )
+        has_key = os.environ.get("CODEX_API_KEY") or os.environ.get("OPENAI_API_KEY")
+        if not has_key and not (user_codex_home() / "auth.json").is_file():
+            raise RunnerNotReady(
+                "No credentials for Codex.",
+                hint="export CODEX_API_KEY=... (or OPENAI_API_KEY), or run `codex login`.",
+            )
+
+    def command(self, request: RunRequest) -> list[str]:
+        cmd = [
+            self.options.cli_path,
+            "exec",
+            "--json",
+            "--ephemeral",
+            "--skip-git-repo-check",
+            "--ignore-user-config",
+            "--sandbox",
+            self.options.sandbox,
+            "-C",
+            str(request.workdir),
+            "-c",
+            "skills.bundled.enabled=false",
+            # Otherwise a network problem makes codex retry forever.
+            "--disable",
+            "unbounded_connection_retries",
+        ]
+        if request.model:
+            cmd += ["-m", request.model]
+        for key, value in self.options.config.items():
+            cmd += ["-c", f"{key}={value}"]
+        return [*cmd, request.prompt]
+
+    def environment(self, home: Path) -> dict[str, str]:
+        codex_home = home / "codex_home"
+        codex_home.mkdir(parents=True)
+        env = dict(os.environ)
+        key = os.environ.get("CODEX_API_KEY") or os.environ.get("OPENAI_API_KEY")
+        if key:
+            env["CODEX_API_KEY"] = key  # codex exec reads CODEX_API_KEY, not OPENAI_API_KEY
+        elif (user_codex_home() / "auth.json").is_file():
+            shutil.copy2(user_codex_home() / "auth.json", codex_home / "auth.json")
+        env["CODEX_HOME"] = str(codex_home)
+        env["HOME"] = str(home)  # ~/.agents/skills is looked up under $HOME
+        return env
+
+    async def run(self, request: RunRequest) -> AgentRun:
+        skill_name = request.skill.name if request.skill else None
+        if request.skill is not None:
+            dest = request.workdir / ".agents" / "skills" / request.skill.name
+            copy_skill(request.skill, dest, exclude=request.skill_exclude)
+
+        home = Path(tempfile.mkdtemp(prefix="skillproof-codex-home-"))
+        proc = None
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                *self.command(request),
+                cwd=request.workdir,
+                env=self.environment(home),
+                stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                limit=STREAM_LIMIT,
+            )
+            stdout, stderr = await proc.communicate()
+        finally:
+            if proc is not None and proc.returncode is None:  # cancelled by the timeout
+                proc.kill()
+                await proc.wait()
+            shutil.rmtree(home, ignore_errors=True)
+
+        run = parse_events(parse_jsonl(stdout.decode(errors="replace")), skill_name)
+        run.model = request.model
+        if run.error and proc.returncode != 0:
+            tail = stderr.decode(errors="replace").strip().splitlines()[-3:]
+            run.error = "\n".join([run.error, *tail]).strip()
+        return run

skillproof/runners/mock.py

@@ -0,0 +1,91 @@
+"""A runner that returns scripted results. Used by our tests; never calls an API.
+
+runner: mock
+runner_options:
+  default:                  # used for cases without their own script
+    with: {text: "...", triggered: true, output_tokens: 300}
+    without: {text: "..."}
+  cases:
+    "summarizes stack trace":
+      with:                 # a list cycles through trials
+        - {text: "NullPointerException at line 3", files: {fix.patch: "..."}}
+        - {text: "not sure"}
+      without: {text: "not sure"}
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skillproof.runners.base import AgentRun, Runner, RunRequest, ToolCall, Usage
+
+
+class MockResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    text: str = ""
+    triggered: bool | None = None  # default: True with the skill, False without
+    files: dict[str, str] = Field(default_factory=dict)  # written into the sandbox
+    tool_calls: list[ToolCall] = Field(default_factory=list)
+    input_tokens: int = 1000
+    output_tokens: int = 200
+    duration_s: float = 1.0
+    timed_out: bool = False
+    error: str | None = None
+
+
+class MockScript(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    with_: MockResponse | list[MockResponse] = Field(default_factory=MockResponse, alias="with")
+    without: MockResponse | list[MockResponse] = Field(default_factory=MockResponse)
+
+
+class MockRunner(Runner):
+    name = "mock"
+    vendor = "mock"
+
+    class Options(BaseModel):
+        model_config = ConfigDict(extra="forbid")
+
+        default: MockScript = Field(default_factory=MockScript)
+        cases: dict[str, MockScript] = Field(default_factory=dict)
+
+    options: Options
+
+    def __init__(self, options: dict[str, Any] | None = None) -> None:
+        super().__init__(options)
+        self.requests: list[RunRequest] = []  # for tests to inspect
+
+    async def run(self, request: RunRequest) -> AgentRun:
+        self.requests.append(request)
+        script = self.options.cases.get(request.case_name, self.options.default)
+        scripted = script.with_ if request.skill is not None else script.without
+        if isinstance(scripted, list):
+            scripted = scripted[request.trial % len(scripted)]
+
+        for rel_path, content in scripted.files.items():
+            target = request.workdir / rel_path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content, encoding="utf-8")
+
+        triggered = scripted.triggered
+        if triggered is None:
+            triggered = request.skill is not None
+
+        return AgentRun(
+            final_text=scripted.text,
+            transcript=[
+                {"role": "user", "content": request.prompt},
+                {"role": "assistant", "content": scripted.text},
+            ],
+            tool_calls=scripted.tool_calls,
+            usage=Usage(input_tokens=scripted.input_tokens, output_tokens=scripted.output_tokens),
+            duration_s=scripted.duration_s,
+            num_turns=1,
+            triggered=triggered,
+            timed_out=scripted.timed_out,
+            error=scripted.error,
+        )