agent-skill-eval 0.2.1__py3-none-any.whl

skill_eval/harnesses/__init__.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import time
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional
+
+from skill_eval.models import AgentType, TimingData
+
+DEFAULT_TIMEOUT_SECONDS = 600
+DEFAULT_MAX_RETRIES = 1
+RETRY_BACKOFF_SECONDS = 2.0
+
+
+class AgentHarness(ABC):
+    agent_type: AgentType
+    # Environment variable each agent CLI reads its API base URL from.
+    base_url_env: Optional[str] = None
+
+    def __init__(
+        self,
+        workspace: Path,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        timeout: Optional[int] = None,
+        max_retries: Optional[int] = None,
+    ):
+        self.workspace = workspace
+        self.model = model
+        self.base_url = base_url
+        self.timeout = timeout or int(os.environ.get("SKILL_EVAL_AGENT_TIMEOUT", DEFAULT_TIMEOUT_SECONDS))
+        if max_retries is None:
+            max_retries = int(os.environ.get("SKILL_EVAL_AGENT_RETRIES", DEFAULT_MAX_RETRIES))
+        self.max_retries = max_retries
+
+    @abstractmethod
+    def build_command(self, prompt: str, output_dir: Path) -> list[str]:
+        pass
+
+    @abstractmethod
+    def parse_output(self, stdout: str, stderr: str) -> tuple[str, TimingData]:
+        pass
+
+    def _build_env(self) -> Optional[dict[str, str]]:
+        if self.base_url and self.base_url_env:
+            env = dict(os.environ)
+            env[self.base_url_env] = self.base_url
+            return env
+        return None
+
+    def _workspace_fingerprint(self) -> Optional[tuple[str, str]]:
+        """(HEAD sha, porcelain status) of the workspace, or None outside git.
+
+        Used to refuse retries after a failed attempt that already mutated
+        the workspace — re-running there would grade the union of both
+        attempts' side effects.
+        """
+        try:
+            head = subprocess.run(
+                ["git", "rev-parse", "HEAD"],
+                cwd=self.workspace,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                timeout=15,
+                check=False,
+            )
+            status = subprocess.run(
+                ["git", "status", "--porcelain"],
+                cwd=self.workspace,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                timeout=15,
+                check=False,
+            )
+        except (subprocess.SubprocessError, OSError):
+            return None
+        if head.returncode != 0 or status.returncode != 0:
+            return None
+        return (head.stdout.strip(), status.stdout)
+
+    def run(self, prompt: str, output_dir: Path) -> tuple[str, TimingData, str, str]:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        cmd = self.build_command(prompt, output_dir)
+        env = self._build_env()
+
+        attempts = self.max_retries + 1
+        stdout = ""
+        stderr = ""
+        exit_code: Optional[int] = None
+        timed_out = False
+        retries_used = 0
+        start = time.time()
+        fingerprint_before = self._workspace_fingerprint()
+
+        for attempt in range(attempts):
+            if attempt > 0:
+                retries_used = attempt
+                time.sleep(RETRY_BACKOFF_SECONDS * attempt)
+            try:
+                result = subprocess.run(
+                    cmd,
+                    cwd=self.workspace,
+                    capture_output=True,
+                    text=True,
+                    encoding="utf-8",
+                    timeout=self.timeout,
+                    env=env,
+                )
+                stdout = result.stdout or ""
+                stderr = result.stderr or ""
+                exit_code = result.returncode
+                timed_out = False
+                if result.returncode == 0:
+                    break
+            except subprocess.TimeoutExpired as e:
+                stdout = (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or "")
+                stderr = (e.stderr or b"").decode() if isinstance(e.stderr, bytes) else (e.stderr or "")
+                stderr += f"\n[skill-eval] agent timed out after {self.timeout}s (attempt {attempt + 1}/{attempts})"
+                exit_code = None
+                timed_out = True
+
+            # Only retry into a pristine workspace: if the failed attempt
+            # already committed/wrote files, a second run would produce a
+            # graded union of both attempts.
+            if attempt < attempts - 1 and fingerprint_before is not None:
+                if self._workspace_fingerprint() != fingerprint_before:
+                    stderr += "\n[skill-eval] not retrying: the failed attempt modified the workspace"
+                    break
+
+        duration_ms = int((time.time() - start) * 1000)
+
+        final_output, timing = self.parse_output(stdout, stderr)
+        if timing.duration_ms == 0:
+            timing.duration_ms = duration_ms
+        timing.exit_code = exit_code
+        timing.timed_out = timed_out
+        timing.retries = retries_used
+
+        with open(output_dir / "stdout.log", "w", encoding="utf-8") as f:
+            f.write(stdout)
+        with open(output_dir / "stderr.log", "w", encoding="utf-8") as f:
+            f.write(stderr)
+
+        return final_output, timing, stdout, stderr
+
+
+class OpenCodeHarness(AgentHarness):
+    agent_type = AgentType.OPENCODE
+    base_url_env = "OPENAI_BASE_URL"
+
+    def build_command(self, prompt: str, output_dir: Path) -> list[str]:
+        cmd = [
+            "opencode",
+            "run",
+            "--dir",
+            str(self.workspace),
+            "--format",
+            "json",
+            "--dangerously-skip-permissions",
+        ]
+        if self.model:
+            cmd.extend(["--model", self.model])
+        cmd.append(prompt)
+        return cmd
+
+    def parse_output(self, stdout: str, stderr: str) -> tuple[str, TimingData]:
+        timing = TimingData()
+        messages = []
+
+        for line in stdout.strip().split("\n"):
+            if not line.strip():
+                continue
+            try:
+                event = json.loads(line)
+                etype = event.get("type", "")
+
+                if etype == "text":
+                    part = event.get("part", {})
+                    text = part.get("text", "")
+                    if text:
+                        messages.append(text)
+
+                elif etype == "step_finish":
+                    part = event.get("part", {})
+                    tokens = part.get("tokens", {})
+                    timing.input_tokens += tokens.get("input", 0)
+                    timing.output_tokens += tokens.get("output", 0)
+                    cache = tokens.get("cache", {})
+                    timing.cached_tokens += cache.get("read", 0)
+
+            except json.JSONDecodeError:
+                continue
+
+        timing.total_tokens = timing.input_tokens + timing.output_tokens
+        return "\n".join(messages), timing
+
+
+class ClaudeCodeHarness(AgentHarness):
+    agent_type = AgentType.CLAUDE_CODE
+    base_url_env = "ANTHROPIC_BASE_URL"
+
+    def build_command(self, prompt: str, output_dir: Path) -> list[str]:
+        cmd = [
+            "claude",
+            "-p",
+            "--output-format",
+            "json",
+            "--dangerously-skip-permissions",
+        ]
+        if self.model:
+            cmd.extend(["--model", self.model])
+        cmd.append(prompt)
+        return cmd
+
+    def parse_output(self, stdout: str, stderr: str) -> tuple[str, TimingData]:
+        timing = TimingData()
+        final_text = ""
+
+        try:
+            data = json.loads(stdout)
+            if isinstance(data, dict):
+                final_text = data.get("result", "")
+                usage = data.get("usage", {})
+                timing.input_tokens = usage.get("input_tokens", 0)
+                timing.output_tokens = usage.get("output_tokens", 0)
+                timing.cached_tokens = usage.get("cache_read_input_tokens", 0)
+                timing.total_tokens = timing.input_tokens + timing.output_tokens
+                duration_s = data.get("duration_ms", 0)
+                if duration_s:
+                    timing.duration_ms = duration_s
+        except json.JSONDecodeError:
+            final_text = stdout
+
+        return final_text, timing
+
+
+class CodexHarness(AgentHarness):
+    agent_type = AgentType.CODEX
+    base_url_env = "OPENAI_BASE_URL"
+
+    def build_command(self, prompt: str, output_dir: Path) -> list[str]:
+        cmd = [
+            "codex",
+            "exec",
+            "--json",
+            "--full-auto",
+        ]
+        if self.model:
+            cmd.extend(["--model", self.model])
+        cmd.append(prompt)
+        return cmd
+
+    def parse_output(self, stdout: str, stderr: str) -> tuple[str, TimingData]:
+        timing = TimingData()
+        messages = []
+
+        for line in stdout.strip().split("\n"):
+            if not line.strip():
+                continue
+            try:
+                event = json.loads(line)
+                etype = event.get("type", "")
+
+                if etype == "item.completed":
+                    item = event.get("item", {})
+                    if item.get("type") == "agent_message":
+                        messages.append(item.get("text", ""))
+
+                elif etype == "turn.completed":
+                    usage = event.get("usage", {})
+                    timing.input_tokens += usage.get("input_tokens", 0)
+                    timing.output_tokens += usage.get("output_tokens", 0)
+                    timing.cached_tokens += usage.get("cached_input_tokens", 0)
+
+            except json.JSONDecodeError:
+                continue
+
+        timing.total_tokens = timing.input_tokens + timing.output_tokens
+        return "\n".join(messages), timing
+
+
+class FakeHarness(AgentHarness):
+    agent_type = AgentType.FAKE
+
+    def build_command(self, prompt: str, output_dir: Path) -> list[str]:
+        return ["fake-agent", prompt]
+
+    def parse_output(self, stdout: str, stderr: str) -> tuple[str, TimingData]:
+        timing = TimingData(total_tokens=1, input_tokens=1, duration_ms=1)
+        return stdout, timing
+
+    def run(self, prompt: str, output_dir: Path) -> tuple[str, TimingData, str, str]:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        skill_installed = any((self.workspace / ".fake" / "skills").glob("*/SKILL.md"))
+        if skill_installed and ("format" in prompt.lower() or "$format-json" in prompt.lower()):
+            final_output = '{\n  "formatted": true,\n  "status": "formatted-json-ok"\n}'
+        else:
+            final_output = "fake-agent baseline output: no formatting performed"
+
+        stdout = final_output
+        stderr = ""
+        timing = TimingData(
+            total_tokens=1, input_tokens=1, output_tokens=0, cached_tokens=0, duration_ms=1, exit_code=0
+        )
+
+        with open(output_dir / "stdout.log", "w", encoding="utf-8") as f:
+            f.write(stdout)
+        with open(output_dir / "stderr.log", "w", encoding="utf-8") as f:
+            f.write(stderr)
+
+        return final_output, timing, stdout, stderr
+
+
+HARNESSES: dict[AgentType, type[AgentHarness]] = {
+    AgentType.OPENCODE: OpenCodeHarness,
+    AgentType.CLAUDE_CODE: ClaudeCodeHarness,
+    AgentType.CODEX: CodexHarness,
+    AgentType.FAKE: FakeHarness,
+}
+
+
+def get_harness(
+    agent_type: AgentType,
+    workspace: Path,
+    model: Optional[str] = None,
+    base_url: Optional[str] = None,
+    timeout: Optional[int] = None,
+    max_retries: Optional[int] = None,
+) -> AgentHarness:
+    cls = HARNESSES[agent_type]
+    return cls(workspace, model, base_url=base_url, timeout=timeout, max_retries=max_retries)

skill_eval/models.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class AgentType(str, Enum):
+    OPENCODE = "opencode"
+    CLAUDE_CODE = "claude-code"
+    CODEX = "codex"
+    FAKE = "fake"
+
+
+class EvalCase(BaseModel):
+    id: int | str
+    prompt: str
+    expected_output: str
+    files: list[str] = Field(default_factory=list)
+    assertions: list[str] = Field(default_factory=list)
+    should_trigger: bool = True
+    force_skill_invocation: bool = False
+    stage_files: bool = False
+
+    @model_validator(mode="after")
+    def validate_contradiction(self) -> EvalCase:
+        if self.force_skill_invocation and not self.should_trigger:
+            raise ValueError("Contradiction: force_skill_invocation=True cannot be set when should_trigger=False.")
+        return self
+
+
+class EvalSuite(BaseModel):
+    skill_name: str
+    evals: list[EvalCase]
+
+
+class GitStateSnapshot(BaseModel):
+    local_branches: list[str] = Field(default_factory=list)
+    remote_branches: list[str] = Field(default_factory=list)
+    current_branch: str = ""
+    head_sha: str = ""
+    commit_count: int = 0
+    commits: list[str] = Field(default_factory=list)
+    remote_names: list[str] = Field(default_factory=list)
+    open_prs: list[dict] = Field(default_factory=list)
+    branch_heads: dict[str, str] = Field(default_factory=dict)
+    remote_branch_heads: dict[str, str] = Field(default_factory=dict)
+    commit_shas: list[str] = Field(default_factory=list)
+
+
+class RunMeta(BaseModel):
+    eval_id: int | str
+    agent: str
+    with_skill: bool
+    iteration: int = 1
+    skill_name: str = ""
+    source_repo: Optional[str] = None
+    run_id: str = ""
+    run_index: int = 1
+
+
+class CleanupManifest(BaseModel):
+    source_repo: Optional[str] = None
+    source_repo_slug: Optional[str] = None
+    remote_branches: list[str] = Field(default_factory=list)
+    pr_numbers: list[int] = Field(default_factory=list)
+    workspaces: list[str] = Field(default_factory=list)
+
+
+class TimingData(BaseModel):
+    total_tokens: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cached_tokens: int = 0
+    duration_ms: int = 0
+    exit_code: Optional[int] = None
+    timed_out: bool = False
+    retries: int = 0
+
+
+class AssertionResult(BaseModel):
+    text: str
+    passed: bool
+    evidence: str
+    method: str = "deterministic"
+    skipped: bool = False
+
+
+class GradingResult(BaseModel):
+    assertion_results: list[AssertionResult]
+    summary: GradingSummary
+
+
+class GradingSummary(BaseModel):
+    passed: int
+    failed: int
+    total: int
+    pass_rate: float
+    skipped: int = 0
+
+
+GradingResult.model_rebuild()
+
+
+class RunConfig(BaseModel):
+    skill_path: Path
+    evals_path: Path
+    workspace_path: Path
+    agents: list[AgentType]
+    concurrency: int = 1
+    with_baseline: bool = True
+    grader_model: str = "deepseek/deepseek-v4-flash"
+    grader_base_url: Optional[str] = None
+
+
+class BenchmarkStats(BaseModel):
+    pass_rate: StatsPair
+    time_seconds: StatsPair
+    tokens: StatsPair
+    # Fraction of runs where ALL assertions passed (pass@1 estimate).
+    full_pass_rate: float = 0.0
+    # Fraction of evals where at least one of k runs fully passed.
+    pass_at_k: float = 0.0
+    k: int = 1
+
+
+class StatsPair(BaseModel):
+    mean: float
+    stddev: float
+
+
+class BenchmarkResult(BaseModel):
+    run_summary: dict[str, BenchmarkStats]
+    # Per-agent with_skill - without_skill deltas, keyed by agent value.
+    deltas: dict[str, DeltaStats] = Field(default_factory=dict)
+    # Kept for backward compatibility: equals the single agent's delta when
+    # exactly one agent was run, otherwise None.
+    delta: Optional[DeltaStats] = None
+
+
+class DeltaStats(BaseModel):
+    pass_rate: float
+    time_seconds: float
+    tokens: float