agent-skill-lab 0.1.0__py3-none-any.whl

agent_skill_lab/__init__.py

@@ -0,0 +1,3 @@
+"""Agent Skill Lab - reproducible A/B testing for Agent Skills."""
+
+__version__ = "0.1.0"

agent_skill_lab/assertions.py

@@ -0,0 +1,168 @@
+"""Deterministic checks run against the agent's workspace after a task.
+
+Everything here is deliberately non-LLM: an assertion either passes or it does
+not, and the same workspace always yields the same verdict.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import Assertion
+
+KNOWN_TYPES = {"command", "file_exists", "file_absent", "file_contains", "file_changed", "file_unchanged"}
+
+# Guards state a constraint that already holds before the agent runs and must
+# still hold afterwards ("don't touch the tests"). Unlike the other types they
+# are expected to pass on a pristine fixture, so a dry run must not call them
+# trivial.
+GUARD_TYPES = {"file_unchanged", "file_absent"}
+
+
+@dataclass
+class AssertionResult:
+    label: str
+    passed: bool
+    weight: float
+    required: bool
+    detail: str = ""
+    kind: str = ""
+    guard: bool = False
+
+    @property
+    def is_guard(self) -> bool:
+        return self.guard
+
+
+def _confine(workdir: Path, rel: str) -> Path | None:
+    """Resolve `rel` inside `workdir`, refusing anything that escapes it.
+
+    A task.yaml is authored by the skill author, not by whoever runs the
+    benchmark. Without this, `path: ../../../.ssh/id_rsa` in a downloaded skill
+    lets its assertions read host files (leaking one bit per pattern), and an
+    absolute path escapes the sandbox entirely.
+    """
+    try:
+        target = (workdir / rel).resolve()
+        target.relative_to(workdir.resolve())
+    except (ValueError, OSError):
+        return None
+    return target
+
+
+def _git_changed(workdir: Path, rel: str) -> bool | None:
+    """True/False if git can answer, None if the workspace is not a git repo."""
+    try:
+        proc = subprocess.run(
+            ["git", "status", "--porcelain", "--", rel],
+            cwd=workdir,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if proc.returncode != 0:
+        return None
+    return bool(proc.stdout.strip())
+
+
+def check(assertion: Assertion, workdir: Path, *, timeout: int = 300) -> AssertionResult:
+    spec = assertion.spec
+    kind = assertion.type
+    label = assertion.label()
+
+    # A guard asserts that something stays absent: file_absent, file_unchanged,
+    # or a file_contains looking for a bad pattern (expect: false). All of these
+    # legitimately hold on the pristine fixture, so a dry run must not call them
+    # trivial.
+    is_guard = kind in GUARD_TYPES or (kind == "file_contains" and not spec.get("expect", True))
+
+    def result(passed: bool, detail: str = "") -> AssertionResult:
+        return AssertionResult(
+            label, passed, assertion.weight, assertion.required, detail, kind, is_guard
+        )
+
+    if kind == "command":
+        command = spec.get("run")
+        if not command:
+            return result(False, "assertion has no 'run' command")
+        expected = int(spec.get("expect_exit", 0))
+        try:
+            proc = subprocess.run(
+                command,
+                shell=True,
+                cwd=workdir,
+                capture_output=True,
+                text=True,
+                timeout=int(spec.get("timeout_seconds", timeout)),
+            )
+        except subprocess.TimeoutExpired:
+            return result(False, "command timed out")
+        except OSError as exc:
+            return result(False, f"command failed to start: {exc}")
+        tail = (proc.stdout + proc.stderr).strip().splitlines()
+        detail = f"exit {proc.returncode} (expected {expected})"
+        if proc.returncode != expected and tail:
+            detail += " | " + tail[-1][:160]
+        return result(proc.returncode == expected, detail)
+
+    rel = spec.get("path")
+    if not rel:
+        return result(False, f"'{kind}' assertion has no 'path'")
+    target = _confine(workdir, rel)
+    if target is None:
+        return result(False, f"path '{rel}' escapes the workspace and is refused")
+
+    if kind == "file_exists":
+        return result(target.is_file(), "" if target.is_file() else "file not found")
+
+    if kind == "file_absent":
+        return result(not target.exists(), "" if not target.exists() else "file still present")
+
+    if kind == "file_contains":
+        pattern = spec.get("pattern")
+        if not pattern:
+            return result(False, "file_contains needs a 'pattern'")
+        if not target.is_file():
+            return result(False, "file not found")
+        text = target.read_text(encoding="utf-8", errors="replace")
+        flags = 0 if spec.get("case_sensitive", True) else re.IGNORECASE
+        found = re.search(str(pattern), text, flags) is not None
+        want = bool(spec.get("expect", True))
+        return result(found == want, "" if found == want else f"pattern {'not ' if want else ''}found")
+
+    if kind in {"file_changed", "file_unchanged"}:
+        # Use the path relative to workdir so git sees a repo-internal target;
+        # `target` is already confined above.
+        changed = _git_changed(workdir, target.relative_to(workdir.resolve()).as_posix())
+        if changed is None:
+            return result(False, "cannot inspect changes: workspace is not a git repo")
+        want_changed = kind == "file_changed"
+        ok = changed == want_changed
+        return result(ok, "" if ok else ("file was modified" if want_changed is False else "file was not modified"))
+
+    return result(False, f"unknown assertion type '{kind}'")
+
+
+def run_all(assertions: list[Assertion], workdir: Path, *, timeout: int = 300) -> list[AssertionResult]:
+    return [check(a, workdir, timeout=timeout) for a in assertions]
+
+
+def score(results: list[AssertionResult]) -> float:
+    """Weighted fraction of assertions that passed (0.0 - 1.0)."""
+    total = sum(r.weight for r in results)
+    if total <= 0:
+        return 0.0
+    return sum(r.weight for r in results if r.passed) / total
+
+
+def succeeded(results: list[AssertionResult]) -> bool:
+    """A task counts as solved only when every required assertion passed."""
+    required = [r for r in results if r.required]
+    if not required:
+        return False
+    return all(r.passed for r in required)

agent_skill_lab/cli.py

@@ -0,0 +1,216 @@
+"""Command line entry point: agent-skill {init,validate,test,report}."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__, reporter, scaffold, storage
+from .config import ConfigError, load_skill, load_suite
+from .runner import AgentError, check_agent_ready, dry_run, estimate, run_suite
+from .scoring import Summary, summarise
+from .validator import validate
+
+
+def _out(message: str = "") -> None:
+    print(message, flush=True)
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    target = Path(args.path)
+    _out(f"Scaffolding skill in {target.resolve()}")
+    for line in scaffold.init_skill(target, force=args.force):
+        _out(line)
+    _out()
+    _out("Next: fill in SKILL.md, then replace the TODOs in evals/task-001.yaml.")
+    _out("`agent-skill validate` fails while placeholders remain - that is on purpose.")
+    return 0
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    report = validate(Path(args.path), include_suite=not args.skill_only)
+    _out(reporter.render_validation(report, f"VALIDATE  {Path(args.path).resolve()}"))
+    return 0 if report.passed else 1
+
+
+def cmd_test(args: argparse.Namespace) -> int:
+    skill_dir = Path(args.path)
+
+    report = validate(skill_dir)
+    if not report.passed:
+        _out(reporter.render_validation(report, f"VALIDATE  {skill_dir.resolve()}"))
+        _out()
+        _out("Fix the errors above before benchmarking. Nothing was run.")
+        return 1
+    if report.warnings and not args.quiet:
+        for warning in report.warnings:
+            _out(f"  [warn] {warning.message}")
+        _out()
+
+    suite = load_suite(
+        skill_dir,
+        repeats=args.repeats,
+        model=args.model,
+        only_tasks=args.task or None,
+    )
+
+    if args.dry_run:
+        _out(reporter.render_dry_run(dry_run(suite)))
+        return 0
+
+    if args.estimate:
+        _out(reporter.render_estimate(estimate(suite), model=suite.agent.model))
+        return 0
+
+    not_ready = check_agent_ready(suite.agent)
+    if not_ready:
+        _out(f"CANNOT RUN: {not_ready}")
+        return 2
+
+    est = estimate(suite)
+    _out(reporter.render_estimate(est, model=suite.agent.model))
+
+    total = len(suite.tasks) * suite.repeats * 2
+    _out(f"Skill:  {suite.skill.name}  ({suite.skill.fingerprint()})")
+    _out(f"Model:  {suite.agent.model}")
+    _out(f"Plan:   {len(suite.tasks)} task(s) x {suite.repeats} repeat(s) x 2 conditions = {total} agent runs")
+    _out("Each run spends real tokens. Ctrl-C to abort.")
+    _out()
+
+    try:
+        results = run_suite(suite, on_event=_out, keep_workspace=args.keep_workspace)
+    except AgentError as exc:
+        _out()
+        _out(f"AGENT ERROR: {exc}")
+        _out("Nothing usable was measured. Check `claude -p 'hi' --output-format json` works first.")
+        return 2
+    except KeyboardInterrupt:
+        _out("\nAborted.")
+        return 130
+
+    summary = summarise(results)
+    path = storage.save_run(
+        skill_dir,
+        skill_name=suite.skill.name,
+        fingerprint=suite.skill.fingerprint(),
+        model=suite.agent.model,
+        results=results,
+        summary=summary,
+    )
+
+    _out(
+        reporter.render_summary(
+            summary,
+            skill_name=suite.skill.name,
+            model=suite.agent.model,
+            fingerprint=suite.skill.fingerprint(),
+        )
+    )
+    _out(f"Saved: {path}")
+
+    success = summary.deltas["success"]
+    if args.strict and not (success.conclusive and success.improved):
+        _out("--strict: no conclusive improvement in task success.")
+        return 1
+    return 0
+
+
+def cmd_report(args: argparse.Namespace) -> int:
+    skill_dir = Path(args.path)
+    record = storage.latest_run(skill_dir)
+    if record is None:
+        _out(f"No recorded runs in {storage.runs_dir(skill_dir)}. Run `agent-skill test` first.")
+        return 1
+
+    summary = Summary(**{**record["summary"], "deltas": {}})
+    from .scoring import Delta  # local import keeps the module import graph flat
+
+    summary.deltas = {k: Delta(**v) for k, v in record["summary"]["deltas"].items()}
+
+    _out(
+        reporter.render_summary(
+            summary,
+            skill_name=record["skill"],
+            model=record["model"],
+            fingerprint=record["fingerprint"],
+        )
+    )
+
+    history = storage.load_history(skill_dir)
+    if len(history) > 1 and args.history:
+        _out("HISTORY")
+        _out(f"  {'timestamp':<18}{'fingerprint':<18}{'success w/':>11}{'tokens w/':>12}")
+        for entry in history[-10:]:
+            _out(
+                f"  {entry['timestamp']:<18}{entry['fingerprint']:<18}"
+                f"{(entry.get('with_success') or 0) * 100:>10.0f}%"
+                f"{(entry.get('with_tokens') or 0):>12,.0f}"
+            )
+        _out()
+        _out("Fingerprint changes mean the skill content changed between runs.")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agent-skill",
+        description="Reproducible A/B testing for Agent Skills.",
+    )
+    parser.add_argument("--version", action="version", version=f"agent-skill-lab {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    init = subparsers.add_parser("init", help="scaffold a skill and eval suite")
+    init.add_argument("path", nargs="?", default=".", help="target directory")
+    init.add_argument("--force", action="store_true", help="overwrite existing files")
+    init.set_defaults(func=cmd_init)
+
+    validate_cmd = subparsers.add_parser("validate", help="check a skill and its eval suite")
+    validate_cmd.add_argument("path", nargs="?", default=".", help="skill directory")
+    validate_cmd.add_argument("--skill-only", action="store_true", help="skip eval suite checks")
+    validate_cmd.set_defaults(func=cmd_validate)
+
+    test = subparsers.add_parser("test", help="run the A/B benchmark")
+    test.add_argument("path", nargs="?", default=".", help="skill directory")
+    test.add_argument("-n", "--repeats", type=int, help="runs per condition per task")
+    test.add_argument("--model", help="override the model from evals.yaml")
+    test.add_argument("--task", action="append", help="only run this task id (repeatable)")
+    test.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="check assertions against the pristine fixture without spawning an agent",
+    )
+    test.add_argument(
+        "--estimate",
+        action="store_true",
+        help="project cost and time, then exit without spawning an agent",
+    )
+    test.add_argument("--keep-workspace", action="store_true", help="do not delete run workspaces")
+    test.add_argument("--strict", action="store_true", help="exit 1 unless success improved conclusively")
+    test.add_argument("--quiet", action="store_true", help="hide validation warnings")
+    test.set_defaults(func=cmd_test)
+
+    report = subparsers.add_parser("report", help="re-render the most recent run")
+    report.add_argument("path", nargs="?", default=".", help="skill directory")
+    report.add_argument("--history", action="store_true", help="also list previous invocations")
+    report.set_defaults(func=cmd_report)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    try:
+        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    except (AttributeError, ValueError):
+        pass
+
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except ConfigError as exc:
+        _out(f"CONFIG ERROR: {exc}")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_skill_lab/config.py

@@ -0,0 +1,258 @@
+"""Loading and parsing of skills and eval suites."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+FRONTMATTER_RE = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n?(.*)\Z", re.S)
+
+# Directories inside a skill that must never be copied into the agent's
+# workspace: the eval suite contains the assertions, and leaking them would let
+# the agent optimise for the grader instead of the task.
+SKILL_COPY_EXCLUDE = {"evals", ".agent-skill", ".git", "__pycache__"}
+
+# A sentinel rather than a word like "TODO": eval prompts legitimately say
+# things like "implement the TODO comments in src/", and flagging those would
+# block real suites.
+PLACEHOLDER_SENTINEL = "REPLACE-ME"
+
+
+class ConfigError(Exception):
+    """Raised when a skill or eval suite cannot be loaded."""
+
+
+@dataclass
+class Skill:
+    dir: Path
+    frontmatter: dict[str, Any]
+    body: str
+    raw: str
+
+    @property
+    def name(self) -> str:
+        return str(self.frontmatter.get("name") or self.dir.name)
+
+    @property
+    def description(self) -> str:
+        return str(self.frontmatter.get("description") or "")
+
+    def fingerprint(self) -> str:
+        """Content hash of SKILL.md plus every bundled resource file.
+
+        Used to tell whether a recorded run still describes the current skill.
+        """
+        h = hashlib.sha256()
+        for path in sorted(iter_skill_files(self.dir)):
+            h.update(path.relative_to(self.dir).as_posix().encode())
+            h.update(path.read_bytes())
+        return h.hexdigest()[:16]
+
+
+def iter_skill_files(skill_dir: Path):
+    for path in sorted(skill_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(skill_dir)
+        if set(rel.parts) & SKILL_COPY_EXCLUDE:
+            continue
+        yield path
+
+
+@dataclass
+class Assertion:
+    type: str
+    weight: float = 1.0
+    required: bool = True
+    spec: dict[str, Any] = field(default_factory=dict)
+
+    def label(self) -> str:
+        if self.type == "command":
+            return f"command: {self.spec.get('run', '')}"
+        target = self.spec.get("path", "")
+        extra = self.spec.get("pattern", "")
+        return f"{self.type}: {target}" + (f" ~ /{extra}/" if extra else "")
+
+
+@dataclass
+class Task:
+    id: str
+    name: str
+    prompt: str
+    path: Path
+    fixture: Path | None
+    assertions: list[Assertion]
+
+
+@dataclass
+class AgentConfig:
+    provider: str = "claude-code"
+    model: str = "sonnet"
+    timeout_seconds: int = 600
+    setting_sources: str = "project"
+    # "default": the agent keeps whatever skills the Claude Code install bundles
+    #   (they appear in both conditions, so they cancel out of the delta).
+    # "bare": run with --bare for a clean-room baseline with no bundled/plugin
+    #   skills. Requires ANTHROPIC_API_KEY, since --bare never reads OAuth.
+    isolation: str = "default"
+    extra_args: list[str] = field(default_factory=list)
+
+    @property
+    def model_is_pinned(self) -> bool:
+        # A pinned id looks like claude-sonnet-4-5-20250929; aliases are bare
+        # words such as "sonnet" or "opus" and float across model releases.
+        return bool(re.search(r"\d{6,}", self.model))
+
+    @property
+    def is_bare(self) -> bool:
+        return self.isolation == "bare"
+
+
+@dataclass
+class Suite:
+    skill: Skill
+    evals_dir: Path
+    agent: AgentConfig
+    repeats: int
+    parallel: int
+    tasks: list[Task]
+
+    @property
+    def state_dir(self) -> Path:
+        return self.skill.dir / ".agent-skill"
+
+
+def parse_frontmatter(raw: str) -> tuple[dict[str, Any], str]:
+    match = FRONTMATTER_RE.match(raw.lstrip(""))
+    if not match:
+        return {}, raw
+    try:
+        data = yaml.safe_load(match.group(1)) or {}
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"invalid YAML frontmatter: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ConfigError("frontmatter must be a YAML mapping")
+    return data, match.group(2)
+
+
+def load_skill(skill_dir: Path) -> Skill:
+    skill_dir = skill_dir.resolve()
+    md = skill_dir / "SKILL.md"
+    if not md.is_file():
+        raise ConfigError(f"no SKILL.md in {skill_dir}")
+    raw = md.read_text(encoding="utf-8")
+    frontmatter, body = parse_frontmatter(raw)
+    return Skill(dir=skill_dir, frontmatter=frontmatter, body=body, raw=raw)
+
+
+def _load_yaml(path: Path) -> dict[str, Any]:
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"{path.name}: invalid YAML: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ConfigError(f"{path.name}: expected a YAML mapping at the top level")
+    return data
+
+
+def load_task(path: Path, evals_dir: Path) -> Task:
+    data = _load_yaml(path)
+    prompt = str(data.get("prompt") or "").strip()
+    if not prompt:
+        raise ConfigError(f"{path.name}: missing 'prompt'")
+
+    fixture = None
+    if data.get("fixture"):
+        fixture = (evals_dir / str(data["fixture"])).resolve()
+
+    assertions = []
+    for index, item in enumerate(data.get("assertions") or []):
+        if not isinstance(item, dict):
+            raise ConfigError(f"{path.name}: assertion #{index + 1} must be a mapping")
+        kind = item.get("type")
+        if not kind:
+            raise ConfigError(f"{path.name}: assertion #{index + 1} has no 'type'")
+        assertions.append(
+            Assertion(
+                type=str(kind),
+                weight=float(item.get("weight", 1.0)),
+                required=bool(item.get("required", True)),
+                spec=item,
+            )
+        )
+
+    return Task(
+        id=str(data.get("id") or path.stem),
+        name=str(data.get("name") or path.stem),
+        prompt=prompt,
+        path=path,
+        fixture=fixture,
+        assertions=assertions,
+    )
+
+
+def load_suite(
+    skill_dir: Path,
+    *,
+    repeats: int | None = None,
+    model: str | None = None,
+    only_tasks: list[str] | None = None,
+) -> Suite:
+    skill = load_skill(skill_dir)
+    evals_dir = skill.dir / "evals"
+    if not evals_dir.is_dir():
+        raise ConfigError(
+            f"no evals/ directory in {skill.dir}. Run `agent-skill init` to scaffold one."
+        )
+
+    config_path = evals_dir / "evals.yaml"
+    config = _load_yaml(config_path) if config_path.is_file() else {}
+
+    agent_cfg = config.get("agent") or {}
+    agent = AgentConfig(
+        provider=str(agent_cfg.get("provider", "claude-code")),
+        model=str(model or agent_cfg.get("model", "sonnet")),
+        timeout_seconds=int(agent_cfg.get("timeout_seconds", 600)),
+        setting_sources=str(agent_cfg.get("setting_sources", "project")),
+        isolation=str(agent_cfg.get("isolation", "default")),
+        extra_args=list(agent_cfg.get("extra_args") or []),
+    )
+
+    run_cfg = config.get("run") or {}
+    listed = config.get("tasks")
+    if listed:
+        task_paths = [evals_dir / str(name) for name in listed]
+    else:
+        task_paths = sorted(evals_dir.glob("task-*.yaml"))
+
+    missing = [p for p in task_paths if not p.is_file()]
+    if missing:
+        raise ConfigError("task file not found: " + ", ".join(p.name for p in missing))
+
+    tasks = [load_task(p, evals_dir) for p in task_paths]
+    if only_tasks:
+        wanted = set(only_tasks)
+        tasks = [t for t in tasks if t.id in wanted]
+        unknown = wanted - {t.id for t in tasks}
+        if unknown:
+            raise ConfigError("unknown task id: " + ", ".join(sorted(unknown)))
+    if not tasks:
+        raise ConfigError(f"no task-*.yaml files found in {evals_dir}")
+
+    return Suite(
+        skill=skill,
+        evals_dir=evals_dir,
+        agent=agent,
+        repeats=int(repeats or run_cfg.get("repeats", 5)),
+        parallel=int(run_cfg.get("parallel", 2)),
+        tasks=tasks,
+    )
+
+
+def looks_like_placeholder(text: str) -> bool:
+    return PLACEHOLDER_SENTINEL in text.upper()