acumen 0.0.1.dev0__py3-none-any.whl

acumen/runner.py

@@ -0,0 +1,343 @@
+"""SDK invocation for a single benchmark run.
+
+Runs one agent in one sandbox, captures its transcript, grades what it wrote, and emits
+``result.json`` — the unit of record. Writing ``result.json`` last is deliberate: its
+presence is what marks a run complete, which is what makes ``--resume`` safe.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TextIO
+
+from claude_agent_sdk import ClaudeAgentOptions, ResultMessage, query
+
+from acumen.env import AuthMode, Target, sdk_version
+from acumen.grade import Grade, Reason, grade_run
+from acumen.paths import (
+    ANSWER_FILE,
+    RESULT_FILE,
+    SCRIPT_FILE,
+    TRANSCRIPT_HTML,
+    TRANSCRIPT_JSONL,
+    RunKey,
+)
+from acumen.prompts import benchmark_prompt
+from acumen.sandbox import Sandbox, sandbox
+from acumen.skills import Skill
+from acumen.tasks import Task
+from acumen.transcript import locate_transcript, render_transcript
+
+#: Result subtypes the CLI reports on a cap breach, mapped to our reason taxonomy.
+_SUBTYPE_REASONS: dict[str, Reason] = {
+    "error_max_budget_usd": "budget",
+    "error_max_turns": "max_turns",
+}
+
+
+@dataclass(frozen=True)
+class RunOutcome:
+    """What a run produced, as recorded in ``result.json``."""
+
+    key: RunKey
+    success: bool
+    reason: Reason
+    payload: dict
+
+
+def _terminal_reason(message: ResultMessage) -> Reason | None:
+    """Map a cap breach or hard error onto a reason, or ``None`` if the agent finished.
+
+    ``subtype`` is a free-form string from the CLI, so match defensively rather than
+    against an exhaustive list we cannot see from here.
+    """
+    subtype = (message.subtype or "").lower()
+    if subtype in _SUBTYPE_REASONS:
+        return _SUBTYPE_REASONS[subtype]
+    if "budget" in subtype:
+        return "budget"
+    if "max_turns" in subtype:
+        return "max_turns"
+    if message.is_error or subtype.startswith("error"):
+        return "error"
+    return None
+
+
+def _find_transcript(box: Sandbox, session_id: str) -> Path | None:
+    return locate_transcript(box.config_dir, box.root, session_id)
+
+
+def _skill_fired(jsonl: Path, name: str) -> bool | None:
+    """Return whether the agent loaded the skill called ``name``, read from the transcript.
+
+    This is the evidence the skill arm actually used the skill — the skill arm must show it
+    loading and the baseline must not — so it is recorded per run rather than left to
+    hand-inspection. ``None`` means the transcript was unavailable, which is not the same
+    as the skill not loading.
+
+    The match is on the skill *identity*, not on the tool: a ``Skill`` call carries the
+    requested skill in ``input.skill``, and the CLI ships bundled skills of its own that the
+    agent can reach in either arm. Loading one of those says nothing about the skill under
+    test, so only ``input.skill == name`` counts.
+    """
+    try:
+        lines = jsonl.read_text().splitlines()
+    except OSError:
+        return None
+    for line in lines:
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if record.get("type") != "assistant":
+            continue
+        content = record.get("message", {}).get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict) or block.get("type") != "tool_use" or block.get("name") != "Skill":
+                continue
+            payload = block.get("input")
+            if isinstance(payload, dict) and payload.get("skill") == name:
+                return True
+    return False
+
+
+def _collect_artifacts(box: Sandbox, run_dir: Path) -> None:
+    for name in (ANSWER_FILE, SCRIPT_FILE):
+        src = box.root / name
+        if src.is_file():
+            shutil.copyfile(src, run_dir / name)
+
+
+def _usage_tokens(usage: dict | None) -> tuple[int, int]:
+    if not usage:
+        return 0, 0
+    inp = int(usage.get("input_tokens", 0) or 0)
+    inp += int(usage.get("cache_read_input_tokens", 0) or 0)
+    inp += int(usage.get("cache_creation_input_tokens", 0) or 0)
+    return inp, int(usage.get("output_tokens", 0) or 0)
+
+
+class StderrFilter:
+    """Dedupe the CLI subprocess stderr the SDK forwards, keeping the first of each line.
+
+    A benchmark pass spawns one CLI subprocess per run, and each reprints the same
+    startup diagnostics — most visibly the ``claude.ai connectors are disabled`` notice,
+    which fires on every spawn because an auth source is present in the agents' env. Left
+    alone that is one identical warning per run.
+
+    Share a single instance across a whole pass and hand it to :func:`run_matrix` as its
+    ``stderr`` callback. The SDK pipes subprocess stderr only when a callback is set and
+    then invokes it one line at a time on the event loop, so this becomes the sole sink:
+    the first time a given line is seen it is forwarded to our own stderr, and every later
+    repeat is dropped. Distinct lines still all pass through; only exact repeats are cut.
+
+    Called on a single-threaded event loop, so the seen-set needs no locking.
+    """
+
+    def __init__(self, sink: TextIO | None = None) -> None:
+        self._seen: set[str] = set()
+        self._sink = sink if sink is not None else sys.stderr
+
+    def __call__(self, line: str) -> None:
+        """Forward ``line`` the first time it is seen; drop it on every later repeat."""
+        if line in self._seen:
+            return
+        self._seen.add(line)
+        print(line, file=self._sink, flush=True)
+
+
+def _build_options(
+    *,
+    box: Sandbox,
+    model: str,
+    max_turns: int,
+    max_usd: float,
+    stderr: Callable[[str], None] | None = None,
+) -> ClaudeAgentOptions:
+    """Assemble the SDK options for one run.
+
+    These options are **identical for every arm** — baseline parity means the
+    noskill and skill arms differ only in whether a skill directory exists inside the
+    sandbox, never in how the agent is configured.
+
+    ``setting_sources`` is always set explicitly, never left to default: the default of
+    ``None`` loads user settings *and* memories, and leaving it unset while
+    passing ``skills`` silently re-enables the ``"user"`` source.
+    ``["project"]`` scopes discovery to ``<sandbox>/.claude/``, which is exactly where
+    the skill is installed — empirically verified, along with the fact that
+    ``[]`` disables skill discovery entirely.
+
+    ``skills`` is deliberately left ``None``: with ``setting_sources=["project"]`` the
+    skill is discovered from the sandbox anyway (verified), whereas naming it here would
+    append ``Skill(<name>)`` to ``allowed_tools`` in the skill arm only — an options-level
+    difference between arms, which is the one thing parity forbids.
+    """
+    return ClaudeAgentOptions(
+        cwd=str(box.root),
+        env=box.env,
+        model=model,
+        max_turns=max_turns,
+        max_budget_usd=max_usd,
+        setting_sources=["project"],
+        permission_mode="bypassPermissions",
+        system_prompt={"type": "preset", "preset": "claude_code"},
+        stderr=stderr,
+    )
+
+
+async def run_once(
+    *,
+    key: RunKey,
+    task: Task,
+    target: Target,
+    run_dir: Path,
+    model: str,
+    max_turns: int,
+    max_usd: float,
+    auth_mode: AuthMode = "api",
+    skill: Skill | None = None,
+    skill_name: str | None = None,
+    sandbox_base: Path | None = None,
+    keep_sandbox: bool = False,
+    stderr: Callable[[str], None] | None = None,
+    env_passthrough: Sequence[str] | None = None,
+) -> RunOutcome:
+    """Execute one benchmark run end to end and write its ``result.json``.
+
+    Parameters
+    ----------
+    key
+        Identifies the run; also determines ``run_dir`` upstream.
+    task
+        The task being run; ``key.split`` selects which half.
+    target
+        The prepared target, supplying the interpreter and fingerprint.
+    run_dir
+        Where the five run files are written.
+    model, max_turns, max_usd
+        Already resolved against per-task overrides by the caller.
+    auth_mode
+        Which credential benchmark runs authenticate with; always ``"api"`` so the recorded
+        ``cost_usd`` reflects real metered spend.
+    skill
+        The skill to install, or ``None`` for the baseline arm. Must agree with
+        ``key.arm``, else the run would be filed under an arm it doesn't belong to.
+    skill_name
+        Name of the skill under test, used to recognise it in the transcript. Defaults to
+        the installed skill's name; pass it in the baseline arm too, where nothing is
+        installed but a stray load of that same skill is exactly what must be caught.
+        Without either, ``skill_loaded`` is left ``None`` — unattributable, not absent.
+    sandbox_base
+        Parent directory for the throwaway sandbox.
+    keep_sandbox
+        Leave the sandbox on disk — useful when hand-checking a failure.
+    stderr
+        Callback for the CLI subprocess's stderr, one line at a time. Pass a shared
+        :class:`StderrFilter` across a pass to collapse the per-spawn startup warnings;
+        ``None`` (the default) lets the subprocess stderr inherit the terminal unfiltered.
+    env_passthrough
+        Extra environment variable names to carry into the sandbox on top of the built-in
+        allowlist (the operator's ``config.env_passthrough``).
+
+    Returns
+    -------
+    The outcome, matching what was written to disk.
+    """
+    if (key.skill is None) != (skill is None):
+        # Filing a skill run under noskill/ (or vice versa) would silently corrupt the
+        # comparison the whole benchmark rests on, so refuse rather than record it.
+        raise ValueError(
+            f"arm {key.arm!r} expects skill {key.skill!r} but was given {skill.version if skill else None!r}"
+        )
+    run_dir.mkdir(parents=True, exist_ok=True)
+    split = task.split(key.split)
+
+    result: ResultMessage | None = None
+    error: str | None = None
+
+    with sandbox(
+        target,
+        auth_mode=auth_mode,
+        base=sandbox_base,
+        keep=keep_sandbox,
+        skill=skill,
+        env_passthrough=env_passthrough,
+    ) as box:
+        prompt = benchmark_prompt(
+            split.prompt,
+            sandbox=box.root,
+            python=target.python,
+            package=target.pkg_name,
+        )
+        options = _build_options(box=box, model=model, max_turns=max_turns, max_usd=max_usd, stderr=stderr)
+        try:
+            async for message in query(prompt=prompt, options=options):
+                if isinstance(message, ResultMessage):
+                    result = message
+        except Exception as err:  # noqa: BLE001 - a crashed agent is a recorded failure, not a crashed pass
+            error = f"{type(err).__name__}: {err}"
+
+        _collect_artifacts(box, run_dir)
+        if result is not None:
+            jsonl = _find_transcript(box, result.session_id)
+            if jsonl is not None:
+                shutil.copyfile(jsonl, run_dir / TRANSCRIPT_JSONL)
+
+    rendered = False
+    skill_loaded: bool | None = None
+    expected_skill = skill.name if skill is not None else skill_name
+    if (run_dir / TRANSCRIPT_JSONL).is_file():
+        rendered = render_transcript(run_dir / TRANSCRIPT_JSONL, run_dir / TRANSCRIPT_HTML)
+        if expected_skill is not None:
+            skill_loaded = _skill_fired(run_dir / TRANSCRIPT_JSONL, expected_skill)
+
+    grade: Grade = grade_run(run_dir, split.answer)
+    if error is not None or result is None:
+        success, reason = False, "error"
+    else:
+        terminal = _terminal_reason(result)
+        if terminal is not None:
+            # A cap breach is a failure regardless of what landed in answer.md.
+            success, reason = False, terminal
+        else:
+            success, reason = grade.success, grade.reason
+
+    inp, out = _usage_tokens(result.usage if result else None)
+    payload = {
+        "task_id": key.task_id,
+        "split": key.split,
+        "skill": key.skill,
+        "arm": key.arm,
+        "model": model,
+        "rep": key.rep,
+        "success": success,
+        "reason": reason,
+        "turns": result.num_turns if result else 0,
+        "cost_usd": (result.total_cost_usd if result else None) or 0.0,
+        "input_tokens": inp,
+        "output_tokens": out,
+        "duration_s": round((result.duration_ms if result else 0) / 1000, 2),
+        "answer": grade.answer,
+        "expected": split.answer.strip(),
+        "pkg_version": target.fingerprint,
+        "commit": target.commit,
+        "skill_hash": skill.hash if skill else None,
+        "skill_name": skill.name if skill else None,
+        # Evidence, not configuration: did the agent actually invoke the Skill tool?
+        "skill_loaded": skill_loaded,
+        "sdk_version": sdk_version(),
+        "session_id": result.session_id if result else None,
+        "stop_reason": result.stop_reason if result else None,
+        "subtype": result.subtype if result else None,
+        "transcript_html": rendered,
+        "error": error or (result.errors if result else None),
+    }
+    (run_dir / RESULT_FILE).write_text(json.dumps(payload, indent=2) + "\n")
+    return RunOutcome(key=key, success=success, reason=reason, payload=payload)

acumen/sandbox.py

@@ -0,0 +1,153 @@
+"""Per-run sandbox: a fresh empty directory with the target venv on PATH.
+
+The agent sees an installed package and nothing else — no repo source, no user
+settings, no memories. Each run gets its own sandbox, throwaway ``HOME`` and throwaway
+``CLAUDE_CONFIG_DIR``, so runs cannot see each other either.
+
+A skill arm differs from the baseline in exactly one way: ``skills/v{n}/`` is
+copied into ``<sandbox>/.claude/skills/<name>/``, where the agent's own project settings
+discovery finds it. Same prompt, same tools, same caps, same env otherwise.
+"""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from collections.abc import Iterator, Sequence
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from acumen.env import AuthMode, Target, build_agent_env
+from acumen.procs import label_env, reap
+from acumen.skills import Skill, content_files
+
+#: Where the ``claude`` CLI looks for project skills, relative to the agent's cwd.
+SKILLS_SUBDIR = Path(".claude") / "skills"
+
+
+@dataclass(frozen=True)
+class Sandbox:
+    """A prepared, isolated working directory for exactly one agent run."""
+
+    root: Path
+    home: Path
+    config_dir: Path
+    env: dict[str, str]
+    authenticated: bool
+    skill: Skill | None = None
+
+    @property
+    def transcript_root(self) -> Path:
+        """Where the ``claude`` CLI writes transcripts for this run."""
+        return self.config_dir / "projects"
+
+    @property
+    def skills_dir(self) -> Path:
+        """The project skills directory the agent discovers, ``<root>/.claude/skills``."""
+        return self.root / SKILLS_SUBDIR
+
+    @property
+    def skill_hash(self) -> str | None:
+        """The installed skill's content hash, or ``None`` for the baseline arm."""
+        return self.skill.hash if self.skill else None
+
+
+def install_skill(root: Path, skill: Skill) -> Path:
+    """Copy a skill's content into a sandbox as a discoverable project skill.
+
+    ``meta.json`` is deliberately left behind: it is acumen's provenance record, and the
+    agent consuming the skill has no business reading the rationale for its own version.
+
+    Parameters
+    ----------
+    root
+        The sandbox root — the agent's ``cwd``.
+    skill
+        The loaded skill to install.
+
+    Returns
+    -------
+    The installed skill directory, ``<root>/.claude/skills/<name>``.
+    """
+    dest = root / SKILLS_SUBDIR / skill.name
+    dest.mkdir(parents=True, exist_ok=True)
+    for src in content_files(skill.directory):
+        target = dest / src.relative_to(skill.directory)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, target)
+    return dest
+
+
+@contextmanager
+def sandbox(
+    target: Target,
+    *,
+    auth_mode: AuthMode,
+    base: Path | None = None,
+    keep: bool = False,
+    skill: Skill | None = None,
+    env_passthrough: Sequence[str] | None = None,
+) -> Iterator[Sandbox]:
+    """Create a fresh sandbox for one run and clean it up afterwards.
+
+    Parameters
+    ----------
+    target
+        The prepared target; its venv ``bin`` goes on the sandbox PATH.
+    auth_mode
+        Which credential the run authenticates with (see :func:`acumen.env.build_agent_env`).
+        Benchmark runs always pass ``"api"`` so the recorded ``cost_usd`` is real.
+    base
+        Parent directory for the sandbox. Defaults to the system temp dir.
+    keep
+        Leave the sandbox on disk after the run — useful when debugging a failure.
+    skill
+        Skill to install for this run, or ``None`` for the baseline arm.
+    env_passthrough
+        Extra environment variable names to carry into the sandbox on top of the built-in
+        allowlist (the operator's ``config.env_passthrough``).
+
+    Yields
+    ------
+    The sandbox, whose ``root`` is the agent's ``cwd``.
+    """
+    if base is not None:
+        base.mkdir(parents=True, exist_ok=True)
+    holder = Path(tempfile.mkdtemp(prefix="acumen-run-", dir=base))
+    try:
+        root = holder / "work"
+        home = holder / "home"
+        config_dir = home / ".claude"
+        for path in (root, home, config_dir, home / "tmp"):
+            path.mkdir(parents=True, exist_ok=True)
+        if skill is not None:
+            install_skill(root, skill)
+        # The marker is what lets the teardown below find whatever this agent leaves running.
+        env = label_env(
+            build_agent_env(
+                config_dir=config_dir,
+                home=home,
+                extra_path=[target.bin_dir],
+                auth_mode=auth_mode,
+                extra_allow=env_passthrough,
+            ),
+            holder,
+        )
+        yield Sandbox(
+            root=root,
+            home=home,
+            config_dir=config_dir,
+            env=env,
+            # Only session mode seeds the OAuth login into the throwaway config dir.
+            authenticated=auth_mode == "session",
+            skill=skill,
+        )
+    finally:
+        # The agent's own descendants outlive it — a backgrounded job, or anything still
+        # running when the CLI was terminated on a cap breach or a Ctrl-C. Kill them before
+        # the directory goes, so nothing is left writing into a path that no longer exists.
+        # ``keep`` preserves the files for inspection, never the processes.
+        reap(holder)
+        if not keep:
+            shutil.rmtree(holder, ignore_errors=True)

acumen/scaffold.py

@@ -0,0 +1,118 @@
+"""Scaffold starter ``config.yaml`` and ``tasks.yaml`` files for ``acumen init``.
+
+The templates are package-agnostic placeholders, kept heavily commented so a new user can
+read what each field does and fill in the blanks rather than starting from an empty file.
+The placeholders (``OWNER/REPO``, the ``REPLACE_ME`` answers) are meant to be edited before
+the first run.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+CONFIG_FILENAME = "config.yaml"
+TASKS_FILENAME = "tasks.yaml"
+
+#: An annotated config with placeholder target/skill fields for the user to fill in.
+CONFIG_TEMPLATE = """\
+# acumen config — the target package and how to benchmark against it.
+# Every field below except `repo` has a default; delete a line to take it.
+
+repo: https://github.com/OWNER/REPO   # GitHub URL, or a local path to an installable package
+ref: main                             # branch, tag, or commit (ignored for local paths)
+extras: []                            # optional pip extras, e.g. [dev, test]
+python: "3.12"                        # interpreter for the target's throwaway venv
+env_passthrough: []                   # extra env vars agents may keep (e.g. [OMP_NUM_THREADS]); the agent env is otherwise a clean allowlist
+
+models: [claude-opus-5, claude-sonnet-5, claude-haiku-4-5-20251001]   # benchmark models; a pass is models x tasks x reps x splits
+n_replicates: 3                       # runs per (model, task, split) cell
+max_concurrency: 4                    # simultaneous agents
+
+max_turns: 40                         # turn cap for benchmark agents; draft/improve/ship/tasks are unbounded
+max_usd: 3.0                          # budget cap (USD) for benchmark agents; draft/improve/ship/tasks are unbounded
+
+draft_model: claude-opus-5            # model that writes the first skill (reads the source)
+improve_model: claude-opus-5          # model that improves a skill from its train results
+tasks_model: claude-opus-5            # model for `acumen tasks` (generates this file); runs code
+ship_model: claude-opus-5             # model for `acumen ship` (wires a multi-framework install-skills script)
+
+# skill_name: your_skill              # name for the built skill; defaults to the repo's name
+"""
+
+#: A tasks file with one placeholder task, showing the required shape. Each task needs a
+#: stable `id` and a train/test pair; the answer is graded by exact string match after
+#: `strip()`.
+TASKS_TEMPLATE = """\
+# acumen tasks — what the agent is asked to do, and the answer it is graded against.
+#
+# Each task has a stable `id` (used in run paths — renaming it orphans old runs) and a
+# train/test pair. The improver only ever sees train results; test is the held-out measure
+# of whether a skill actually helps. Answers are compared by EXACT string match after
+# strip(), so keep them to a single unambiguous token.
+#
+# Do NOT name the target package in a prompt. The agent is already told, before every task,
+# that your package (from `config.yaml`) is installed and is the one to use — so "using
+# <package>, ..." is redundant. Describe only the analysis: the goal, the input, and exactly
+# what to report. (`acumen tasks` follows the same rule when it generates this file.)
+#
+# A good task is one your target package solves and the baseline gets wrong — that is where
+# a skill has room to help. Replace the placeholders below with your own, or run
+# `acumen tasks` to generate this file automatically from the target package.
+#
+# Optional per-task overrides: `max_turns`, `max_usd`, `model`.
+
+tasks:
+  - id: example_task
+    train:
+      prompt: >-
+        Describe, concretely, the analysis the agent should carry out on the training input
+        — the data, the goal, and exactly what to report. Do not name the package (it is
+        provided). End by asking for only the final answer, so it grades as a single token.
+      answer: REPLACE_ME_TRAIN
+    test:
+      prompt: >-
+        The same kind of analysis on a different, held-out input. This split is what measures
+        whether the skill generalizes; the improver never sees its results.
+      answer: REPLACE_ME_TEST
+"""
+
+
+class InitError(ValueError):
+    """Raised when scaffolding cannot proceed — e.g. files exist and ``force`` is unset."""
+
+
+def scaffold(directory: Path, *, force: bool = False) -> list[Path]:
+    """Write starter ``config.yaml`` and ``tasks.yaml`` into ``directory``.
+
+    Parameters
+    ----------
+    directory
+        Where to scaffold. Created if it does not exist.
+    force
+        Overwrite existing files. Without it, an existing ``config.yaml`` or ``tasks.yaml``
+        is left untouched and an error is raised, so a real project is never clobbered.
+
+    Returns
+    -------
+    The paths written, in the order they were written.
+
+    Raises
+    ------
+    InitError
+        If a target file exists and ``force`` is not set.
+    """
+    directory.mkdir(parents=True, exist_ok=True)
+    targets = {directory / CONFIG_FILENAME: CONFIG_TEMPLATE, directory / TASKS_FILENAME: TASKS_TEMPLATE}
+
+    if not force:
+        existing = [p for p in targets if p.exists()]
+        if existing:
+            names = ", ".join(p.name for p in existing)
+            verb = "exists" if len(existing) == 1 else "exist"
+            raise InitError(f"{names} already {verb} in {directory} — pass --force to overwrite")
+
+    written = []
+    for path, template in targets.items():
+        path.write_text(template)
+        written.append(path)
+    return written