acumen 0.0.1.dev0__py3-none-any.whl

acumen/__init__.py

@@ -0,0 +1,160 @@
+"""Build, benchmark, and optimize Claude skills for Python packages."""
+
+from importlib.metadata import version
+
+from acumen.bench import PlannedRun, build_matrix, pending, run_matrix, summarize
+from acumen.config import Config, ConfigError, load_config, parse_config
+from acumen.draft import DraftError, DraftResult, draft_skill
+from acumen.env import (
+    AuthMode,
+    EnvError,
+    Target,
+    api_auth_available,
+    auth_available,
+    build_agent_env,
+    check_auth,
+    prepare_target,
+    resolve_auth_mode,
+    scrubbed_env,
+    session_auth_available,
+)
+from acumen.grade import Grade, Reason, grade_answer, grade_run
+from acumen.improve import (
+    ImproveError,
+    ImproveResult,
+    TrainRun,
+    collect_train_runs,
+    find_test_access,
+    improve_skill,
+    make_test_guard,
+)
+from acumen.logs import LiveLog
+from acumen.paths import RunKey, Split, arm_name, is_complete, parse_run_dir, run_dir, skill_from_arm
+from acumen.report import (
+    Report,
+    ReportError,
+    SkillTests,
+    arm_metrics,
+    build_report,
+    load_results,
+    resolve_palette,
+    skill_tests,
+    tradeoff_figure,
+)
+from acumen.runner import RunOutcome, run_once
+from acumen.sandbox import Sandbox, install_skill, sandbox
+from acumen.scaffold import InitError, scaffold
+from acumen.ship import ShipError, ShipResult, installer_exists, ship_skill
+from acumen.skills import (
+    Skill,
+    SkillError,
+    SkillMeta,
+    available_versions,
+    latest_version,
+    load_skill,
+    next_version,
+    skill_content,
+    skill_hash,
+)
+from acumen.taskgen import (
+    TaskGenError,
+    TaskGenResult,
+    build_filtered_source,
+    dump_tasks,
+    find_skill_access,
+    generate_tasks,
+    make_skill_guard,
+)
+from acumen.tasks import Task, TaskError, TaskSplit, load_tasks, parse_tasks
+from acumen.transcript import locate_transcript, render_transcript
+
+__all__ = [
+    "AuthMode",
+    "Config",
+    "ConfigError",
+    "DraftError",
+    "DraftResult",
+    "EnvError",
+    "Grade",
+    "ImproveError",
+    "ImproveResult",
+    "InitError",
+    "LiveLog",
+    "PlannedRun",
+    "Reason",
+    "Report",
+    "ReportError",
+    "RunKey",
+    "RunOutcome",
+    "Sandbox",
+    "ShipError",
+    "ShipResult",
+    "Skill",
+    "SkillError",
+    "SkillMeta",
+    "Split",
+    "Target",
+    "Task",
+    "TaskError",
+    "TaskGenError",
+    "TaskGenResult",
+    "TaskSplit",
+    "TrainRun",
+    "__version__",
+    "api_auth_available",
+    "arm_metrics",
+    "arm_name",
+    "auth_available",
+    "available_versions",
+    "check_auth",
+    "build_agent_env",
+    "build_filtered_source",
+    "build_matrix",
+    "build_report",
+    "collect_train_runs",
+    "draft_skill",
+    "dump_tasks",
+    "find_skill_access",
+    "find_test_access",
+    "generate_tasks",
+    "grade_answer",
+    "grade_run",
+    "improve_skill",
+    "install_skill",
+    "installer_exists",
+    "is_complete",
+    "make_skill_guard",
+    "make_test_guard",
+    "latest_version",
+    "load_config",
+    "locate_transcript",
+    "load_results",
+    "load_skill",
+    "load_tasks",
+    "next_version",
+    "parse_config",
+    "parse_run_dir",
+    "parse_tasks",
+    "pending",
+    "prepare_target",
+    "render_transcript",
+    "resolve_auth_mode",
+    "resolve_palette",
+    "run_dir",
+    "run_matrix",
+    "run_once",
+    "sandbox",
+    "scaffold",
+    "scrubbed_env",
+    "session_auth_available",
+    "ship_skill",
+    "SkillTests",
+    "skill_content",
+    "skill_from_arm",
+    "skill_hash",
+    "skill_tests",
+    "summarize",
+    "tradeoff_figure",
+]
+
+__version__ = version("acumen")

acumen/_skills/data/SKILL.md

@@ -0,0 +1,109 @@
+---
+name: acumen
+description: Use for any question or task involving the Python package `acumen` (its CLI or its API) — setting up a benchmark project against a target package, writing or generating benchmark tasks, drafting/improving/hand-editing an agent Skill, running and interpreting skill-vs-baseline benchmark passes, shipping a skill into the target package, diagnosing a run, and choosing what to do next in that loop — open it before answering or running anything, because acumen's defaults, guardrails and correct next step are not guessable from the command names.
+---
+
+# acumen
+
+One project directory targets one Python package. Everything is driven by the `acumen`
+CLI (a thin shell over an importable API — see `references/python-api.md`). Commands are
+run from the project dir and default to `config.yaml`, `tasks.yaml`, `skills/`, `runs/`,
+`logs/`.
+
+```bash
+acumen init                # scaffold config.yaml + tasks.yaml (placeholders the user must fill)
+acumen tasks               # optional: mine the package for tasks, run them for ground truth
+acumen bench --no-skill    # baseline arm  (this is also the default arm)
+acumen draft               # agent reads the package source -> skills/v1/
+acumen bench --skill v1
+acumen improve             # agent reads v1's TRAIN runs -> skills/v2/
+acumen bench --skill v2
+acumen report              # report.html + report.csv over every arm on disk
+acumen ship --skill v2     # wire a `<dist>-install-skills` script into the target package
+```
+
+That is the shape of the loop, not a script to execute unattended, and not a fixed order:
+several stages have more than one correct next move, and some stages call for talking to
+the user rather than running the next command at all.
+
+## The loop is collaborative — propose, discuss, then run
+
+`bench` bills the Anthropic API for every cell and the other commands drive long
+autonomous agents, so the user decides each step and when to stop. At every stage: state
+what you would do next and why, and let the user choose. Never chain commands
+unattended, and never decide on your own that a skill is good enough.
+
+**Before running anything, get from the user what acumen cannot infer** — the target, the
+models, the budgets, whether tasks and results look right, whether to ship. Guessing those
+is the most common way to waste a pass. The step after a command is often a question, not
+a command.
+
+| Just happened | What to propose next |
+|---|---|
+| `acumen init` | Both files are placeholders (`repo: OWNER/REPO`, `REPLACE_ME` answers). Ask the user for what only they can supply — the target repo/path, ref, models, budgets — and fill in `config.yaml` with their answers. Do not guess a target or run the next command. |
+| `config.yaml` filled | Offer both ways to get tasks: `acumen tasks` to generate them, or writing `tasks.yaml` by hand. Review generated tasks with the user before benching. |
+| config + tasks ready | `acumen bench --no-skill` and `acumen draft` are **both** correct next steps — they are independent, and you need the baseline arm and a skill arm before any comparison means anything. Offer both. |
+| a skill version exists | `acumen bench --skill vN`. |
+| any `bench` finished | `acumen report` — the per-arm, per-split numbers come from the report; do not judge a version by eyeballing runs or by one arm's pass count. |
+| `report` written | Discuss the results with the user and propose next steps: `improve` + re-bench if it has not beaten the baseline or the previous version, `ship` once a version has proven out, or stop. The user decides. |
+| user has a skill dir from elsewhere | Copy it into `skills/v1/` (next unused version) inside the acumen project — versions are only ever read from `skills/`; there is no import command and no external path flag. |
+
+## Route by goal
+
+| Goal | Command | Depth |
+|---|---|---|
+| Start a project for package X | `acumen init`, then fill `config.yaml` with the user | `references/setup.md` |
+| Get benchmark tasks without writing them | `acumen tasks [--force] [--feedback "…"]` | `references/setup.md` |
+| Write tasks by hand | edit `tasks.yaml` | `references/setup.md` |
+| Get a first skill | `acumen draft [--feedback "…"]` | `references/authoring.md` |
+| Measure whether the skill helps | `acumen bench --no-skill` and `acumen bench --skill vN` | `references/benchmark.md` |
+| Make the skill better | `acumen improve [--from vN]` then bench the new version | `references/authoring.md` |
+| See results / decide when to stop | `acumen report` | `references/benchmark.md` |
+| Hand-edit a skill version | copy `skills/vN` → `skills/v(N+1)`, edit, bench it | `references/authoring.md` |
+| Give package users the skill | `acumen ship --skill vN` | `references/ship.md` |
+| Script any of this in Python | `from acumen import …` | `references/python-api.md` |
+| A run failed / the skill did nothing | inspect `runs/…/result.json`, `logs/*.jsonl` | `references/benchmark.md` |
+
+## Preconditions
+
+- Python ≥ 3.12, and **`uv` on PATH** — acumen builds the target's venv with it.
+- The target (`repo`: a GitHub URL or a local path) must be pip-installable and declare
+  `[project].name` in `pyproject.toml`. A local `repo` path is resolved **relative to
+  `config.yaml`**.
+- Credentials: `bench` **always bills the Anthropic API** and refuses to run without
+  `ANTHROPIC_API_KEY` (or `ANTHROPIC_AUTH_TOKEN` / Bedrock / Vertex) — it records real
+  per-run `cost_usd`, and it has no `--auth` flag. `tasks`/`draft`/`improve`/`ship`
+  default to the Claude subscription if you are logged in (`--auth {auto,session,api}`).
+- The target is cloned + installed into a venv cached under `~/.cache/acumen`, keyed by
+  (repo, ref). Use `--refresh-target` after changing the target's own source.
+
+## What guessing gets wrong
+
+1. **Grading is exact string match on `answer.md` after `strip()`, case-sensitive.**
+   Nothing is normalized. `**TOKEN**` against `TOKEN` fails (recorded as `format_error`,
+   not `wrong_answer`). Task answers must be one short unambiguous token.
+2. **A pass is models × tasks × splits × replicates.** The scaffolded config lists 3
+   models and `n_replicates: 3`, so *one* task costs 18 agent runs *per arm*. Check with
+   `acumen bench --dry-run` (it plans and exits, spending nothing) and agree the size with
+   the user before spending. Trim with `models:`, `n_replicates: 1`, `--task ID`, `--split`.
+3. **`max_turns`/`max_usd` in `config.yaml` cap benchmark agents only.** `draft`,
+   `improve`, `tasks`, and `ship` are **unbounded** unless you pass `--max-turns`/`--max-usd`.
+4. **Skill versions are immutable.** `draft` refuses (exit 2) if any `skills/vN` exists;
+   `improve` always writes the next unused directory. Never edit a benched version in
+   place — its hash is recorded in every `result.json`.
+5. **`SKILL.md` frontmatter `name` must equal `config.skill_name`**, which defaults to the
+   repo's last path component, slugified and lowercased (`.../My_Pkg` → `my_pkg`). A
+   mismatch makes `bench`/`improve`/`ship` fail on load. `description` must be non-empty.
+6. **`improve` needs benched train evidence for its parent.** Run
+   `acumen bench --skill vN` before `acumen improve`, or it errors with nothing to read.
+7. **Never leak the test split.** `improve` is structurally and hook-blocked from
+   `runs/*/test/`; don't defeat that by pasting test answers into `--feedback`. A widening
+   train/test gap in the report is the overfitting signal you are watching for.
+8. **Task prompts must not name the target package** — the harness already tells the agent
+   which package to use and that it is installed.
+9. **Resume is automatic**: a run is "complete" when its `result.json` exists and is
+   non-empty, and completed runs are skipped. `--no-resume` re-runs them. Renaming a task
+   `id` orphans its old runs (the id is a path component).
+10. **Confirm the skill actually loaded.** `bench` prints `skill loaded in N/M runs` from
+    per-run `skill_loaded` evidence, and warns when a skill arm never fired the Skill tool
+    (that arm measures nothing) or when the baseline did.

acumen/_skills/data/references/authoring.md

@@ -0,0 +1,91 @@
+# Authoring skill versions: draft, improve, hand-edit
+
+## The skill directory contract
+
+```
+skills/v1/
+  SKILL.md          # required, YAML frontmatter with `name` and `description`
+  references/*.md   # optional
+  meta.json         # acumen bookkeeping — NOT part of the skill
+```
+
+- `name` must equal `config.skill_name` exactly; `description` must be non-empty. Both are
+  enforced on load by `bench`, `improve`, and `ship`.
+- `meta.json` (`version`, `parent`, `rationale`, `hash`, optional `feedback`) is written by
+  acumen. It is excluded from the content hash and is never copied to a consuming agent.
+- The content hash covers every other file's relative path and bytes, and is recorded in
+  every `result.json` — so editing a version after benching it silently invalidates the
+  comparison. **Versions are immutable; always make a new one.**
+
+## `acumen draft`
+
+```bash
+acumen draft [--force] [--feedback "…"] [--model M] [--max-turns N] [--max-usd X]
+             [--stream] [--log-dir logs] [--auth auto|session|api]
+```
+
+One agent that reads the target's **source** (docstrings, examples, docs) and writes
+`skills/v1/`. It works in a staging dir; only a skill that loads and validates is promoted,
+so a failed draft leaves no broken version behind. Exits 2 if any version already exists —
+pass `--force` to add another, or use `improve` to build on the latest.
+
+## `acumen improve`
+
+```bash
+acumen improve [--from vN] [--feedback "…"] [--model M] [--max-turns N] [--max-usd X]
+               [--stream] [--log-dir logs] [--auth auto|session|api]
+```
+
+Reads **how the parent version performed on the train split** — not the package source —
+and edits a copy of it into the next version.
+
+- Defaults to the latest version; `--from vN` picks another. Always writes the next unused
+  directory.
+- Requires `runs/skill_vN/train/**/result.json` to exist: bench the parent first.
+- The agent gets a curated copy of train runs (`SUMMARY.md` with failures first, plus each
+  run's prompt, expected, actual, `script.py`, and transcript). The real `runs/` tree is not
+  in scope, and a `PreToolUse` hook denies any path resolving under `runs/*/test/`.
+- The CLI warns if the new version is byte-identical to its parent (the improver changed
+  nothing) — that is a signal to give it sharper evidence or `--feedback`.
+
+## `--feedback` on `tasks` / `draft` / `improve`
+
+Free text injected into the agent's prompt as *subordinate* guidance — it never overrides
+the isolation or anti-overfit rules. Use it for what the agent cannot infer: package
+context, what to emphasise, which functionality to skip. For `draft`/`improve` it is
+recorded in `meta.json` and shown in the report. **Never paste test-split answers into
+`improve --feedback`** — that defeats the split.
+
+## Writing or editing a version by hand
+
+Perfectly supported; the agents are a convenience, not a requirement. A skill authored
+outside the project is benched the same way: copy the directory in as the next unused
+`skills/vN` — `bench` only ever loads versions from the project's `skills/` root.
+
+```bash
+cp -r skills/v2 skills/v3    # then edit skills/v3/, and delete its inherited meta.json
+acumen bench --skill v3
+```
+
+A hand-made version with no `meta.json` benches fine; it just has no rationale/diff in the
+report. What makes a version score well:
+
+- The `description` decides whether the skill loads at all — name the goals a user would
+  actually phrase, honestly. An unloaded skill is a wasted arm.
+- Keep `SKILL.md` short and push depth into `references/`; every token is paid on every
+  task, including ones where the skill is irrelevant.
+- Spend words on what an agent would get wrong by guessing: non-obvious defaults, required
+  preprocessing, the function that looks right but isn't, argument shapes/orientation,
+  where output lands, the right order of steps.
+- **Never name a dataset, parameter value, column, or expected answer from the train
+  tasks.** That is overfitting, and the test split will catch it.
+- Verify every claim against the installed package before writing it down.
+
+## Watching a long agent run
+
+`tasks`/`draft`/`improve`/`ship` each write `logs/acumen-<command>-<YYYYMMDD-HHMMSS>.jsonl`,
+one compact event per SDK message, **flushed as it goes** — read that file to follow
+progress instead of streaming into your context. Tool results are recorded by status and
+size, not inlined. `--stream` mirrors the conversation to the terminal; `--log-dir` moves
+the logs. A rendered `.html` transcript lands beside the jsonl at the end (skipped, with a
+note, if `claude-code-log` is missing).

acumen/_skills/data/references/benchmark.md

@@ -0,0 +1,100 @@
+# Benchmarking and reading results
+
+## `acumen bench`
+
+```bash
+acumen bench [--no-skill | --skill v1] [--split train|test]... [--task ID]...
+             [--replicates N] [--max-concurrency N] [--dry-run] [--no-resume]
+             [--keep-sandboxes] [--refresh-target]
+             [--config config.yaml] [--tasks tasks.yaml] [--runs runs] [--skills skills]
+             [--cache ~/.cache/acumen]
+```
+
+- Neither `--no-skill` nor `--skill` → the **baseline** arm (they are mutually exclusive).
+- `--split` and `--task` are repeatable; omitted means all.
+- `--dry-run` prints the planned matrix and exits **before** target prep and before any
+  agent runs — free, and the right way to check the size of a pass.
+- `--replicates` / `--max-concurrency` override the config for this pass only.
+- Always bills the API (`ANTHROPIC_API_KEY` etc.); there is no `--auth` here, because the
+  recorded `cost_usd` must be real metered spend.
+- Ctrl-C is safe: completed runs are preserved and the next invocation resumes.
+
+**Arm parity is the whole point.** Both arms get an identical prompt, tools, caps, and
+environment; the only difference is that a skill arm copies the skill's content files into
+`<sandbox>/.claude/skills/<skill_name>/` where project settings discovery finds it.
+`meta.json` is never copied. Each run gets a fresh empty sandbox with the target venv on
+PATH, a throwaway `HOME` and `CLAUDE_CONFIG_DIR` — no repo source, no user settings, no
+`CLAUDE.md` memories, no visibility of other runs.
+
+## The run tree
+
+```
+runs/{arm}/{split}/{model}/{task_id}/rep_{n}/
+```
+
+`arm` is `noskill` or `skill_v1`, `skill_v2`, …; `model` is slugified. Each completed leaf
+holds five files:
+
+| File | What |
+|---|---|
+| `answer.md` | What the agent wrote — the only thing graded |
+| `script.py` | The agent's reproduction script (absent if it ran no code) |
+| `transcript.jsonl` | SDK-native transcript |
+| `transcript.html` | Rendered transcript (via `claude-code-log`) |
+| `result.json` | The unit of record — written **last** |
+
+`result.json` presence + non-zero size is what marks a run complete, which is what makes
+resume safe. Useful fields: `success`, `reason`, `answer`, `expected`, `model`, `turns`,
+`cost_usd`, `input_tokens`, `output_tokens`, `duration_s`, `skill_hash`, `skill_name`,
+`skill_loaded`, `pkg_version`, `commit`, `session_id`, `subtype`, `error`.
+
+## Reasons
+
+| `reason` | Meaning |
+|---|---|
+| `ok` | Exact match — the only success |
+| `wrong_answer` | Content differs |
+| `format_error` | Content would match but formatting broke the exact match (bold, quotes, "Answer:", code fence, trailing period) |
+| `no_answer_file` | The agent never wrote `answer.md` |
+| `budget` / `max_turns` | Cap breached — a **failure regardless** of what `answer.md` contains |
+| `error` | The agent crashed or produced no result |
+
+A run of `format_error`s means the harness's answer-format instruction is losing to the
+task prompt; a run of `no_answer_file` usually means the task is too big for `max_turns`.
+
+## Diagnosing a disappointing arm
+
+1. `skill loaded in 0/N runs` on a skill arm → the arm measured nothing. The skill's
+   `description` is what decides whether it loads; make it name the goals the task prompts
+   actually phrase.
+2. The Skill tool firing in *baseline* runs is also flagged — that baseline is not clean.
+3. Open `runs/.../transcript.html` for a failing run, or re-run one cell with
+   `--task ID --split train --replicates 1 --keep-sandboxes` and inspect the sandbox.
+4. Compare `answer` vs `expected` in `result.json` before blaming the skill — a systematic
+   near-miss is a task-authoring problem, not a skill problem.
+
+## `acumen report`
+
+```bash
+acumen report [--runs runs] [--tasks tasks.yaml] [--skills skills] [--out report.html]
+              [--palette claude-opus-5=#3b7ea1]...
+```
+
+- Writes `report.html` **and a sidecar `report.csv`** next to it, overwriting both; the
+  report always reflects every run currently on disk across every arm.
+- Reads only `result.json` files — never transcripts. Fails if `runs/` holds none.
+- **The figures show the TEST split** (the held-out measure). The per-run table below them
+  lists both splits.
+- Sections: Overview (success rate, tokens, cost, time per arm, bars coloured by model),
+  Per-task breakdown, Runs table (links to each `transcript.html`), and — when `--skills`
+  resolves — Skill versions with each version's rationale and diff against its parent.
+- `--tasks` / `--skills` are optional; missing ones only drop their sections (it says so).
+- `--palette MODEL=COLOUR` is repeatable and accepts comma-separated pairs. A key that
+  matches no model in the data, or an unparseable colour, is rejected up front.
+
+Run `acumen report` after every bench — it is how a version is judged, and the only view
+that compares arms and splits side by side. Then take the results to the user: say what
+improved, what did not, and what you would do next (`improve` and re-bench, more tasks,
+`ship`, or stop). Train improving while test does not is overfitting — that gap is exactly
+what the split exists to expose — but the call on when to stop and what to ship is the
+user's.

acumen/_skills/data/references/python-api.md

@@ -0,0 +1,106 @@
+# Python API
+
+Everything is re-exported from the top level: `from acumen import build_report, load_config, …`.
+The CLI (`acumen.cli:main`) is a thin shell over it, so anything the CLI does is scriptable.
+`main(argv)` returns an exit code and converts the library's error types into `error: …` on
+stderr — call the functions directly if you want the exceptions.
+
+## Loading config and tasks
+
+```python
+from pathlib import Path
+from acumen import load_config, load_tasks, parse_config, parse_tasks
+
+cfg = load_config(Path("config.yaml"))    # validates; resolves a local repo relative to the file
+tasks = load_tasks(Path("tasks.yaml"))    # list[Task]; task.split("train") -> TaskSplit(prompt, answer)
+```
+
+`Config` is a frozen dataclass — override with `dataclasses.replace(cfg, n_replicates=1)`.
+`parse_config` / `parse_tasks` take already-parsed dicts, for building them in memory.
+
+## Preparing the target
+
+```python
+from acumen import prepare_target
+from acumen.env import DEFAULT_CACHE_ROOT
+
+target = prepare_target(cfg, DEFAULT_CACHE_ROOT, refresh=False)
+target.python      # venv interpreter with the package installed
+target.bin_dir     # what goes on an agent's PATH
+target.fingerprint # "<pkg> <version>", as recorded in result.json
+```
+
+Needs `uv` on PATH. Cached by (repo, ref) under `~/.cache/acumen`.
+
+## Planning and running a pass
+
+```python
+import asyncio
+from acumen import build_matrix, pending, run_matrix, load_skill, summarize
+
+skill = load_skill(Path("skills"), "v1", expect_name=cfg.skill_name)   # None for the baseline
+planned = build_matrix(cfg, tasks, skill="v1", splits=("train",), task_ids=["t1"])
+todo = pending(planned, Path("runs"), resume=True)
+outcomes = asyncio.run(run_matrix(
+    todo, target=target, runs_root=Path("runs"),
+    max_concurrency=cfg.max_concurrency, auth_mode="api", skill=skill,
+    on_start=lambda p: None, on_done=lambda o: None,
+    env_passthrough=cfg.env_passthrough,
+))
+summarize(outcomes)   # {"ok": 5, "wrong_answer": 1, ...}
+```
+
+`run_matrix` and `run_once` are coroutines. `run_once` records agent crashes as failed runs
+rather than raising, so one bad run never kills a pass. `run_once` raises `ValueError` if
+`key.arm` and the `skill` argument disagree — the arm is the source of truth.
+
+`build_matrix` (and `--dry-run`) is pure planning: no agents, no network, no cost.
+
+## Grading and paths
+
+```python
+from acumen import grade_answer, grade_run, run_dir, parse_run_dir, is_complete, RunKey, arm_name
+
+grade_answer("**SPI1**", "SPI1")   # Grade(success=False, reason='format_error', answer='**SPI1**')
+run_dir(Path("runs"), RunKey(arm=arm_name("v1"), split="test", model="claude-opus-5",
+                            task_id="t1", rep=1))
+is_complete(d)                     # a non-empty result.json is what "done" means
+```
+
+## The meta-agents
+
+`draft_skill`, `improve_skill`, `generate_tasks`, `ship_skill` are all coroutines taking
+keyword-only args (`cfg=`, `target=`, plus their own roots) and returning a result dataclass
+(`DraftResult`, `ImproveResult`, `TaskGenResult`, `ShipResult`) carrying the new `Skill`,
+`cost_usd`, `turns`, and log paths. `max_turns`/`max_usd` default to `None` = **unbounded**.
+Pass a `LiveLog` as `log=` for the JSONL feed:
+
+```python
+from acumen import LiveLog, draft_skill
+log = LiveLog.open(Path("logs"), "draft", stream=False)
+with log:
+    result = asyncio.run(draft_skill(cfg=cfg, target=target, skills_root=Path("skills"),
+                                     auth_mode="session", log=log))
+```
+
+## Aggregating results
+
+```python
+from acumen import load_results, arm_metrics, build_report
+
+df = load_results(Path("runs"))            # one row per result.json, + total_tokens, arm_label
+arm_metrics(df[df["split"] == "test"])     # per-arm rate, stderr, tokens, cost, time, n
+report = build_report(Path("runs"), Path("report.html"), tasks, skills_root=Path("skills"))
+report.n_runs, report.results              # the DataFrame behind the HTML
+```
+
+`build_report` writes `report.html` **and `report.csv`** (same stem) and returns `Report`.
+Filter `df` to a split yourself before `arm_metrics` — it does not.
+
+## Introspection helpers
+
+`available_versions`, `latest_version`, `next_version`, `skill_hash`, `skill_content`
+(for diffing), `installer_exists(src_dir)`, `collect_train_runs(runs_root, arm, tasks)`,
+and the two pure guards `find_test_access` / `find_skill_access` (testable without an agent).
+`scrubbed_env` / `build_agent_env` / `sandbox` / `install_skill` let you reproduce a run's
+exact isolated environment by hand.

acumen/_skills/data/references/setup.md

@@ -0,0 +1,94 @@
+# Project setup: `config.yaml` and `tasks.yaml`
+
+`acumen init [--dir DIR] [--force]` writes both files as annotated placeholders. It refuses
+to clobber either unless `--force`. Both loaders are **strict: unknown keys are rejected**.
+
+Nothing else can run until `config.yaml` is filled in: the scaffold ships
+`repo: https://github.com/OWNER/REPO` and a `REPLACE_ME` example task. Ask the user for the
+values only they know (target repo/path, ref, extras, which models, budgets) instead of
+inventing them.
+
+## config.yaml
+
+Only `repo` is required; delete a line to take its default.
+
+| Key | Default | Notes |
+|---|---|---|
+| `repo` | — | GitHub URL (`https://`, `git@`, `ssh://`, `git://`) or a local path. A local path is resolved relative to `config.yaml` and must exist. |
+| `ref` | `main` | Branch/tag/commit; ignored for local paths. |
+| `extras` | `[]` | pip extras installed with the target, e.g. `[test]`. |
+| `python` | `"3.12"` | Interpreter for the target's venv (quote it — `3.10` unquoted is a float). |
+| `env_passthrough` | `[]` | Extra env var names agents may keep. The agent env is a clean allowlist (auth/proxy/TLS only); **everything else from your shell is blanked**. A target needing `OMP_NUM_THREADS`, `R_HOME`, a service key, etc. must name it here. |
+| `models` | `[claude-opus-5]` | Benchmark models. Duplicates rejected. `models[0]` is the default for the four `*_model` keys below. |
+| `n_replicates` | `3` | Runs per (model, task, split) cell. |
+| `max_concurrency` | `4` | Simultaneous benchmark agents. |
+| `max_turns` | `40` | **Benchmark agents only.** |
+| `max_usd` | `3.0` | **Benchmark agents only.** |
+| `draft_model` / `improve_model` / `tasks_model` / `ship_model` | `models[0]` | Meta-agent models; overridable per command with `--model`. |
+| `skill_name` | repo basename, slugified + lowercased | Must equal the `name:` in the skill's frontmatter. |
+
+The scaffold's `models:` line lists three models — with `n_replicates: 3` that is 18 runs
+per task per arm. Cut it down before the first real pass.
+
+## tasks.yaml
+
+```yaml
+tasks:
+  - id: some_analysis            # unique, filesystem-safe: [A-Za-z0-9._-]; used as a path component
+    train:
+      prompt: >-
+        One paragraph: the goal, the input, and exactly what to report.
+      answer: ONE_TOKEN
+    test:
+      prompt: >-
+        The same analysis on a different input / target.
+      answer: ANOTHER_TOKEN
+    # optional per-task overrides — the only other allowed keys:
+    # max_turns: 60
+    # max_usd: 5.0
+    # model: claude-sonnet-5
+```
+
+Both `train` and `test` are required, each with non-empty `prompt` and `answer`. Both
+splits always run in a pass; only train results ever reach `improve`.
+
+### Writing a task that measures anything
+
+- **Answers are graded by exact string match after `strip()`, case-sensitive.** Keep the
+  answer to one token: a name, a category, a count, a number at a stated precision. End the
+  prompt by stating exactly what to report and in what form.
+- **Do not name the target package**, and do not mention a version — the harness preamble
+  already tells the agent the package is installed and which interpreter to use.
+- **State the goal, not the recipe.** No numbered steps, no function/argument names, no
+  description of the data's shape or columns. What is being measured is whether the agent
+  can find the "how" itself; a prompt that spells out the calls measures nothing.
+- Train and test must be **the same analysis on different inputs**, with different correct
+  answers — otherwise a skill can pass by memorizing one answer.
+- A good task is one the target package solves and the **no-skill baseline gets wrong** —
+  a task the baseline already passes leaves no room to show a gain. The `--no-skill` arm is
+  what tells you; it can be run before or after the first skill exists.
+- Renaming an `id` orphans every existing run under it.
+
+## `acumen tasks` — generate tasks.yaml
+
+```bash
+acumen tasks [--out tasks.yaml] [--force] [--feedback "skip the plotting API"] \
+             [--model M] [--max-turns N] [--max-usd X] [--stream] [--log-dir logs]
+```
+
+One autonomous agent that reads the package source **and executes it in the target venv**:
+every ground-truth answer comes from a script it actually ran, not from docs. It enumerates
+the package's tutorials/vignettes and writes at least one task per tutorial, so expect a
+sizeable file. Unbounded in turns and cost by default.
+
+- Refuses to overwrite an existing `--out` without `--force` (the check runs *before* the
+  costly target prep).
+- **There is no append mode.** The generator never reads your existing tasks file, so
+  appending would silently duplicate. To combine, generate to a separate path and merge by
+  hand.
+- Existing skills and agent-instruction files (`SKILL.md`, `.claude/`, `CLAUDE.md`,
+  `AGENTS.md`, `.cursor/`, root `skills/`) are stripped from the source copy it reads and
+  blocked by a hook, so pre-written guidance cannot bias which tasks it picks.
+- The output is validated through the strict tasks loader before it is written — it can
+  never emit a file the rest of the pipeline would reject.
+- **Review what it produces.** The answers are only as good as the scripts it ran.