acumen 0.0.1.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
acumen/taskgen.py ADDED
@@ -0,0 +1,424 @@
1
+ """The task-generation agent: mine the target package for real analyses and write ``tasks.yaml``.
2
+
3
+ ``acumen tasks`` autonomously benchmarks the target's *functionalities* — the analyses a user
4
+ would actually run — and writes a benchmark-ready ``tasks.yaml``. Like the drafter it
5
+ reads the package **source** (it has to understand the API to design a real pipeline) and, like
6
+ no benchmark agent, it also **runs code in the venv**: every ground-truth answer is obtained by
7
+ executing the pipeline and reading the real output, never by copying doc output.
8
+
9
+ There is no test-split guard here (unlike the improver) — no runs exist yet, so there is nothing
10
+ to leak. Isolation is otherwise the same as the other meta-agents: scrubbed env, throwaway
11
+ ``HOME`` and ``CLAUDE_CONFIG_DIR``.
12
+
13
+ **Existing skills must not bias task generation.** A target repo may already ship skills or
14
+ agent-instruction files (``SKILL.md``, ``.claude/skills/``, ``CLAUDE.md``, ``AGENTS.md``,
15
+ ``.cursor/``, Copilot instructions). If the generator read them, it would mine the tasks the
16
+ author already anticipated and phrase them the way the skill does — defeating the point of an
17
+ independent benchmark. So, as with the test-split guard, this is enforced two ways: the agent
18
+ reads a **filtered copy** of the source with those artifacts stripped out
19
+ (:func:`build_filtered_source`), and a ``PreToolUse`` hook denies any tool call that resolves to
20
+ one of them — or to the original unfiltered tree — wherever the agent points it
21
+ (:func:`find_skill_access`). ``setting_sources=[]`` additionally means no skill is ever
22
+ discovered or loaded into the generator itself.
23
+
24
+ The script the agent writes to confirm an answer is **scratch**: it lives in a throwaway work
25
+ dir that is deleted when this returns, so nothing about the ground-truth pipeline is persisted —
26
+ only the task (prompt + answer) lands in ``tasks.yaml``. The output is validated through
27
+ :func:`acumen.tasks.parse_tasks` before it is written, so ``acumen tasks`` can never emit a
28
+ ``tasks.yaml`` the rest of the pipeline would reject.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import shutil
34
+ import tempfile
35
+ from dataclasses import dataclass
36
+ from pathlib import Path
37
+ from typing import Any
38
+
39
+ import yaml
40
+ from claude_agent_sdk import ClaudeAgentOptions, HookMatcher, ResultMessage, query
41
+
42
+ from acumen.config import Config
43
+ from acumen.env import AuthMode, Target, build_agent_env
44
+ from acumen.logs import LiveLog
45
+ from acumen.procs import label_env, reap
46
+ from acumen.prompts import taskgen_prompt
47
+ from acumen.tasks import Task, TaskError, load_tasks
48
+
49
+ #: The filename the generation agent writes and we harvest from its work dir.
50
+ TASKS_FILE = "tasks.yaml"
51
+
52
+ #: Basenames anywhere in the tree that are agent-facing skill/guidance artifacts. Reading any
53
+ #: of these would let existing guidance bias which tasks the generator mines, so they are both
54
+ #: stripped from the source copy the agent reads and denied by the guard hook.
55
+ _ARTIFACT_BASENAMES = frozenset({"SKILL.md", "CLAUDE.md", "AGENTS.md", "copilot-instructions.md"})
56
+
57
+ #: Directory names that hold agent guidance; dropped from the copy and denied wherever they
58
+ #: appear on a resolved path (``.claude/skills``, ``.cursor/rules``, …).
59
+ _ARTIFACT_DIRS = frozenset({".claude", ".cursor"})
60
+
61
+ #: tool_input keys carrying a filesystem path — the coarse set the improver's guard also uses.
62
+ _PATH_KEYS = ("file_path", "path", "notebook_path", "filename")
63
+
64
+ #: Shell metacharacters we split a Bash command on to recover path-like tokens.
65
+ _SHELL_SPLIT = str.maketrans(dict.fromkeys("\"'`|&;<>()$" + "{}", " "))
66
+
67
+
68
+ class TaskGenError(RuntimeError):
69
+ """Raised when tasks could not be generated."""
70
+
71
+
72
+ @dataclass(frozen=True)
73
+ class TaskGenResult:
74
+ """The outcome of a generation run: the tasks written and what it cost."""
75
+
76
+ tasks: list[Task]
77
+ out_path: Path
78
+ cost_usd: float
79
+ turns: int
80
+ #: Live log paths for this run, when a :class:`LiveLog` was attached.
81
+ log_jsonl: Path | None = None
82
+ log_html: Path | None = None
83
+
84
+
85
+ # ── Skill-bias isolation ───────────────────────────────────────────────────────────────
86
+
87
+
88
+ def _copy_ignore(root: Path):
89
+ """A ``shutil.copytree`` ignore callback that drops skill/guidance artifacts (and ``.git``).
90
+
91
+ Skill directories and agent-instruction files are removed so the generator physically
92
+ cannot read them. A top-level ``skills/`` is treated as agent skills (the packaging
93
+ convention) and dropped at the root only, so a legitimately-named source directory deeper
94
+ in the tree is left alone.
95
+ """
96
+ root = root.resolve()
97
+
98
+ def ignore(dir_path: str, names: list[str]) -> set[str]:
99
+ here = Path(dir_path).resolve()
100
+ drop: set[str] = set()
101
+ for name in names:
102
+ if name == ".git" or name in _ARTIFACT_DIRS or name in _ARTIFACT_BASENAMES:
103
+ drop.add(name)
104
+ elif here == root and name == "skills":
105
+ drop.add(name)
106
+ return drop
107
+
108
+ return ignore
109
+
110
+
111
+ def build_filtered_source(src: Path, dest: Path) -> Path:
112
+ """Copy ``src`` to ``dest`` with skills and agent-guidance stripped out.
113
+
114
+ This is the structural half of the skill-bias isolation: the generator's ``add_dirs`` points
115
+ at the returned copy, not the real checkout, so existing skills are simply absent from what
116
+ it can read.
117
+
118
+ Parameters
119
+ ----------
120
+ src
121
+ The real target source checkout.
122
+ dest
123
+ Where to write the filtered copy; must not already exist.
124
+
125
+ Returns
126
+ -------
127
+ ``dest``.
128
+ """
129
+ shutil.copytree(src, dest, ignore=_copy_ignore(src), symlinks=True)
130
+ return dest
131
+
132
+
133
+ def _artifact_hit(candidate: str, original_src: Path) -> str | None:
134
+ """Return ``candidate`` if it resolves to a skill/guidance artifact or the original tree."""
135
+ try:
136
+ resolved = Path(candidate).expanduser().resolve()
137
+ except (OSError, RuntimeError, ValueError):
138
+ return None
139
+ if resolved.name in _ARTIFACT_BASENAMES:
140
+ return candidate
141
+ if set(resolved.parts) & _ARTIFACT_DIRS:
142
+ return candidate
143
+ # The unfiltered source tree is off-limits — the agent must read the filtered copy, so any
144
+ # path back into the original checkout (which still holds the stripped artifacts) is denied.
145
+ try:
146
+ resolved.relative_to(original_src)
147
+ except ValueError:
148
+ return None
149
+ return candidate
150
+
151
+
152
+ def find_skill_access(tool_name: str, tool_input: dict[str, Any], original_src: Path) -> str | None:
153
+ """Return the first path in a tool call that reaches a skill/guidance artifact, else ``None``.
154
+
155
+ Pure and side-effect free, so the enforcement can be exercised directly without standing up
156
+ an agent (mirrors :func:`acumen.improve.find_test_access`). Checks the path-bearing
157
+ tool_input keys and — for shell tools — the metacharacter-split command tokens, since a Bash
158
+ call can name a path no structured field would.
159
+
160
+ Parameters
161
+ ----------
162
+ tool_name
163
+ The tool being invoked; unused today but kept so the guard can special-case tools.
164
+ tool_input
165
+ The tool's arguments.
166
+ original_src
167
+ The real (unfiltered) source checkout, resolved by the caller.
168
+
169
+ Returns
170
+ -------
171
+ The offending path string, or ``None`` if the call touches no artifact.
172
+ """
173
+ for key in _PATH_KEYS:
174
+ value = tool_input.get(key)
175
+ if isinstance(value, str):
176
+ hit = _artifact_hit(value, original_src)
177
+ if hit is not None:
178
+ return hit
179
+ command = tool_input.get("command")
180
+ if isinstance(command, str):
181
+ for raw in command.translate(_SHELL_SPLIT).split():
182
+ token = raw.rstrip(",;")
183
+ if not token:
184
+ continue
185
+ hit = _artifact_hit(token, original_src)
186
+ if hit is not None:
187
+ return hit
188
+ return None
189
+
190
+
191
+ def make_skill_guard(original_src: Path) -> HookMatcher:
192
+ """Build the ``PreToolUse`` hook that denies the generator any existing skill/guidance.
193
+
194
+ ``matcher=None`` fires the hook for every tool. Paths are resolved against the real source
195
+ checkout, so the guard holds regardless of the agent's ``cwd``.
196
+ """
197
+ root = original_src.resolve()
198
+
199
+ async def guard(input_data: dict[str, Any], tool_use_id: str | None, context: Any) -> dict[str, Any]:
200
+ hit = find_skill_access(
201
+ input_data.get("tool_name", ""),
202
+ input_data.get("tool_input", {}) or {},
203
+ root,
204
+ )
205
+ if hit is None:
206
+ return {}
207
+ return {
208
+ "hookSpecificOutput": {
209
+ "hookEventName": "PreToolUse",
210
+ "permissionDecision": "deny",
211
+ "permissionDecisionReason": (
212
+ f"acumen hides existing skills and agent-instruction files from the task "
213
+ f"generator so they cannot bias task selection ({hit}). Design tasks from the "
214
+ "package's API and its user-facing docs only, using the provided source copy."
215
+ ),
216
+ }
217
+ }
218
+
219
+ return HookMatcher(matcher=None, hooks=[guard])
220
+
221
+
222
+ # ── Task serialisation ─────────────────────────────────────────────────────────────────
223
+
224
+
225
+ def _task_to_dict(task: Task) -> dict[str, object]:
226
+ """Serialise a :class:`Task` back to the ``tasks.yaml`` mapping shape."""
227
+ entry: dict[str, object] = {
228
+ "id": task.id,
229
+ "train": {"prompt": task.train.prompt, "answer": task.train.answer},
230
+ "test": {"prompt": task.test.prompt, "answer": task.test.answer},
231
+ }
232
+ if task.max_turns is not None:
233
+ entry["max_turns"] = task.max_turns
234
+ if task.max_usd is not None:
235
+ entry["max_usd"] = task.max_usd
236
+ if task.model is not None:
237
+ entry["model"] = task.model
238
+ return entry
239
+
240
+
241
+ class _TaskDumper(yaml.SafeDumper):
242
+ """A SafeDumper that renders multi-line strings as literal blocks, for readable prompts."""
243
+
244
+
245
+ def _represent_str(dumper: yaml.Dumper, value: str) -> yaml.ScalarNode:
246
+ # Multi-line prompts are far easier to review as `|` literal blocks than folded/quoted.
247
+ style = "|" if "\n" in value else None
248
+ return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style)
249
+
250
+
251
+ _TaskDumper.add_representer(str, _represent_str)
252
+
253
+
254
+ def dump_tasks(tasks: list[Task]) -> str:
255
+ """Render tasks as ``tasks.yaml`` text, preserving key order and multi-line prompts.
256
+
257
+ The inverse of :func:`acumen.tasks.load_tasks`. Round-tripping through this is how the
258
+ combined (existing + generated) task set is validated before anything is written to disk.
259
+ Multi-line prompts are emitted as ``|`` literal blocks so a reviewer can read them.
260
+ """
261
+ doc = {"tasks": [_task_to_dict(task) for task in tasks]}
262
+ return yaml.dump(doc, Dumper=_TaskDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=100)
263
+
264
+
265
+ def _validate_generated(staged: Path) -> list[Task]:
266
+ """Load and validate the agent's ``tasks.yaml``, mapping failures to a TaskGenError."""
267
+ if not staged.is_file():
268
+ raise TaskGenError(
269
+ f"the task-generation agent did not write {TASKS_FILE} — nothing to save. "
270
+ "Inspect the prompt or raise max_turns."
271
+ )
272
+ try:
273
+ return load_tasks(staged)
274
+ except TaskError as err:
275
+ raise TaskGenError(f"the generated {TASKS_FILE} is not valid: {err}") from err
276
+
277
+
278
+ async def generate_tasks(
279
+ *,
280
+ cfg: Config,
281
+ target: Target,
282
+ out_path: Path,
283
+ auth_mode: AuthMode = "session",
284
+ model: str | None = None,
285
+ max_turns: int | None = None,
286
+ max_usd: float | None = None,
287
+ force: bool = False,
288
+ feedback: str | None = None,
289
+ log: LiveLog | None = None,
290
+ ) -> TaskGenResult:
291
+ """Generate a ``tasks.yaml`` for the target package by mining and executing its analyses.
292
+
293
+ Parameters
294
+ ----------
295
+ cfg
296
+ The pass config; supplies ``tasks_model``.
297
+ target
298
+ The prepared target, supplying the source checkout and the interpreter to run
299
+ pipelines against for ground truth.
300
+ auth_mode
301
+ Which credential the generation agent authenticates with — ``"session"`` (the Claude
302
+ subscription) or ``"api"`` (see :func:`acumen.env.build_agent_env`).
303
+ out_path
304
+ Where the tasks are written. Refuses to overwrite an existing file unless ``force``
305
+ is set. There is deliberately no "append" mode: the agent generates blind to the
306
+ existing file (it never reads it — see the module docstring on isolation), so it cannot
307
+ avoid re-covering functionality already present, and appending would silently grow the
308
+ set with semantic duplicates. To combine generated tasks with a curated file, write to a
309
+ separate ``out_path`` and merge by hand, where the overlap can actually be judged.
310
+ model
311
+ Override for the generation model; defaults to ``cfg.tasks_model``.
312
+ max_turns, max_usd
313
+ Caps for the generation agent. **Unbounded by default**: generating tasks means
314
+ running package code iteratively, so no default budget is imposed — pass explicit caps
315
+ to bound it.
316
+ force
317
+ Overwrite an existing ``out_path`` (e.g. the placeholder from ``acumen init``).
318
+ feedback
319
+ Optional maintainer guidance, injected into the generation prompt as a subordinated
320
+ block — e.g. which functionality to skip. Nothing but ``tasks.yaml`` is persisted (task
321
+ generation writes no meta), so the feedback is not recorded on disk.
322
+ log
323
+ A :class:`LiveLog` to stream the agent's messages to and render an HTML log from.
324
+
325
+ Returns
326
+ -------
327
+ The generation result: the task set written and what it cost.
328
+ """
329
+ if out_path.exists() and not force:
330
+ raise TaskGenError(f"{out_path} already exists — pass force=True to overwrite it")
331
+
332
+ holder = Path(tempfile.mkdtemp(prefix="acumen-tasks-"))
333
+ try:
334
+ work = holder / "work"
335
+ home = holder / "home"
336
+ config_dir = home / ".claude"
337
+ for path in (work, home, config_dir, home / "tmp"):
338
+ path.mkdir(parents=True, exist_ok=True)
339
+ staged = work / TASKS_FILE
340
+
341
+ # The agent reads a copy of the source with skills/agent-guidance stripped, never the
342
+ # real checkout — so existing skills cannot bias the tasks it generates.
343
+ source_copy = build_filtered_source(target.src_dir, holder / "source")
344
+
345
+ # Marks the agent's processes so the teardown below can find what it leaves running.
346
+ env = label_env(
347
+ build_agent_env(
348
+ config_dir=config_dir,
349
+ home=home,
350
+ extra_path=[target.bin_dir],
351
+ auth_mode=auth_mode,
352
+ extra_allow=cfg.env_passthrough,
353
+ ),
354
+ holder,
355
+ )
356
+
357
+ prompt = taskgen_prompt(
358
+ package=target.pkg_name,
359
+ src=source_copy,
360
+ python=target.python,
361
+ out=staged,
362
+ feedback=feedback,
363
+ )
364
+ options = ClaudeAgentOptions(
365
+ cwd=str(work),
366
+ env=env,
367
+ model=model or cfg.tasks_model,
368
+ # No default budget cap: only bound the agent if the caller asked.
369
+ max_turns=max_turns,
370
+ max_budget_usd=max_usd,
371
+ # The generator reads the target source, like the drafter; benchmark agents
372
+ # never do. It points at the *filtered* copy, not the real checkout.
373
+ add_dirs=[str(source_copy)],
374
+ # No skill discovery at all — the generator must not load a skill that would bias it.
375
+ setting_sources=[],
376
+ permission_mode="bypassPermissions",
377
+ system_prompt={"type": "preset", "preset": "claude_code"},
378
+ # Belt-and-braces over the filtered copy: deny any call that reaches an existing
379
+ # skill/guidance artifact or the original unfiltered source, wherever pointed.
380
+ hooks={"PreToolUse": [make_skill_guard(target.src_dir)]},
381
+ )
382
+
383
+ result: ResultMessage | None = None
384
+ agent_error: Exception | None = None
385
+ try:
386
+ async for message in query(prompt=prompt, options=options):
387
+ if log is not None:
388
+ log.append(message)
389
+ if isinstance(message, ResultMessage):
390
+ result = message
391
+ except Exception as err: # noqa: BLE001 - a failed generation is an error to report, re-raised below
392
+ agent_error = err
393
+ finally:
394
+ # Render the HTML log while the throwaway config dir still holds the native
395
+ # transcript — in a finally so an aborted run (the SDK raises on a cap breach,
396
+ # after yielding the result) is still inspectable.
397
+ if log is not None:
398
+ log.finalize(config_dir=config_dir, work_dir=work, result=result)
399
+
400
+ if agent_error is not None:
401
+ raise TaskGenError(
402
+ f"the task-generation agent failed: {type(agent_error).__name__}: {agent_error}"
403
+ ) from agent_error
404
+ if result is None:
405
+ raise TaskGenError("the task-generation agent produced no result message")
406
+ if result.is_error:
407
+ raise TaskGenError(f"the task-generation agent errored: {result.subtype} {result.errors or ''}".strip())
408
+
409
+ generated = _validate_generated(staged)
410
+
411
+ out_path.parent.mkdir(parents=True, exist_ok=True)
412
+ out_path.write_text(dump_tasks(generated))
413
+ return TaskGenResult(
414
+ tasks=generated,
415
+ out_path=out_path,
416
+ cost_usd=result.total_cost_usd or 0.0,
417
+ turns=result.num_turns,
418
+ log_jsonl=log.jsonl_path if log is not None else None,
419
+ log_html=log.html_path if log is not None and log.html_rendered else None,
420
+ )
421
+ finally:
422
+ # Kill anything the agent left running before removing the directory it runs in.
423
+ reap(holder)
424
+ shutil.rmtree(holder, ignore_errors=True)
acumen/tasks.py ADDED
@@ -0,0 +1,144 @@
1
+ """Task schema and loader for ``tasks.yaml``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import yaml
10
+
11
+ from acumen.paths import Split, is_safe_component
12
+
13
+
14
+ class TaskError(ValueError):
15
+ """Raised when ``tasks.yaml`` is malformed."""
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class TaskSplit:
20
+ """One half of a task: a prompt and the answer it must produce."""
21
+
22
+ prompt: str
23
+ answer: str
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class Task:
28
+ """A single benchmark task, with a train/test pair and optional overrides."""
29
+
30
+ id: str
31
+ train: TaskSplit
32
+ test: TaskSplit
33
+ max_turns: int | None = None
34
+ max_usd: float | None = None
35
+ model: str | None = None
36
+
37
+ def split(self, split: Split) -> TaskSplit:
38
+ """Return the ``train`` or ``test`` half of this task."""
39
+ if split not in ("train", "test"):
40
+ raise TaskError(f"no such split: {split!r}")
41
+ return self.train if split == "train" else self.test
42
+
43
+
44
+ def _require_str(value: Any, where: str) -> str:
45
+ if not isinstance(value, str) or not value.strip():
46
+ raise TaskError(f"{where} must be a non-empty string, got {value!r}")
47
+ return value
48
+
49
+
50
+ def _parse_split(raw: Any, where: str) -> TaskSplit:
51
+ if not isinstance(raw, dict):
52
+ raise TaskError(f"{where} must be a mapping with 'prompt' and 'answer', got {type(raw).__name__}")
53
+ unknown = set(raw) - {"prompt", "answer"}
54
+ if unknown:
55
+ raise TaskError(f"{where} has unknown keys: {sorted(unknown)}")
56
+ for key in ("prompt", "answer"):
57
+ if key not in raw:
58
+ raise TaskError(f"{where} is missing '{key}'")
59
+ return TaskSplit(
60
+ prompt=_require_str(raw["prompt"], f"{where}.prompt"), answer=_require_str(raw["answer"], f"{where}.answer")
61
+ )
62
+
63
+
64
+ def _parse_task(raw: Any, index: int) -> Task:
65
+ where = f"tasks[{index}]"
66
+ if not isinstance(raw, dict):
67
+ raise TaskError(f"{where} must be a mapping, got {type(raw).__name__}")
68
+ unknown = set(raw) - {"id", "train", "test", "max_turns", "max_usd", "model"}
69
+ if unknown:
70
+ raise TaskError(f"{where} has unknown keys: {sorted(unknown)}")
71
+ if "id" not in raw:
72
+ raise TaskError(f"{where} is missing 'id'")
73
+ task_id = _require_str(raw["id"], f"{where}.id")
74
+ if not is_safe_component(task_id):
75
+ raise TaskError(f"{where}.id {task_id!r} is not filesystem-safe — use only letters, digits, '.', '_', '-'")
76
+ where = f"task {task_id!r}"
77
+ for key in ("train", "test"):
78
+ if key not in raw:
79
+ raise TaskError(f"{where} is missing the '{key}' split — both splits are required")
80
+ max_turns = raw.get("max_turns")
81
+ if max_turns is not None and (not isinstance(max_turns, int) or isinstance(max_turns, bool) or max_turns < 1):
82
+ raise TaskError(f"{where}.max_turns must be a positive integer, got {max_turns!r}")
83
+ max_usd = raw.get("max_usd")
84
+ if max_usd is not None:
85
+ if isinstance(max_usd, bool) or not isinstance(max_usd, int | float) or max_usd <= 0:
86
+ raise TaskError(f"{where}.max_usd must be a positive number, got {max_usd!r}")
87
+ max_usd = float(max_usd)
88
+ model = raw.get("model")
89
+ if model is not None:
90
+ model = _require_str(model, f"{where}.model")
91
+ return Task(
92
+ id=task_id,
93
+ train=_parse_split(raw["train"], f"{where}.train"),
94
+ test=_parse_split(raw["test"], f"{where}.test"),
95
+ max_turns=max_turns,
96
+ max_usd=max_usd,
97
+ model=model,
98
+ )
99
+
100
+
101
+ def parse_tasks(raw: Any) -> list[Task]:
102
+ """Validate an already-parsed ``tasks.yaml`` document.
103
+
104
+ Parameters
105
+ ----------
106
+ raw
107
+ The mapping loaded from YAML.
108
+
109
+ Returns
110
+ -------
111
+ The tasks, in file order.
112
+ """
113
+ if not isinstance(raw, dict):
114
+ raise TaskError(f"tasks.yaml must be a mapping with a 'tasks' key, got {type(raw).__name__}")
115
+ if "tasks" not in raw:
116
+ raise TaskError("tasks.yaml is missing the top-level 'tasks' key")
117
+ entries = raw["tasks"]
118
+ if not isinstance(entries, list) or not entries:
119
+ raise TaskError("'tasks' must be a non-empty list")
120
+ tasks = [_parse_task(entry, i) for i, entry in enumerate(entries)]
121
+ seen: dict[str, int] = {}
122
+ for i, task in enumerate(tasks):
123
+ if task.id in seen:
124
+ raise TaskError(
125
+ f"duplicate task id {task.id!r} (tasks[{seen[task.id]}] and tasks[{i}]) — ids must be unique"
126
+ )
127
+ seen[task.id] = i
128
+ return tasks
129
+
130
+
131
+ def load_tasks(path: Path) -> list[Task]:
132
+ """Load and validate ``tasks.yaml`` from disk."""
133
+ try:
134
+ text = path.read_text()
135
+ except OSError as err:
136
+ raise TaskError(f"cannot read tasks file {path}: {err}") from err
137
+ try:
138
+ raw = yaml.safe_load(text)
139
+ except yaml.YAMLError as err:
140
+ raise TaskError(f"{path} is not valid YAML: {err}") from err
141
+ try:
142
+ return parse_tasks(raw)
143
+ except TaskError as err:
144
+ raise TaskError(f"{path}: {err}") from err
acumen/transcript.py ADDED
@@ -0,0 +1,77 @@
1
+ """Locating and rendering SDK-native transcripts — shared by the runner and the meta-agents.
2
+
3
+ The benchmark runner has always done this per run: after an agent finishes, find the
4
+ transcript the ``claude`` CLI wrote (under ``<CLAUDE_CONFIG_DIR>/projects/<key>/<session>.jsonl``),
5
+ copy it into the run dir, and render it to HTML with ``claude-code-log``. This module factors
6
+ that machinery out here so the four single-agent commands (``draft``/``improve``/``tasks``/``ship``)
7
+ can render an HTML log of their own agent the same way, instead of losing the transcript to the
8
+ ``rmtree`` of a throwaway config dir.
9
+
10
+ The transcript path is fully determined by the throwaway ``CLAUDE_CONFIG_DIR`` and the agent's
11
+ ``cwd`` (both of which acumen sets) plus the ``session_id`` on the ``ResultMessage``.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import shutil
17
+ import subprocess
18
+ import sys
19
+ from pathlib import Path
20
+
21
+ from claude_agent_sdk import project_key_for_directory
22
+
23
+
24
+ def locate_transcript(config_dir: Path, work_dir: Path, session_id: str) -> Path | None:
25
+ """Find the SDK-native transcript for a finished agent run.
26
+
27
+ The ``claude`` CLI writes transcripts to
28
+ ``<config_dir>/projects/<project_key_for(work_dir)>/<session_id>.jsonl``. That encoding is
29
+ deterministic, but fall back to a glob rather than lose the transcript if the CLI ever keys
30
+ the directory differently.
31
+
32
+ Parameters
33
+ ----------
34
+ config_dir
35
+ The ``CLAUDE_CONFIG_DIR`` the agent ran under (throwaway, for isolated agents; the
36
+ user's real one for the unisolated ship agent).
37
+ work_dir
38
+ The agent's ``cwd`` — what the project key is computed from.
39
+ session_id
40
+ ``ResultMessage.session_id`` for the run.
41
+
42
+ Returns
43
+ -------
44
+ The transcript path, or ``None`` if it could not be found.
45
+ """
46
+ transcript_root = config_dir / "projects"
47
+ key = project_key_for_directory(str(work_dir))
48
+ direct = transcript_root / key / f"{session_id}.jsonl"
49
+ if direct.is_file():
50
+ return direct
51
+ matches = sorted(transcript_root.glob(f"**/{session_id}.jsonl"))
52
+ return matches[0] if matches else None
53
+
54
+
55
+ def claude_code_log() -> str | None:
56
+ """Locate the ``claude-code-log`` CLI.
57
+
58
+ It is a dependency of acumen, so it lives next to the interpreter running us — look there
59
+ before PATH, which won't contain the venv's ``bin`` when acumen is invoked by absolute path
60
+ rather than through an activated venv.
61
+ """
62
+ local = Path(sys.executable).parent / "claude-code-log"
63
+ if local.is_file():
64
+ return str(local)
65
+ return shutil.which("claude-code-log")
66
+
67
+
68
+ def render_transcript(jsonl: Path, html: Path) -> bool:
69
+ """Render an SDK-native transcript to a standalone HTML file via ``claude-code-log``.
70
+
71
+ Returns whether the render succeeded and produced the file.
72
+ """
73
+ cli = claude_code_log()
74
+ if cli is None:
75
+ return False
76
+ proc = subprocess.run([cli, str(jsonl), "-o", str(html)], capture_output=True, text=True)
77
+ return proc.returncode == 0 and html.is_file()