acumen 0.0.1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acumen/__init__.py +160 -0
- acumen/_skills/__init__.py +0 -0
- acumen/_skills/data/SKILL.md +109 -0
- acumen/_skills/data/references/authoring.md +91 -0
- acumen/_skills/data/references/benchmark.md +100 -0
- acumen/_skills/data/references/python-api.md +106 -0
- acumen/_skills/data/references/setup.md +94 -0
- acumen/_skills/data/references/ship.md +53 -0
- acumen/_skills/install.py +185 -0
- acumen/assets/banner.svg +501 -0
- acumen/assets/logo.svg +47 -0
- acumen/bench.py +202 -0
- acumen/cli.py +626 -0
- acumen/config.py +204 -0
- acumen/draft.py +202 -0
- acumen/env.py +602 -0
- acumen/grade.py +104 -0
- acumen/improve.py +596 -0
- acumen/logs.py +255 -0
- acumen/paths.py +164 -0
- acumen/procs.py +195 -0
- acumen/prompts.py +942 -0
- acumen/report.py +1858 -0
- acumen/runner.py +343 -0
- acumen/sandbox.py +153 -0
- acumen/scaffold.py +118 -0
- acumen/ship.py +298 -0
- acumen/skills.py +298 -0
- acumen/taskgen.py +424 -0
- acumen/tasks.py +144 -0
- acumen/transcript.py +77 -0
- acumen-0.0.1.dev0.dist-info/METADATA +168 -0
- acumen-0.0.1.dev0.dist-info/RECORD +36 -0
- acumen-0.0.1.dev0.dist-info/WHEEL +4 -0
- acumen-0.0.1.dev0.dist-info/entry_points.txt +3 -0
- acumen-0.0.1.dev0.dist-info/licenses/LICENSE +21 -0
acumen/taskgen.py
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
"""The task-generation agent: mine the target package for real analyses and write ``tasks.yaml``.
|
|
2
|
+
|
|
3
|
+
``acumen tasks`` autonomously benchmarks the target's *functionalities* — the analyses a user
|
|
4
|
+
would actually run — and writes a benchmark-ready ``tasks.yaml``. Like the drafter it
|
|
5
|
+
reads the package **source** (it has to understand the API to design a real pipeline) and, like
|
|
6
|
+
no benchmark agent, it also **runs code in the venv**: every ground-truth answer is obtained by
|
|
7
|
+
executing the pipeline and reading the real output, never by copying doc output.
|
|
8
|
+
|
|
9
|
+
There is no test-split guard here (unlike the improver) — no runs exist yet, so there is nothing
|
|
10
|
+
to leak. Isolation is otherwise the same as the other meta-agents: scrubbed env, throwaway
|
|
11
|
+
``HOME`` and ``CLAUDE_CONFIG_DIR``.
|
|
12
|
+
|
|
13
|
+
**Existing skills must not bias task generation.** A target repo may already ship skills or
|
|
14
|
+
agent-instruction files (``SKILL.md``, ``.claude/skills/``, ``CLAUDE.md``, ``AGENTS.md``,
|
|
15
|
+
``.cursor/``, Copilot instructions). If the generator read them, it would mine the tasks the
|
|
16
|
+
author already anticipated and phrase them the way the skill does — defeating the point of an
|
|
17
|
+
independent benchmark. So, as with the test-split guard, this is enforced two ways: the agent
|
|
18
|
+
reads a **filtered copy** of the source with those artifacts stripped out
|
|
19
|
+
(:func:`build_filtered_source`), and a ``PreToolUse`` hook denies any tool call that resolves to
|
|
20
|
+
one of them — or to the original unfiltered tree — wherever the agent points it
|
|
21
|
+
(:func:`find_skill_access`). ``setting_sources=[]`` additionally means no skill is ever
|
|
22
|
+
discovered or loaded into the generator itself.
|
|
23
|
+
|
|
24
|
+
The script the agent writes to confirm an answer is **scratch**: it lives in a throwaway work
|
|
25
|
+
dir that is deleted when this returns, so nothing about the ground-truth pipeline is persisted —
|
|
26
|
+
only the task (prompt + answer) lands in ``tasks.yaml``. The output is validated through
|
|
27
|
+
:func:`acumen.tasks.parse_tasks` before it is written, so ``acumen tasks`` can never emit a
|
|
28
|
+
``tasks.yaml`` the rest of the pipeline would reject.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import shutil
|
|
34
|
+
import tempfile
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from typing import Any
|
|
38
|
+
|
|
39
|
+
import yaml
|
|
40
|
+
from claude_agent_sdk import ClaudeAgentOptions, HookMatcher, ResultMessage, query
|
|
41
|
+
|
|
42
|
+
from acumen.config import Config
|
|
43
|
+
from acumen.env import AuthMode, Target, build_agent_env
|
|
44
|
+
from acumen.logs import LiveLog
|
|
45
|
+
from acumen.procs import label_env, reap
|
|
46
|
+
from acumen.prompts import taskgen_prompt
|
|
47
|
+
from acumen.tasks import Task, TaskError, load_tasks
|
|
48
|
+
|
|
49
|
+
#: The filename the generation agent writes and we harvest from its work dir.
|
|
50
|
+
TASKS_FILE = "tasks.yaml"
|
|
51
|
+
|
|
52
|
+
#: Basenames anywhere in the tree that are agent-facing skill/guidance artifacts. Reading any
|
|
53
|
+
#: of these would let existing guidance bias which tasks the generator mines, so they are both
|
|
54
|
+
#: stripped from the source copy the agent reads and denied by the guard hook.
|
|
55
|
+
_ARTIFACT_BASENAMES = frozenset({"SKILL.md", "CLAUDE.md", "AGENTS.md", "copilot-instructions.md"})
|
|
56
|
+
|
|
57
|
+
#: Directory names that hold agent guidance; dropped from the copy and denied wherever they
|
|
58
|
+
#: appear on a resolved path (``.claude/skills``, ``.cursor/rules``, …).
|
|
59
|
+
_ARTIFACT_DIRS = frozenset({".claude", ".cursor"})
|
|
60
|
+
|
|
61
|
+
#: tool_input keys carrying a filesystem path — the coarse set the improver's guard also uses.
|
|
62
|
+
_PATH_KEYS = ("file_path", "path", "notebook_path", "filename")
|
|
63
|
+
|
|
64
|
+
#: Shell metacharacters we split a Bash command on to recover path-like tokens.
|
|
65
|
+
_SHELL_SPLIT = str.maketrans(dict.fromkeys("\"'`|&;<>()$" + "{}", " "))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class TaskGenError(RuntimeError):
|
|
69
|
+
"""Raised when tasks could not be generated."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class TaskGenResult:
|
|
74
|
+
"""The outcome of a generation run: the tasks written and what it cost."""
|
|
75
|
+
|
|
76
|
+
tasks: list[Task]
|
|
77
|
+
out_path: Path
|
|
78
|
+
cost_usd: float
|
|
79
|
+
turns: int
|
|
80
|
+
#: Live log paths for this run, when a :class:`LiveLog` was attached.
|
|
81
|
+
log_jsonl: Path | None = None
|
|
82
|
+
log_html: Path | None = None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# ── Skill-bias isolation ───────────────────────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _copy_ignore(root: Path):
|
|
89
|
+
"""A ``shutil.copytree`` ignore callback that drops skill/guidance artifacts (and ``.git``).
|
|
90
|
+
|
|
91
|
+
Skill directories and agent-instruction files are removed so the generator physically
|
|
92
|
+
cannot read them. A top-level ``skills/`` is treated as agent skills (the packaging
|
|
93
|
+
convention) and dropped at the root only, so a legitimately-named source directory deeper
|
|
94
|
+
in the tree is left alone.
|
|
95
|
+
"""
|
|
96
|
+
root = root.resolve()
|
|
97
|
+
|
|
98
|
+
def ignore(dir_path: str, names: list[str]) -> set[str]:
|
|
99
|
+
here = Path(dir_path).resolve()
|
|
100
|
+
drop: set[str] = set()
|
|
101
|
+
for name in names:
|
|
102
|
+
if name == ".git" or name in _ARTIFACT_DIRS or name in _ARTIFACT_BASENAMES:
|
|
103
|
+
drop.add(name)
|
|
104
|
+
elif here == root and name == "skills":
|
|
105
|
+
drop.add(name)
|
|
106
|
+
return drop
|
|
107
|
+
|
|
108
|
+
return ignore
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def build_filtered_source(src: Path, dest: Path) -> Path:
|
|
112
|
+
"""Copy ``src`` to ``dest`` with skills and agent-guidance stripped out.
|
|
113
|
+
|
|
114
|
+
This is the structural half of the skill-bias isolation: the generator's ``add_dirs`` points
|
|
115
|
+
at the returned copy, not the real checkout, so existing skills are simply absent from what
|
|
116
|
+
it can read.
|
|
117
|
+
|
|
118
|
+
Parameters
|
|
119
|
+
----------
|
|
120
|
+
src
|
|
121
|
+
The real target source checkout.
|
|
122
|
+
dest
|
|
123
|
+
Where to write the filtered copy; must not already exist.
|
|
124
|
+
|
|
125
|
+
Returns
|
|
126
|
+
-------
|
|
127
|
+
``dest``.
|
|
128
|
+
"""
|
|
129
|
+
shutil.copytree(src, dest, ignore=_copy_ignore(src), symlinks=True)
|
|
130
|
+
return dest
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _artifact_hit(candidate: str, original_src: Path) -> str | None:
|
|
134
|
+
"""Return ``candidate`` if it resolves to a skill/guidance artifact or the original tree."""
|
|
135
|
+
try:
|
|
136
|
+
resolved = Path(candidate).expanduser().resolve()
|
|
137
|
+
except (OSError, RuntimeError, ValueError):
|
|
138
|
+
return None
|
|
139
|
+
if resolved.name in _ARTIFACT_BASENAMES:
|
|
140
|
+
return candidate
|
|
141
|
+
if set(resolved.parts) & _ARTIFACT_DIRS:
|
|
142
|
+
return candidate
|
|
143
|
+
# The unfiltered source tree is off-limits — the agent must read the filtered copy, so any
|
|
144
|
+
# path back into the original checkout (which still holds the stripped artifacts) is denied.
|
|
145
|
+
try:
|
|
146
|
+
resolved.relative_to(original_src)
|
|
147
|
+
except ValueError:
|
|
148
|
+
return None
|
|
149
|
+
return candidate
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def find_skill_access(tool_name: str, tool_input: dict[str, Any], original_src: Path) -> str | None:
|
|
153
|
+
"""Return the first path in a tool call that reaches a skill/guidance artifact, else ``None``.
|
|
154
|
+
|
|
155
|
+
Pure and side-effect free, so the enforcement can be exercised directly without standing up
|
|
156
|
+
an agent (mirrors :func:`acumen.improve.find_test_access`). Checks the path-bearing
|
|
157
|
+
tool_input keys and — for shell tools — the metacharacter-split command tokens, since a Bash
|
|
158
|
+
call can name a path no structured field would.
|
|
159
|
+
|
|
160
|
+
Parameters
|
|
161
|
+
----------
|
|
162
|
+
tool_name
|
|
163
|
+
The tool being invoked; unused today but kept so the guard can special-case tools.
|
|
164
|
+
tool_input
|
|
165
|
+
The tool's arguments.
|
|
166
|
+
original_src
|
|
167
|
+
The real (unfiltered) source checkout, resolved by the caller.
|
|
168
|
+
|
|
169
|
+
Returns
|
|
170
|
+
-------
|
|
171
|
+
The offending path string, or ``None`` if the call touches no artifact.
|
|
172
|
+
"""
|
|
173
|
+
for key in _PATH_KEYS:
|
|
174
|
+
value = tool_input.get(key)
|
|
175
|
+
if isinstance(value, str):
|
|
176
|
+
hit = _artifact_hit(value, original_src)
|
|
177
|
+
if hit is not None:
|
|
178
|
+
return hit
|
|
179
|
+
command = tool_input.get("command")
|
|
180
|
+
if isinstance(command, str):
|
|
181
|
+
for raw in command.translate(_SHELL_SPLIT).split():
|
|
182
|
+
token = raw.rstrip(",;")
|
|
183
|
+
if not token:
|
|
184
|
+
continue
|
|
185
|
+
hit = _artifact_hit(token, original_src)
|
|
186
|
+
if hit is not None:
|
|
187
|
+
return hit
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def make_skill_guard(original_src: Path) -> HookMatcher:
|
|
192
|
+
"""Build the ``PreToolUse`` hook that denies the generator any existing skill/guidance.
|
|
193
|
+
|
|
194
|
+
``matcher=None`` fires the hook for every tool. Paths are resolved against the real source
|
|
195
|
+
checkout, so the guard holds regardless of the agent's ``cwd``.
|
|
196
|
+
"""
|
|
197
|
+
root = original_src.resolve()
|
|
198
|
+
|
|
199
|
+
async def guard(input_data: dict[str, Any], tool_use_id: str | None, context: Any) -> dict[str, Any]:
|
|
200
|
+
hit = find_skill_access(
|
|
201
|
+
input_data.get("tool_name", ""),
|
|
202
|
+
input_data.get("tool_input", {}) or {},
|
|
203
|
+
root,
|
|
204
|
+
)
|
|
205
|
+
if hit is None:
|
|
206
|
+
return {}
|
|
207
|
+
return {
|
|
208
|
+
"hookSpecificOutput": {
|
|
209
|
+
"hookEventName": "PreToolUse",
|
|
210
|
+
"permissionDecision": "deny",
|
|
211
|
+
"permissionDecisionReason": (
|
|
212
|
+
f"acumen hides existing skills and agent-instruction files from the task "
|
|
213
|
+
f"generator so they cannot bias task selection ({hit}). Design tasks from the "
|
|
214
|
+
"package's API and its user-facing docs only, using the provided source copy."
|
|
215
|
+
),
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
return HookMatcher(matcher=None, hooks=[guard])
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ── Task serialisation ─────────────────────────────────────────────────────────────────
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _task_to_dict(task: Task) -> dict[str, object]:
|
|
226
|
+
"""Serialise a :class:`Task` back to the ``tasks.yaml`` mapping shape."""
|
|
227
|
+
entry: dict[str, object] = {
|
|
228
|
+
"id": task.id,
|
|
229
|
+
"train": {"prompt": task.train.prompt, "answer": task.train.answer},
|
|
230
|
+
"test": {"prompt": task.test.prompt, "answer": task.test.answer},
|
|
231
|
+
}
|
|
232
|
+
if task.max_turns is not None:
|
|
233
|
+
entry["max_turns"] = task.max_turns
|
|
234
|
+
if task.max_usd is not None:
|
|
235
|
+
entry["max_usd"] = task.max_usd
|
|
236
|
+
if task.model is not None:
|
|
237
|
+
entry["model"] = task.model
|
|
238
|
+
return entry
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class _TaskDumper(yaml.SafeDumper):
|
|
242
|
+
"""A SafeDumper that renders multi-line strings as literal blocks, for readable prompts."""
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _represent_str(dumper: yaml.Dumper, value: str) -> yaml.ScalarNode:
|
|
246
|
+
# Multi-line prompts are far easier to review as `|` literal blocks than folded/quoted.
|
|
247
|
+
style = "|" if "\n" in value else None
|
|
248
|
+
return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
_TaskDumper.add_representer(str, _represent_str)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def dump_tasks(tasks: list[Task]) -> str:
|
|
255
|
+
"""Render tasks as ``tasks.yaml`` text, preserving key order and multi-line prompts.
|
|
256
|
+
|
|
257
|
+
The inverse of :func:`acumen.tasks.load_tasks`. Round-tripping through this is how the
|
|
258
|
+
combined (existing + generated) task set is validated before anything is written to disk.
|
|
259
|
+
Multi-line prompts are emitted as ``|`` literal blocks so a reviewer can read them.
|
|
260
|
+
"""
|
|
261
|
+
doc = {"tasks": [_task_to_dict(task) for task in tasks]}
|
|
262
|
+
return yaml.dump(doc, Dumper=_TaskDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=100)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _validate_generated(staged: Path) -> list[Task]:
|
|
266
|
+
"""Load and validate the agent's ``tasks.yaml``, mapping failures to a TaskGenError."""
|
|
267
|
+
if not staged.is_file():
|
|
268
|
+
raise TaskGenError(
|
|
269
|
+
f"the task-generation agent did not write {TASKS_FILE} — nothing to save. "
|
|
270
|
+
"Inspect the prompt or raise max_turns."
|
|
271
|
+
)
|
|
272
|
+
try:
|
|
273
|
+
return load_tasks(staged)
|
|
274
|
+
except TaskError as err:
|
|
275
|
+
raise TaskGenError(f"the generated {TASKS_FILE} is not valid: {err}") from err
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
async def generate_tasks(
|
|
279
|
+
*,
|
|
280
|
+
cfg: Config,
|
|
281
|
+
target: Target,
|
|
282
|
+
out_path: Path,
|
|
283
|
+
auth_mode: AuthMode = "session",
|
|
284
|
+
model: str | None = None,
|
|
285
|
+
max_turns: int | None = None,
|
|
286
|
+
max_usd: float | None = None,
|
|
287
|
+
force: bool = False,
|
|
288
|
+
feedback: str | None = None,
|
|
289
|
+
log: LiveLog | None = None,
|
|
290
|
+
) -> TaskGenResult:
|
|
291
|
+
"""Generate a ``tasks.yaml`` for the target package by mining and executing its analyses.
|
|
292
|
+
|
|
293
|
+
Parameters
|
|
294
|
+
----------
|
|
295
|
+
cfg
|
|
296
|
+
The pass config; supplies ``tasks_model``.
|
|
297
|
+
target
|
|
298
|
+
The prepared target, supplying the source checkout and the interpreter to run
|
|
299
|
+
pipelines against for ground truth.
|
|
300
|
+
auth_mode
|
|
301
|
+
Which credential the generation agent authenticates with — ``"session"`` (the Claude
|
|
302
|
+
subscription) or ``"api"`` (see :func:`acumen.env.build_agent_env`).
|
|
303
|
+
out_path
|
|
304
|
+
Where the tasks are written. Refuses to overwrite an existing file unless ``force``
|
|
305
|
+
is set. There is deliberately no "append" mode: the agent generates blind to the
|
|
306
|
+
existing file (it never reads it — see the module docstring on isolation), so it cannot
|
|
307
|
+
avoid re-covering functionality already present, and appending would silently grow the
|
|
308
|
+
set with semantic duplicates. To combine generated tasks with a curated file, write to a
|
|
309
|
+
separate ``out_path`` and merge by hand, where the overlap can actually be judged.
|
|
310
|
+
model
|
|
311
|
+
Override for the generation model; defaults to ``cfg.tasks_model``.
|
|
312
|
+
max_turns, max_usd
|
|
313
|
+
Caps for the generation agent. **Unbounded by default**: generating tasks means
|
|
314
|
+
running package code iteratively, so no default budget is imposed — pass explicit caps
|
|
315
|
+
to bound it.
|
|
316
|
+
force
|
|
317
|
+
Overwrite an existing ``out_path`` (e.g. the placeholder from ``acumen init``).
|
|
318
|
+
feedback
|
|
319
|
+
Optional maintainer guidance, injected into the generation prompt as a subordinated
|
|
320
|
+
block — e.g. which functionality to skip. Nothing but ``tasks.yaml`` is persisted (task
|
|
321
|
+
generation writes no meta), so the feedback is not recorded on disk.
|
|
322
|
+
log
|
|
323
|
+
A :class:`LiveLog` to stream the agent's messages to and render an HTML log from.
|
|
324
|
+
|
|
325
|
+
Returns
|
|
326
|
+
-------
|
|
327
|
+
The generation result: the task set written and what it cost.
|
|
328
|
+
"""
|
|
329
|
+
if out_path.exists() and not force:
|
|
330
|
+
raise TaskGenError(f"{out_path} already exists — pass force=True to overwrite it")
|
|
331
|
+
|
|
332
|
+
holder = Path(tempfile.mkdtemp(prefix="acumen-tasks-"))
|
|
333
|
+
try:
|
|
334
|
+
work = holder / "work"
|
|
335
|
+
home = holder / "home"
|
|
336
|
+
config_dir = home / ".claude"
|
|
337
|
+
for path in (work, home, config_dir, home / "tmp"):
|
|
338
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
339
|
+
staged = work / TASKS_FILE
|
|
340
|
+
|
|
341
|
+
# The agent reads a copy of the source with skills/agent-guidance stripped, never the
|
|
342
|
+
# real checkout — so existing skills cannot bias the tasks it generates.
|
|
343
|
+
source_copy = build_filtered_source(target.src_dir, holder / "source")
|
|
344
|
+
|
|
345
|
+
# Marks the agent's processes so the teardown below can find what it leaves running.
|
|
346
|
+
env = label_env(
|
|
347
|
+
build_agent_env(
|
|
348
|
+
config_dir=config_dir,
|
|
349
|
+
home=home,
|
|
350
|
+
extra_path=[target.bin_dir],
|
|
351
|
+
auth_mode=auth_mode,
|
|
352
|
+
extra_allow=cfg.env_passthrough,
|
|
353
|
+
),
|
|
354
|
+
holder,
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
prompt = taskgen_prompt(
|
|
358
|
+
package=target.pkg_name,
|
|
359
|
+
src=source_copy,
|
|
360
|
+
python=target.python,
|
|
361
|
+
out=staged,
|
|
362
|
+
feedback=feedback,
|
|
363
|
+
)
|
|
364
|
+
options = ClaudeAgentOptions(
|
|
365
|
+
cwd=str(work),
|
|
366
|
+
env=env,
|
|
367
|
+
model=model or cfg.tasks_model,
|
|
368
|
+
# No default budget cap: only bound the agent if the caller asked.
|
|
369
|
+
max_turns=max_turns,
|
|
370
|
+
max_budget_usd=max_usd,
|
|
371
|
+
# The generator reads the target source, like the drafter; benchmark agents
|
|
372
|
+
# never do. It points at the *filtered* copy, not the real checkout.
|
|
373
|
+
add_dirs=[str(source_copy)],
|
|
374
|
+
# No skill discovery at all — the generator must not load a skill that would bias it.
|
|
375
|
+
setting_sources=[],
|
|
376
|
+
permission_mode="bypassPermissions",
|
|
377
|
+
system_prompt={"type": "preset", "preset": "claude_code"},
|
|
378
|
+
# Belt-and-braces over the filtered copy: deny any call that reaches an existing
|
|
379
|
+
# skill/guidance artifact or the original unfiltered source, wherever pointed.
|
|
380
|
+
hooks={"PreToolUse": [make_skill_guard(target.src_dir)]},
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
result: ResultMessage | None = None
|
|
384
|
+
agent_error: Exception | None = None
|
|
385
|
+
try:
|
|
386
|
+
async for message in query(prompt=prompt, options=options):
|
|
387
|
+
if log is not None:
|
|
388
|
+
log.append(message)
|
|
389
|
+
if isinstance(message, ResultMessage):
|
|
390
|
+
result = message
|
|
391
|
+
except Exception as err: # noqa: BLE001 - a failed generation is an error to report, re-raised below
|
|
392
|
+
agent_error = err
|
|
393
|
+
finally:
|
|
394
|
+
# Render the HTML log while the throwaway config dir still holds the native
|
|
395
|
+
# transcript — in a finally so an aborted run (the SDK raises on a cap breach,
|
|
396
|
+
# after yielding the result) is still inspectable.
|
|
397
|
+
if log is not None:
|
|
398
|
+
log.finalize(config_dir=config_dir, work_dir=work, result=result)
|
|
399
|
+
|
|
400
|
+
if agent_error is not None:
|
|
401
|
+
raise TaskGenError(
|
|
402
|
+
f"the task-generation agent failed: {type(agent_error).__name__}: {agent_error}"
|
|
403
|
+
) from agent_error
|
|
404
|
+
if result is None:
|
|
405
|
+
raise TaskGenError("the task-generation agent produced no result message")
|
|
406
|
+
if result.is_error:
|
|
407
|
+
raise TaskGenError(f"the task-generation agent errored: {result.subtype} {result.errors or ''}".strip())
|
|
408
|
+
|
|
409
|
+
generated = _validate_generated(staged)
|
|
410
|
+
|
|
411
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
412
|
+
out_path.write_text(dump_tasks(generated))
|
|
413
|
+
return TaskGenResult(
|
|
414
|
+
tasks=generated,
|
|
415
|
+
out_path=out_path,
|
|
416
|
+
cost_usd=result.total_cost_usd or 0.0,
|
|
417
|
+
turns=result.num_turns,
|
|
418
|
+
log_jsonl=log.jsonl_path if log is not None else None,
|
|
419
|
+
log_html=log.html_path if log is not None and log.html_rendered else None,
|
|
420
|
+
)
|
|
421
|
+
finally:
|
|
422
|
+
# Kill anything the agent left running before removing the directory it runs in.
|
|
423
|
+
reap(holder)
|
|
424
|
+
shutil.rmtree(holder, ignore_errors=True)
|
acumen/tasks.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Task schema and loader for ``tasks.yaml``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
from acumen.paths import Split, is_safe_component
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TaskError(ValueError):
|
|
15
|
+
"""Raised when ``tasks.yaml`` is malformed."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class TaskSplit:
|
|
20
|
+
"""One half of a task: a prompt and the answer it must produce."""
|
|
21
|
+
|
|
22
|
+
prompt: str
|
|
23
|
+
answer: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Task:
|
|
28
|
+
"""A single benchmark task, with a train/test pair and optional overrides."""
|
|
29
|
+
|
|
30
|
+
id: str
|
|
31
|
+
train: TaskSplit
|
|
32
|
+
test: TaskSplit
|
|
33
|
+
max_turns: int | None = None
|
|
34
|
+
max_usd: float | None = None
|
|
35
|
+
model: str | None = None
|
|
36
|
+
|
|
37
|
+
def split(self, split: Split) -> TaskSplit:
|
|
38
|
+
"""Return the ``train`` or ``test`` half of this task."""
|
|
39
|
+
if split not in ("train", "test"):
|
|
40
|
+
raise TaskError(f"no such split: {split!r}")
|
|
41
|
+
return self.train if split == "train" else self.test
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _require_str(value: Any, where: str) -> str:
|
|
45
|
+
if not isinstance(value, str) or not value.strip():
|
|
46
|
+
raise TaskError(f"{where} must be a non-empty string, got {value!r}")
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _parse_split(raw: Any, where: str) -> TaskSplit:
|
|
51
|
+
if not isinstance(raw, dict):
|
|
52
|
+
raise TaskError(f"{where} must be a mapping with 'prompt' and 'answer', got {type(raw).__name__}")
|
|
53
|
+
unknown = set(raw) - {"prompt", "answer"}
|
|
54
|
+
if unknown:
|
|
55
|
+
raise TaskError(f"{where} has unknown keys: {sorted(unknown)}")
|
|
56
|
+
for key in ("prompt", "answer"):
|
|
57
|
+
if key not in raw:
|
|
58
|
+
raise TaskError(f"{where} is missing '{key}'")
|
|
59
|
+
return TaskSplit(
|
|
60
|
+
prompt=_require_str(raw["prompt"], f"{where}.prompt"), answer=_require_str(raw["answer"], f"{where}.answer")
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _parse_task(raw: Any, index: int) -> Task:
|
|
65
|
+
where = f"tasks[{index}]"
|
|
66
|
+
if not isinstance(raw, dict):
|
|
67
|
+
raise TaskError(f"{where} must be a mapping, got {type(raw).__name__}")
|
|
68
|
+
unknown = set(raw) - {"id", "train", "test", "max_turns", "max_usd", "model"}
|
|
69
|
+
if unknown:
|
|
70
|
+
raise TaskError(f"{where} has unknown keys: {sorted(unknown)}")
|
|
71
|
+
if "id" not in raw:
|
|
72
|
+
raise TaskError(f"{where} is missing 'id'")
|
|
73
|
+
task_id = _require_str(raw["id"], f"{where}.id")
|
|
74
|
+
if not is_safe_component(task_id):
|
|
75
|
+
raise TaskError(f"{where}.id {task_id!r} is not filesystem-safe — use only letters, digits, '.', '_', '-'")
|
|
76
|
+
where = f"task {task_id!r}"
|
|
77
|
+
for key in ("train", "test"):
|
|
78
|
+
if key not in raw:
|
|
79
|
+
raise TaskError(f"{where} is missing the '{key}' split — both splits are required")
|
|
80
|
+
max_turns = raw.get("max_turns")
|
|
81
|
+
if max_turns is not None and (not isinstance(max_turns, int) or isinstance(max_turns, bool) or max_turns < 1):
|
|
82
|
+
raise TaskError(f"{where}.max_turns must be a positive integer, got {max_turns!r}")
|
|
83
|
+
max_usd = raw.get("max_usd")
|
|
84
|
+
if max_usd is not None:
|
|
85
|
+
if isinstance(max_usd, bool) or not isinstance(max_usd, int | float) or max_usd <= 0:
|
|
86
|
+
raise TaskError(f"{where}.max_usd must be a positive number, got {max_usd!r}")
|
|
87
|
+
max_usd = float(max_usd)
|
|
88
|
+
model = raw.get("model")
|
|
89
|
+
if model is not None:
|
|
90
|
+
model = _require_str(model, f"{where}.model")
|
|
91
|
+
return Task(
|
|
92
|
+
id=task_id,
|
|
93
|
+
train=_parse_split(raw["train"], f"{where}.train"),
|
|
94
|
+
test=_parse_split(raw["test"], f"{where}.test"),
|
|
95
|
+
max_turns=max_turns,
|
|
96
|
+
max_usd=max_usd,
|
|
97
|
+
model=model,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def parse_tasks(raw: Any) -> list[Task]:
|
|
102
|
+
"""Validate an already-parsed ``tasks.yaml`` document.
|
|
103
|
+
|
|
104
|
+
Parameters
|
|
105
|
+
----------
|
|
106
|
+
raw
|
|
107
|
+
The mapping loaded from YAML.
|
|
108
|
+
|
|
109
|
+
Returns
|
|
110
|
+
-------
|
|
111
|
+
The tasks, in file order.
|
|
112
|
+
"""
|
|
113
|
+
if not isinstance(raw, dict):
|
|
114
|
+
raise TaskError(f"tasks.yaml must be a mapping with a 'tasks' key, got {type(raw).__name__}")
|
|
115
|
+
if "tasks" not in raw:
|
|
116
|
+
raise TaskError("tasks.yaml is missing the top-level 'tasks' key")
|
|
117
|
+
entries = raw["tasks"]
|
|
118
|
+
if not isinstance(entries, list) or not entries:
|
|
119
|
+
raise TaskError("'tasks' must be a non-empty list")
|
|
120
|
+
tasks = [_parse_task(entry, i) for i, entry in enumerate(entries)]
|
|
121
|
+
seen: dict[str, int] = {}
|
|
122
|
+
for i, task in enumerate(tasks):
|
|
123
|
+
if task.id in seen:
|
|
124
|
+
raise TaskError(
|
|
125
|
+
f"duplicate task id {task.id!r} (tasks[{seen[task.id]}] and tasks[{i}]) — ids must be unique"
|
|
126
|
+
)
|
|
127
|
+
seen[task.id] = i
|
|
128
|
+
return tasks
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def load_tasks(path: Path) -> list[Task]:
|
|
132
|
+
"""Load and validate ``tasks.yaml`` from disk."""
|
|
133
|
+
try:
|
|
134
|
+
text = path.read_text()
|
|
135
|
+
except OSError as err:
|
|
136
|
+
raise TaskError(f"cannot read tasks file {path}: {err}") from err
|
|
137
|
+
try:
|
|
138
|
+
raw = yaml.safe_load(text)
|
|
139
|
+
except yaml.YAMLError as err:
|
|
140
|
+
raise TaskError(f"{path} is not valid YAML: {err}") from err
|
|
141
|
+
try:
|
|
142
|
+
return parse_tasks(raw)
|
|
143
|
+
except TaskError as err:
|
|
144
|
+
raise TaskError(f"{path}: {err}") from err
|
acumen/transcript.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Locating and rendering SDK-native transcripts — shared by the runner and the meta-agents.
|
|
2
|
+
|
|
3
|
+
The benchmark runner has always done this per run: after an agent finishes, find the
|
|
4
|
+
transcript the ``claude`` CLI wrote (under ``<CLAUDE_CONFIG_DIR>/projects/<key>/<session>.jsonl``),
|
|
5
|
+
copy it into the run dir, and render it to HTML with ``claude-code-log``. This module factors
|
|
6
|
+
that machinery out here so the four single-agent commands (``draft``/``improve``/``tasks``/``ship``)
|
|
7
|
+
can render an HTML log of their own agent the same way, instead of losing the transcript to the
|
|
8
|
+
``rmtree`` of a throwaway config dir.
|
|
9
|
+
|
|
10
|
+
The transcript path is fully determined by the throwaway ``CLAUDE_CONFIG_DIR`` and the agent's
|
|
11
|
+
``cwd`` (both of which acumen sets) plus the ``session_id`` on the ``ResultMessage``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from claude_agent_sdk import project_key_for_directory
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def locate_transcript(config_dir: Path, work_dir: Path, session_id: str) -> Path | None:
|
|
25
|
+
"""Find the SDK-native transcript for a finished agent run.
|
|
26
|
+
|
|
27
|
+
The ``claude`` CLI writes transcripts to
|
|
28
|
+
``<config_dir>/projects/<project_key_for(work_dir)>/<session_id>.jsonl``. That encoding is
|
|
29
|
+
deterministic, but fall back to a glob rather than lose the transcript if the CLI ever keys
|
|
30
|
+
the directory differently.
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
config_dir
|
|
35
|
+
The ``CLAUDE_CONFIG_DIR`` the agent ran under (throwaway, for isolated agents; the
|
|
36
|
+
user's real one for the unisolated ship agent).
|
|
37
|
+
work_dir
|
|
38
|
+
The agent's ``cwd`` — what the project key is computed from.
|
|
39
|
+
session_id
|
|
40
|
+
``ResultMessage.session_id`` for the run.
|
|
41
|
+
|
|
42
|
+
Returns
|
|
43
|
+
-------
|
|
44
|
+
The transcript path, or ``None`` if it could not be found.
|
|
45
|
+
"""
|
|
46
|
+
transcript_root = config_dir / "projects"
|
|
47
|
+
key = project_key_for_directory(str(work_dir))
|
|
48
|
+
direct = transcript_root / key / f"{session_id}.jsonl"
|
|
49
|
+
if direct.is_file():
|
|
50
|
+
return direct
|
|
51
|
+
matches = sorted(transcript_root.glob(f"**/{session_id}.jsonl"))
|
|
52
|
+
return matches[0] if matches else None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def claude_code_log() -> str | None:
|
|
56
|
+
"""Locate the ``claude-code-log`` CLI.
|
|
57
|
+
|
|
58
|
+
It is a dependency of acumen, so it lives next to the interpreter running us — look there
|
|
59
|
+
before PATH, which won't contain the venv's ``bin`` when acumen is invoked by absolute path
|
|
60
|
+
rather than through an activated venv.
|
|
61
|
+
"""
|
|
62
|
+
local = Path(sys.executable).parent / "claude-code-log"
|
|
63
|
+
if local.is_file():
|
|
64
|
+
return str(local)
|
|
65
|
+
return shutil.which("claude-code-log")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def render_transcript(jsonl: Path, html: Path) -> bool:
|
|
69
|
+
"""Render an SDK-native transcript to a standalone HTML file via ``claude-code-log``.
|
|
70
|
+
|
|
71
|
+
Returns whether the render succeeded and produced the file.
|
|
72
|
+
"""
|
|
73
|
+
cli = claude_code_log()
|
|
74
|
+
if cli is None:
|
|
75
|
+
return False
|
|
76
|
+
proc = subprocess.run([cli, str(jsonl), "-o", str(html)], capture_output=True, text=True)
|
|
77
|
+
return proc.returncode == 0 and html.is_file()
|