acumen 0.0.1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acumen/__init__.py +160 -0
- acumen/_skills/__init__.py +0 -0
- acumen/_skills/data/SKILL.md +109 -0
- acumen/_skills/data/references/authoring.md +91 -0
- acumen/_skills/data/references/benchmark.md +100 -0
- acumen/_skills/data/references/python-api.md +106 -0
- acumen/_skills/data/references/setup.md +94 -0
- acumen/_skills/data/references/ship.md +53 -0
- acumen/_skills/install.py +185 -0
- acumen/assets/banner.svg +501 -0
- acumen/assets/logo.svg +47 -0
- acumen/bench.py +202 -0
- acumen/cli.py +626 -0
- acumen/config.py +204 -0
- acumen/draft.py +202 -0
- acumen/env.py +602 -0
- acumen/grade.py +104 -0
- acumen/improve.py +596 -0
- acumen/logs.py +255 -0
- acumen/paths.py +164 -0
- acumen/procs.py +195 -0
- acumen/prompts.py +942 -0
- acumen/report.py +1858 -0
- acumen/runner.py +343 -0
- acumen/sandbox.py +153 -0
- acumen/scaffold.py +118 -0
- acumen/ship.py +298 -0
- acumen/skills.py +298 -0
- acumen/taskgen.py +424 -0
- acumen/tasks.py +144 -0
- acumen/transcript.py +77 -0
- acumen-0.0.1.dev0.dist-info/METADATA +168 -0
- acumen-0.0.1.dev0.dist-info/RECORD +36 -0
- acumen-0.0.1.dev0.dist-info/WHEEL +4 -0
- acumen-0.0.1.dev0.dist-info/entry_points.txt +3 -0
- acumen-0.0.1.dev0.dist-info/licenses/LICENSE +21 -0
acumen/env.py
ADDED
|
@@ -0,0 +1,602 @@
|
|
|
1
|
+
"""Target intake and process environment.
|
|
2
|
+
|
|
3
|
+
Two jobs:
|
|
4
|
+
|
|
5
|
+
* :func:`prepare_target` — clone (or adopt a local path), build one venv with the target
|
|
6
|
+
package installed, and record the resolved commit and package version. Cached by
|
|
7
|
+
(repo, ref) so a pass doesn't re-clone.
|
|
8
|
+
* :func:`scrubbed_env` — the filtered environment benchmark agents run under: auth and
|
|
9
|
+
PATH only, a throwaway ``HOME`` and ``CLAUDE_CONFIG_DIR``, and nothing that could leak
|
|
10
|
+
the user's own settings or memories into a run.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import tomllib
|
|
22
|
+
from collections.abc import Sequence
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Literal
|
|
26
|
+
|
|
27
|
+
from acumen.config import Config
|
|
28
|
+
from acumen.paths import slugify
|
|
29
|
+
|
|
30
|
+
READY_MARKER = ".acumen-ready"
|
|
31
|
+
|
|
32
|
+
#: Environment variables carried into agent runs. Everything else is dropped.
|
|
33
|
+
#: Auth and provider routing, because a run cannot authenticate without them; proxy and
|
|
34
|
+
#: TLS settings, because web access is part of the benchmark.
|
|
35
|
+
ENV_ALLOWLIST = (
|
|
36
|
+
"ANTHROPIC_API_KEY",
|
|
37
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
38
|
+
"ANTHROPIC_BASE_URL",
|
|
39
|
+
"ANTHROPIC_CUSTOM_HEADERS",
|
|
40
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
41
|
+
"CLAUDE_CODE_USE_BEDROCK",
|
|
42
|
+
"CLAUDE_CODE_USE_VERTEX",
|
|
43
|
+
"AWS_ACCESS_KEY_ID",
|
|
44
|
+
"AWS_SECRET_ACCESS_KEY",
|
|
45
|
+
"AWS_SESSION_TOKEN",
|
|
46
|
+
"AWS_REGION",
|
|
47
|
+
"AWS_DEFAULT_REGION",
|
|
48
|
+
"AWS_PROFILE",
|
|
49
|
+
"ANTHROPIC_VERTEX_PROJECT_ID",
|
|
50
|
+
"CLOUD_ML_REGION",
|
|
51
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
52
|
+
"HTTP_PROXY",
|
|
53
|
+
"HTTPS_PROXY",
|
|
54
|
+
"NO_PROXY",
|
|
55
|
+
"http_proxy",
|
|
56
|
+
"https_proxy",
|
|
57
|
+
"no_proxy",
|
|
58
|
+
"SSL_CERT_FILE",
|
|
59
|
+
"SSL_CERT_DIR",
|
|
60
|
+
"NODE_EXTRA_CA_CERTS",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
_BASE_PATH = ("/usr/local/bin", "/usr/bin", "/bin")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class EnvError(RuntimeError):
|
|
67
|
+
"""Raised when the target cannot be prepared."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class Target:
|
|
72
|
+
"""A prepared benchmark target: source checkout plus the venv it's installed into."""
|
|
73
|
+
|
|
74
|
+
source: str
|
|
75
|
+
ref: str
|
|
76
|
+
src_dir: Path
|
|
77
|
+
venv_dir: Path
|
|
78
|
+
commit: str
|
|
79
|
+
pkg_name: str
|
|
80
|
+
pkg_version: str
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def bin_dir(self) -> Path:
|
|
84
|
+
"""The venv's ``bin`` directory — what goes on an agent's PATH."""
|
|
85
|
+
return self.venv_dir / ("Scripts" if os.name == "nt" else "bin")
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def python(self) -> Path:
|
|
89
|
+
"""The venv interpreter, with the target package importable."""
|
|
90
|
+
return self.bin_dir / ("python.exe" if os.name == "nt" else "python")
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def fingerprint(self) -> str:
|
|
94
|
+
"""The ``pkg_version`` string recorded in ``result.json``, e.g. ``numpy 2.1.0``."""
|
|
95
|
+
return f"{self.pkg_name} {self.pkg_version}"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _run(cmd: list[str], *, cwd: Path | None = None) -> str:
|
|
99
|
+
proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
|
|
100
|
+
if proc.returncode != 0:
|
|
101
|
+
raise EnvError(f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr.strip()}")
|
|
102
|
+
return proc.stdout.strip()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def cache_key(repo: str, ref: str) -> str:
|
|
106
|
+
"""Return the cache directory name for a (repo, ref) pair."""
|
|
107
|
+
digest = hashlib.sha256(f"{repo}\n{ref}".encode()).hexdigest()[:12]
|
|
108
|
+
stem = slugify(Path(repo.rstrip("/")).name.removesuffix(".git") or "target")
|
|
109
|
+
return f"{stem}-{digest}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _clone(repo: str, ref: str, dest: Path) -> None:
|
|
113
|
+
if dest.exists():
|
|
114
|
+
shutil.rmtree(dest)
|
|
115
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
116
|
+
_run(["git", "clone", "--filter=blob:none", "--quiet", repo, str(dest)])
|
|
117
|
+
_run(["git", "-c", "advice.detachedHead=false", "checkout", "--quiet", ref], cwd=dest)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _resolve_commit(src_dir: Path) -> str:
|
|
121
|
+
try:
|
|
122
|
+
return _run(["git", "rev-parse", "HEAD"], cwd=src_dir)
|
|
123
|
+
except EnvError:
|
|
124
|
+
return "local" # a local path that isn't a git checkout is still a valid target
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _package_name(src_dir: Path) -> str:
|
|
128
|
+
pyproject = src_dir / "pyproject.toml"
|
|
129
|
+
if not pyproject.is_file():
|
|
130
|
+
raise EnvError(f"{src_dir} has no pyproject.toml — acumen needs an installable package")
|
|
131
|
+
try:
|
|
132
|
+
data = tomllib.loads(pyproject.read_text())
|
|
133
|
+
except (OSError, tomllib.TOMLDecodeError) as err:
|
|
134
|
+
raise EnvError(f"cannot parse {pyproject}: {err}") from err
|
|
135
|
+
name = data.get("project", {}).get("name")
|
|
136
|
+
if not name:
|
|
137
|
+
raise EnvError(f"{pyproject} does not declare [project].name")
|
|
138
|
+
return str(name)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _installed_version(python: Path, pkg_name: str) -> str:
|
|
142
|
+
code = f"import importlib.metadata as m; print(m.version({pkg_name!r}))"
|
|
143
|
+
try:
|
|
144
|
+
return _run([str(python), "-c", code])
|
|
145
|
+
except EnvError as err:
|
|
146
|
+
raise EnvError(f"{pkg_name} is not importable in the target venv after install: {err}") from err
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def prepare_target(cfg: Config, cache_root: Path, *, refresh: bool = False) -> Target:
|
|
150
|
+
"""Clone or adopt the target package and install it into a cached venv.
|
|
151
|
+
|
|
152
|
+
Parameters
|
|
153
|
+
----------
|
|
154
|
+
cfg
|
|
155
|
+
The pass config; supplies ``repo``, ``ref``, ``extras`` and ``python``.
|
|
156
|
+
cache_root
|
|
157
|
+
Directory to hold checkouts and venvs, keyed by (repo, ref).
|
|
158
|
+
refresh
|
|
159
|
+
Rebuild even if a ready-marked cache entry exists.
|
|
160
|
+
|
|
161
|
+
Returns
|
|
162
|
+
-------
|
|
163
|
+
The prepared target, with the resolved commit and installed package version.
|
|
164
|
+
"""
|
|
165
|
+
if shutil.which("uv") is None:
|
|
166
|
+
raise EnvError("uv is not on PATH — acumen uses it to build the target venv")
|
|
167
|
+
entry = cache_root / cache_key(cfg.repo, cfg.ref)
|
|
168
|
+
venv_dir = entry / "venv"
|
|
169
|
+
marker = entry / READY_MARKER
|
|
170
|
+
|
|
171
|
+
if cfg.is_local:
|
|
172
|
+
src_dir = Path(cfg.repo).expanduser().resolve()
|
|
173
|
+
if not src_dir.is_dir():
|
|
174
|
+
raise EnvError(f"local repo path does not exist: {src_dir}")
|
|
175
|
+
else:
|
|
176
|
+
src_dir = entry / "src"
|
|
177
|
+
|
|
178
|
+
if not refresh and marker.is_file():
|
|
179
|
+
try:
|
|
180
|
+
cached = json.loads(marker.read_text())
|
|
181
|
+
target = Target(
|
|
182
|
+
source=cfg.repo,
|
|
183
|
+
ref=cfg.ref,
|
|
184
|
+
src_dir=Path(cached["src_dir"]),
|
|
185
|
+
venv_dir=venv_dir,
|
|
186
|
+
commit=cached["commit"],
|
|
187
|
+
pkg_name=cached["pkg_name"],
|
|
188
|
+
pkg_version=cached["pkg_version"],
|
|
189
|
+
)
|
|
190
|
+
except (OSError, KeyError, json.JSONDecodeError):
|
|
191
|
+
target = None # a corrupt marker just means we rebuild
|
|
192
|
+
else:
|
|
193
|
+
# A local target's working tree can move under us; a clone at a pinned ref cannot.
|
|
194
|
+
if target.python.is_file() and (not cfg.is_local or _resolve_commit(src_dir) == target.commit):
|
|
195
|
+
return target
|
|
196
|
+
|
|
197
|
+
if not cfg.is_local:
|
|
198
|
+
_clone(cfg.repo, cfg.ref, src_dir)
|
|
199
|
+
|
|
200
|
+
entry.mkdir(parents=True, exist_ok=True)
|
|
201
|
+
if venv_dir.exists():
|
|
202
|
+
shutil.rmtree(venv_dir)
|
|
203
|
+
_run(["uv", "venv", "--python", cfg.python, str(venv_dir)])
|
|
204
|
+
|
|
205
|
+
pkg_name = _package_name(src_dir)
|
|
206
|
+
spec = str(src_dir)
|
|
207
|
+
if cfg.extras:
|
|
208
|
+
spec = f"{src_dir}[{','.join(cfg.extras)}]"
|
|
209
|
+
python = venv_dir / ("Scripts" if os.name == "nt" else "bin") / ("python.exe" if os.name == "nt" else "python")
|
|
210
|
+
_run(["uv", "pip", "install", "--python", str(python), spec])
|
|
211
|
+
|
|
212
|
+
target = Target(
|
|
213
|
+
source=cfg.repo,
|
|
214
|
+
ref=cfg.ref,
|
|
215
|
+
src_dir=src_dir,
|
|
216
|
+
venv_dir=venv_dir,
|
|
217
|
+
commit=_resolve_commit(src_dir),
|
|
218
|
+
pkg_name=pkg_name,
|
|
219
|
+
pkg_version=_installed_version(python, pkg_name),
|
|
220
|
+
)
|
|
221
|
+
marker.write_text(
|
|
222
|
+
json.dumps(
|
|
223
|
+
{
|
|
224
|
+
"src_dir": str(target.src_dir),
|
|
225
|
+
"commit": target.commit,
|
|
226
|
+
"pkg_name": target.pkg_name,
|
|
227
|
+
"pkg_version": target.pkg_version,
|
|
228
|
+
},
|
|
229
|
+
indent=2,
|
|
230
|
+
)
|
|
231
|
+
)
|
|
232
|
+
return target
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def claude_cli_dir() -> Path | None:
|
|
236
|
+
"""Return the directory holding the ``claude`` CLI, which the SDK shells out to."""
|
|
237
|
+
found = shutil.which("claude")
|
|
238
|
+
return Path(found).parent if found else None
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
#: Allowlisted variables that, present and non-empty, let a run authenticate on their own:
|
|
242
|
+
#: a direct Anthropic key/token, or a flag routing to a cloud provider whose own (AWS/GCP)
|
|
243
|
+
#: credentials then apply. A subset of :data:`ENV_ALLOWLIST` — the entries that carry or
|
|
244
|
+
#: enable authentication, as opposed to proxy/TLS routing.
|
|
245
|
+
AUTH_ENV_VARS = (
|
|
246
|
+
"ANTHROPIC_API_KEY",
|
|
247
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
248
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
249
|
+
"CLAUDE_CODE_USE_BEDROCK",
|
|
250
|
+
"CLAUDE_CODE_USE_VERTEX",
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
#: A resolved authentication mode for an agent run. ``"session"`` bills the user's Claude
|
|
254
|
+
#: subscription via an OAuth login; ``"api"`` bills the Anthropic API (or a cloud provider)
|
|
255
|
+
#: per token — the only mode that yields the real ``cost_usd`` the benchmark records.
|
|
256
|
+
AuthMode = Literal["session", "api"]
|
|
257
|
+
|
|
258
|
+
#: Allowlisted variables that carry *metered* (API/cloud) authentication, as distinct from a
|
|
259
|
+
#: subscription OAuth login. A subset of :data:`AUTH_ENV_VARS` that deliberately excludes
|
|
260
|
+
#: ``CLAUDE_CODE_OAUTH_TOKEN`` — that token bills the subscription, not the API. A run in
|
|
261
|
+
#: ``"api"`` mode keeps these and drops the OAuth token; a ``"session"`` run does the reverse.
|
|
262
|
+
API_AUTH_ENV_VARS = (
|
|
263
|
+
"ANTHROPIC_API_KEY",
|
|
264
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
265
|
+
"CLAUDE_CODE_USE_BEDROCK",
|
|
266
|
+
"CLAUDE_CODE_USE_VERTEX",
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
#: The subscription (OAuth) token variable — a portable ``claude setup-token`` credential.
|
|
270
|
+
#: Kept in ``"session"`` mode and dropped in ``"api"`` mode so exactly one auth path is live.
|
|
271
|
+
SESSION_AUTH_ENV_VAR = "CLAUDE_CODE_OAUTH_TOKEN"
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _credentials_path() -> Path:
|
|
275
|
+
"""The user's Claude OAuth credentials file — what :func:`seed_credentials` copies."""
|
|
276
|
+
base = os.environ.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude"
|
|
277
|
+
return Path(base) / ".credentials.json"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def auth_available() -> bool:
|
|
281
|
+
"""Whether an agent run could authenticate to the Claude API.
|
|
282
|
+
|
|
283
|
+
A run gets its credential one of two ways: an allowlisted auth variable in the
|
|
284
|
+
environment (a direct Anthropic key/token, or a Bedrock/Vertex routing flag whose own
|
|
285
|
+
cloud credentials then apply), or a Claude credentials file that
|
|
286
|
+
:func:`seed_credentials` copies into the throwaway config dir (an OAuth login). An
|
|
287
|
+
empty variable does not count.
|
|
288
|
+
"""
|
|
289
|
+
if any(os.environ.get(var) for var in AUTH_ENV_VARS):
|
|
290
|
+
return True
|
|
291
|
+
return _credentials_path().is_file()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def check_auth() -> None:
|
|
295
|
+
"""Raise :class:`EnvError` if no credential is reachable for an agent run.
|
|
296
|
+
|
|
297
|
+
A preflight guard for the agentic commands. Without it an unauthenticated ``bench``
|
|
298
|
+
would run the whole matrix, record every run as an ``error``, and still exit 0, while
|
|
299
|
+
the meta-agents (``draft``/``improve``/``tasks``/``ship``) would fail deep in the SDK
|
|
300
|
+
with a raw message — so fail early, before the costly target prep, with a clear fix.
|
|
301
|
+
"""
|
|
302
|
+
if auth_available():
|
|
303
|
+
return
|
|
304
|
+
raise EnvError(
|
|
305
|
+
"no Claude credentials found — an agent run cannot authenticate. Set "
|
|
306
|
+
"ANTHROPIC_API_KEY (or ANTHROPIC_AUTH_TOKEN / CLAUDE_CODE_OAUTH_TOKEN), enable a "
|
|
307
|
+
"provider with CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX, or log in with "
|
|
308
|
+
"`claude` so ~/.claude/.credentials.json exists."
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _has_oauth_credentials() -> bool:
|
|
313
|
+
"""Whether the user's credentials file holds a Claude subscription (OAuth) login.
|
|
314
|
+
|
|
315
|
+
``claude`` login writes ``.credentials.json`` as ``{"claudeAiOauth": {...}}`` — the
|
|
316
|
+
presence of that object is what distinguishes a subscription login from a bare API-key
|
|
317
|
+
setup, and it is the signal that "session usage" is available. A missing, unreadable, or
|
|
318
|
+
API-only credentials file is not a subscription.
|
|
319
|
+
"""
|
|
320
|
+
try:
|
|
321
|
+
data = json.loads(_credentials_path().read_text())
|
|
322
|
+
except (OSError, ValueError):
|
|
323
|
+
return False
|
|
324
|
+
return isinstance(data.get("claudeAiOauth"), dict)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def session_auth_available() -> bool:
|
|
328
|
+
"""Whether an agent run could bill the Claude subscription ("session usage").
|
|
329
|
+
|
|
330
|
+
True when a portable ``CLAUDE_CODE_OAUTH_TOKEN`` is set, or the user has a subscription
|
|
331
|
+
OAuth login on disk (:func:`_has_oauth_credentials`). An empty variable does not count.
|
|
332
|
+
"""
|
|
333
|
+
if os.environ.get(SESSION_AUTH_ENV_VAR):
|
|
334
|
+
return True
|
|
335
|
+
return _has_oauth_credentials()
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def api_auth_available() -> bool:
|
|
339
|
+
"""Whether an agent run could bill the Anthropic API (or a cloud provider) per token.
|
|
340
|
+
|
|
341
|
+
True when any metered auth variable is set (:data:`API_AUTH_ENV_VARS`): a direct
|
|
342
|
+
Anthropic key/token, or a Bedrock/Vertex routing flag whose own cloud credentials then
|
|
343
|
+
apply. The subscription OAuth token is deliberately excluded — it bills the plan, not
|
|
344
|
+
the API. An empty variable does not count.
|
|
345
|
+
"""
|
|
346
|
+
return any(os.environ.get(var) for var in API_AUTH_ENV_VARS)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def resolve_auth_mode(requested: str, *, allow_session: bool) -> AuthMode:
|
|
350
|
+
"""Resolve a requested auth choice to the concrete mode a run will use, or fail loudly.
|
|
351
|
+
|
|
352
|
+
A preflight guard that replaces :func:`check_auth` on the agentic commands: it both
|
|
353
|
+
validates that the chosen credential is actually reachable and reports which mode the
|
|
354
|
+
run will bill, so the choice is never silent.
|
|
355
|
+
|
|
356
|
+
Parameters
|
|
357
|
+
----------
|
|
358
|
+
requested
|
|
359
|
+
The user's choice: ``"auto"`` (prefer the subscription, else the API), ``"session"``
|
|
360
|
+
(force the subscription), or ``"api"`` (force the API).
|
|
361
|
+
allow_session
|
|
362
|
+
Whether the subscription is a permitted mode. ``bench`` passes ``False`` because it
|
|
363
|
+
records real per-run ``cost_usd``, which only means anything under metered API
|
|
364
|
+
billing — so bench always resolves to ``"api"`` and rejects ``"session"``.
|
|
365
|
+
|
|
366
|
+
Returns
|
|
367
|
+
-------
|
|
368
|
+
``"session"`` or ``"api"``.
|
|
369
|
+
"""
|
|
370
|
+
if not allow_session:
|
|
371
|
+
if requested == "session":
|
|
372
|
+
raise EnvError(
|
|
373
|
+
"bench records real per-run cost and must bill the API, so --auth session is "
|
|
374
|
+
"not available for it — the Claude subscription does not meter per-run spend. "
|
|
375
|
+
"Run the single-agent commands (draft/improve/tasks/ship) on the session instead."
|
|
376
|
+
)
|
|
377
|
+
if not api_auth_available():
|
|
378
|
+
raise EnvError(
|
|
379
|
+
"no API credential found — bench must bill the API. Set ANTHROPIC_API_KEY (or "
|
|
380
|
+
"ANTHROPIC_AUTH_TOKEN), or enable a provider with CLAUDE_CODE_USE_BEDROCK / "
|
|
381
|
+
"CLAUDE_CODE_USE_VERTEX."
|
|
382
|
+
)
|
|
383
|
+
return "api"
|
|
384
|
+
|
|
385
|
+
if requested == "session":
|
|
386
|
+
if not session_auth_available():
|
|
387
|
+
raise EnvError(
|
|
388
|
+
"no Claude subscription login found for --auth session. Log in with `claude` so "
|
|
389
|
+
"~/.claude/.credentials.json exists, or set CLAUDE_CODE_OAUTH_TOKEN (from "
|
|
390
|
+
"`claude setup-token`)."
|
|
391
|
+
)
|
|
392
|
+
return "session"
|
|
393
|
+
if requested == "api":
|
|
394
|
+
if not api_auth_available():
|
|
395
|
+
raise EnvError(
|
|
396
|
+
"no API credential found for --auth api. Set ANTHROPIC_API_KEY (or "
|
|
397
|
+
"ANTHROPIC_AUTH_TOKEN), or enable a provider with CLAUDE_CODE_USE_BEDROCK / "
|
|
398
|
+
"CLAUDE_CODE_USE_VERTEX."
|
|
399
|
+
)
|
|
400
|
+
return "api"
|
|
401
|
+
|
|
402
|
+
# "auto": prefer the subscription (it's what the user is paying a flat rate for), fall
|
|
403
|
+
# back to the metered API, and only then give up.
|
|
404
|
+
if session_auth_available():
|
|
405
|
+
return "session"
|
|
406
|
+
if api_auth_available():
|
|
407
|
+
return "api"
|
|
408
|
+
raise EnvError(
|
|
409
|
+
"no Claude credentials found — an agent run cannot authenticate. Log in with `claude` "
|
|
410
|
+
"so ~/.claude/.credentials.json exists (subscription), or set ANTHROPIC_API_KEY (or "
|
|
411
|
+
"ANTHROPIC_AUTH_TOKEN / CLAUDE_CODE_OAUTH_TOKEN), or enable a provider with "
|
|
412
|
+
"CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX."
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def seed_credentials(config_dir: Path) -> bool:
|
|
417
|
+
"""Copy the user's Claude credentials into a throwaway config dir.
|
|
418
|
+
|
|
419
|
+
A scrubbed ``HOME`` hides ``~/.claude/.credentials.json``, which is how
|
|
420
|
+
OAuth-authenticated users are logged in — without this an isolated run cannot
|
|
421
|
+
authenticate at all. Only the credentials file is copied: settings, memories and
|
|
422
|
+
project history stay behind, which is the isolation the benchmark actually needs.
|
|
423
|
+
|
|
424
|
+
Returns
|
|
425
|
+
-------
|
|
426
|
+
Whether a credentials file was found and copied.
|
|
427
|
+
"""
|
|
428
|
+
real = _credentials_path()
|
|
429
|
+
if not real.is_file():
|
|
430
|
+
return False
|
|
431
|
+
config_dir.mkdir(parents=True, exist_ok=True)
|
|
432
|
+
dest = config_dir / ".credentials.json"
|
|
433
|
+
shutil.copyfile(real, dest)
|
|
434
|
+
dest.chmod(0o600)
|
|
435
|
+
return True
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _apply_auth_mode(env: dict[str, str], auth_mode: AuthMode | None) -> None:
|
|
439
|
+
"""Neutralize the credential variables the chosen mode must not authenticate with.
|
|
440
|
+
|
|
441
|
+
So exactly one auth path is live: ``"session"`` neutralizes the metered API/cloud variables
|
|
442
|
+
and keeps the subscription OAuth token; ``"api"`` neutralizes the OAuth token and keeps the
|
|
443
|
+
metered ones. ``None`` leaves the allowlisted credentials untouched (the historical
|
|
444
|
+
behavior, for callers that don't select a mode).
|
|
445
|
+
|
|
446
|
+
The variables are set to ``""``, not deleted. The SDK builds the agent subprocess env as
|
|
447
|
+
``{**os.environ, **options.env}`` — it merges our mapping *over* the inherited environment —
|
|
448
|
+
so a credential we merely omit falls back through from ``os.environ`` unchanged and the run
|
|
449
|
+
silently authenticates with the wrong one. An explicit empty value overrides the inherited
|
|
450
|
+
one, which the CLI reads as unset.
|
|
451
|
+
"""
|
|
452
|
+
if auth_mode == "session":
|
|
453
|
+
drop = API_AUTH_ENV_VARS
|
|
454
|
+
elif auth_mode == "api":
|
|
455
|
+
drop = (SESSION_AUTH_ENV_VAR,)
|
|
456
|
+
else:
|
|
457
|
+
return
|
|
458
|
+
for var in drop:
|
|
459
|
+
env[var] = ""
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def scrubbed_env(
|
|
463
|
+
*,
|
|
464
|
+
config_dir: Path,
|
|
465
|
+
home: Path,
|
|
466
|
+
extra_path: list[Path] | None = None,
|
|
467
|
+
auth_mode: AuthMode | None = None,
|
|
468
|
+
extra_allow: Sequence[str] | None = None,
|
|
469
|
+
) -> dict[str, str]:
|
|
470
|
+
"""Build the environment an isolated agent runs under.
|
|
471
|
+
|
|
472
|
+
The result is a clean allowlist: only :data:`ENV_ALLOWLIST` (plus ``extra_allow``) and
|
|
473
|
+
a handful of throwaway overrides survive; **every other variable inherited from the
|
|
474
|
+
operator's shell is blanked**. ``HOME`` and ``CLAUDE_CONFIG_DIR`` point at throwaway
|
|
475
|
+
directories so no user settings or ``CLAUDE.md`` memories are discoverable.
|
|
476
|
+
|
|
477
|
+
The blanking is not cosmetic. The SDK builds the agent subprocess env as
|
|
478
|
+
``{**os.environ, **options.env}`` — it merges this mapping *over* the inherited
|
|
479
|
+
environment (see :func:`_apply_auth_mode`), so a variable we merely *omit* falls back
|
|
480
|
+
through from ``os.environ`` unchanged. To actually keep an ambient secret out of the
|
|
481
|
+
(web-enabled) agent we must return it set to ``""``, which overrides the inherited value
|
|
482
|
+
and the CLI reads as unset. So we enumerate everything in ``os.environ`` that isn't
|
|
483
|
+
allowlisted or overridden and empty it explicitly.
|
|
484
|
+
|
|
485
|
+
Parameters
|
|
486
|
+
----------
|
|
487
|
+
config_dir
|
|
488
|
+
Throwaway ``CLAUDE_CONFIG_DIR``; also where the transcript will land.
|
|
489
|
+
home
|
|
490
|
+
Throwaway ``HOME``.
|
|
491
|
+
extra_path
|
|
492
|
+
Directories to prepend to ``PATH`` — the target venv's ``bin`` goes here, so
|
|
493
|
+
``python`` in the sandbox is the interpreter with the package installed.
|
|
494
|
+
auth_mode
|
|
495
|
+
Which credential the run should authenticate with. ``"session"`` keeps only the
|
|
496
|
+
subscription OAuth token, ``"api"`` keeps only the metered API/cloud credentials, and
|
|
497
|
+
``None`` leaves both in place (the historical behavior). Note that ``"session"`` needs
|
|
498
|
+
:func:`seed_credentials` to have placed the OAuth login in ``config_dir`` — pair the
|
|
499
|
+
two via :func:`build_agent_env`.
|
|
500
|
+
extra_allow
|
|
501
|
+
Additional variable names to carry through from ``os.environ`` on top of
|
|
502
|
+
:data:`ENV_ALLOWLIST` — the operator's declared ``env_passthrough``, for a target
|
|
503
|
+
that needs a runtime variable the built-in allowlist doesn't cover.
|
|
504
|
+
|
|
505
|
+
Returns
|
|
506
|
+
-------
|
|
507
|
+
The environment mapping to hand to the SDK.
|
|
508
|
+
"""
|
|
509
|
+
allow = (*ENV_ALLOWLIST, *(extra_allow or ()))
|
|
510
|
+
env = {key: os.environ[key] for key in allow if key in os.environ}
|
|
511
|
+
_apply_auth_mode(env, auth_mode)
|
|
512
|
+
|
|
513
|
+
path_parts = [str(p) for p in (extra_path or [])]
|
|
514
|
+
cli_dir = claude_cli_dir()
|
|
515
|
+
if cli_dir is not None:
|
|
516
|
+
path_parts.append(str(cli_dir))
|
|
517
|
+
node_dir = shutil.which("node")
|
|
518
|
+
if node_dir is not None:
|
|
519
|
+
path_parts.append(str(Path(node_dir).parent))
|
|
520
|
+
path_parts.extend(_BASE_PATH)
|
|
521
|
+
seen: list[str] = []
|
|
522
|
+
for part in path_parts:
|
|
523
|
+
if part not in seen:
|
|
524
|
+
seen.append(part)
|
|
525
|
+
|
|
526
|
+
env["PATH"] = os.pathsep.join(seen)
|
|
527
|
+
env["HOME"] = str(home)
|
|
528
|
+
env["CLAUDE_CONFIG_DIR"] = str(config_dir)
|
|
529
|
+
# Skill discovery needs setting_sources=["project"], but project discovery
|
|
530
|
+
# also walks UP from cwd and auto-loads every CLAUDE.md it passes. Verified: an agent
|
|
531
|
+
# recited a canary planted in its sandbox's parent directory having made zero tool
|
|
532
|
+
# calls. Sandboxes live under a temp dir whose ancestors we do not control, so a stray
|
|
533
|
+
# CLAUDE.md anywhere above them would silently enter every run's context and break the
|
|
534
|
+
# memory isolation. This disables memory discovery outright; skills still load (verified).
|
|
535
|
+
env["CLAUDE_CODE_DISABLE_CLAUDE_MDS"] = "1"
|
|
536
|
+
env["TMPDIR"] = str(home / "tmp")
|
|
537
|
+
env["LANG"] = os.environ.get("LANG", "C.UTF-8")
|
|
538
|
+
# Keep pip/uv from reaching into the real user's caches and configs.
|
|
539
|
+
env["XDG_CONFIG_HOME"] = str(home / ".config")
|
|
540
|
+
env["XDG_CACHE_HOME"] = str(home / ".cache")
|
|
541
|
+
|
|
542
|
+
# Blank every inherited variable we did not deliberately keep or override. Omission is a
|
|
543
|
+
# no-op under the SDK's env merge, so the only way to stop the operator's ambient secrets
|
|
544
|
+
# (cloud creds, service tokens, anything exported in the shell) from reaching a web-enabled
|
|
545
|
+
# agent is to override each one with an empty value. Anything a target legitimately needs is
|
|
546
|
+
# declared via extra_allow (config env_passthrough) and was kept above.
|
|
547
|
+
for key in os.environ:
|
|
548
|
+
if key not in env:
|
|
549
|
+
env[key] = ""
|
|
550
|
+
return env
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def build_agent_env(
|
|
554
|
+
*,
|
|
555
|
+
config_dir: Path,
|
|
556
|
+
home: Path,
|
|
557
|
+
extra_path: list[Path] | None = None,
|
|
558
|
+
auth_mode: AuthMode,
|
|
559
|
+
extra_allow: Sequence[str] | None = None,
|
|
560
|
+
) -> dict[str, str]:
|
|
561
|
+
"""Prepare an isolated agent's environment for a resolved auth mode.
|
|
562
|
+
|
|
563
|
+
Pairs the two steps that must agree: in ``"session"`` mode the subscription OAuth login is
|
|
564
|
+
seeded into ``config_dir`` (only ``.credentials.json`` is copied — never settings, memories,
|
|
565
|
+
or skills, so isolation is unchanged), and the API/cloud variables are stripped from the
|
|
566
|
+
env; in ``"api"`` mode nothing is seeded and the OAuth token is stripped instead. Either
|
|
567
|
+
way exactly one credential reaches the run, so billing is deterministic.
|
|
568
|
+
|
|
569
|
+
Every isolated agent (bench sandboxes and the draft/improve/tasks meta-agents) builds its
|
|
570
|
+
env through here, so the seed-and-scrub pairing lives in one place. ``extra_allow`` (the
|
|
571
|
+
operator's ``env_passthrough``) names variables to carry through on top of the built-in
|
|
572
|
+
allowlist, for a target that needs one at runtime.
|
|
573
|
+
"""
|
|
574
|
+
if auth_mode == "session":
|
|
575
|
+
seed_credentials(config_dir)
|
|
576
|
+
return scrubbed_env(
|
|
577
|
+
config_dir=config_dir, home=home, extra_path=extra_path, auth_mode=auth_mode, extra_allow=extra_allow
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _default_cache_root() -> Path:
|
|
582
|
+
base = os.environ.get("XDG_CACHE_HOME")
|
|
583
|
+
root = Path(base) if base else Path.home() / ".cache"
|
|
584
|
+
return root / "acumen"
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
DEFAULT_CACHE_ROOT = _default_cache_root()
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def sdk_version() -> str:
|
|
591
|
+
"""Return the installed ``claude-agent-sdk`` version, for the run fingerprint."""
|
|
592
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
593
|
+
|
|
594
|
+
try:
|
|
595
|
+
return version("claude-agent-sdk")
|
|
596
|
+
except PackageNotFoundError: # pragma: no cover - the SDK is a hard dependency
|
|
597
|
+
return "unknown"
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def python_version() -> str:
|
|
601
|
+
"""Return the interpreter version running acumen itself."""
|
|
602
|
+
return f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
|