outerloop-science 0.1.0.dev1__py3-none-any.whl → 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +7 -7
- outerloop/appauth.py +3 -3
- outerloop/appmanifest.py +1 -1
- outerloop/attempt.py +55 -49
- outerloop/cli.py +150 -74
- outerloop/compute.py +59 -9
- outerloop/contract.py +3 -2
- outerloop/followup.py +14 -17
- outerloop/github.py +4 -4
- outerloop/harness.py +18 -7
- outerloop/image.py +372 -0
- outerloop/init.py +229 -14
- outerloop/limits.py +1 -1
- outerloop/measure.py +2 -2
- outerloop/orchestrator.py +1 -1
- outerloop/paths.py +2 -1
- outerloop/review.py +1 -1
- outerloop/steward.py +3 -3
- outerloop/syscall.py +1 -0
- outerloop/tick.py +302 -77
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/METADATA +2 -2
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/RECORD +26 -25
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/licenses/NOTICE +0 -0
outerloop/__init__.py
CHANGED
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
|
|
5
|
-
__version__ = "0.1.0.
|
|
5
|
+
__version__ = "0.1.0.dev2"
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def _bridge_legacy_env() -> None:
|
|
9
|
-
"""Accept the
|
|
10
|
-
its `
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
"""Accept the pre-rename `AUTORESEARCH_*` names at the process boundary for
|
|
10
|
+
one release: copy each into its `OUTERLOOP_*` twin when the twin is unset,
|
|
11
|
+
so the kernel reads one name. The chain scripts do the same in shell.
|
|
12
|
+
Removed in the release after 0.1."""
|
|
13
13
|
for key, value in list(os.environ.items()):
|
|
14
|
-
if key.startswith("
|
|
15
|
-
os.environ.setdefault("
|
|
14
|
+
if key.startswith("AUTORESEARCH_"):
|
|
15
|
+
os.environ.setdefault("OUTERLOOP_" + key[len("AUTORESEARCH_") :], value)
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
_bridge_legacy_env()
|
outerloop/appauth.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
`AppInstallationTokenProvider` satisfies the `TokenProvider` protocol used
|
|
4
4
|
throughout `github.py`. Role CLIs construct bot auth through
|
|
5
5
|
`resolve_bot_auth`, which selects this provider when an App config file is
|
|
6
|
-
given (`--github-app-file` / `
|
|
6
|
+
given (`--github-app-file` / `OUTERLOOP_GITHUB_APP_FILE`) and falls back
|
|
7
7
|
to the PAT file otherwise — the cutover flag, revertible by unsetting the
|
|
8
8
|
env. Each `token()` mints a short-lived JWT (RS256, signed by the App
|
|
9
9
|
private key), exchanges it for a ~1h installation token scoped to the
|
|
@@ -172,8 +172,8 @@ def signer_from_private_key(pem_path: Path) -> Signer:
|
|
|
172
172
|
from cryptography.hazmat.primitives.asymmetric import padding, rsa
|
|
173
173
|
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
174
174
|
raise ImportError(
|
|
175
|
-
"GitHub App auth needs the
|
|
176
|
-
"install
|
|
175
|
+
"GitHub App auth needs the cryptography package "
|
|
176
|
+
"(pip install cryptography), a base dependency since 0.1.0.dev2"
|
|
177
177
|
) from exc
|
|
178
178
|
|
|
179
179
|
key = serialization.load_pem_private_key(pem_path.read_bytes(), password=None)
|
outerloop/appmanifest.py
CHANGED
|
@@ -14,7 +14,7 @@ and GitHub redirects back to that page with a one-time code the page displays.
|
|
|
14
14
|
The adopter pastes the code here; we exchange it for the app id + key, write
|
|
15
15
|
`github_app.<slug>.json` + the PEM (both 0600), help install the App, and capture
|
|
16
16
|
the installation id. The written files are what `resolve_bot_auth` reads via
|
|
17
|
-
`
|
|
17
|
+
`OUTERLOOP_GITHUB_APP_FILE`, the same path `outerloop start` uses.
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
outerloop/attempt.py
CHANGED
|
@@ -44,7 +44,7 @@ from outerloop.github import (
|
|
|
44
44
|
contract_at,
|
|
45
45
|
ensure_regular_git_dir,
|
|
46
46
|
)
|
|
47
|
-
from outerloop.harness import Harness, SessionResult, redact
|
|
47
|
+
from outerloop.harness import Harness, SessionResult, default_binary, redact
|
|
48
48
|
from outerloop.markers import has_marker
|
|
49
49
|
from outerloop.measure import DispatchedMeasurer, DispatchSettings
|
|
50
50
|
from outerloop.orchestrator import (
|
|
@@ -110,18 +110,19 @@ def resolve_author_key_file(backend: str, explicit: str = "") -> str:
|
|
|
110
110
|
config choice, not a key swap, and an in-flight run of either backend can
|
|
111
111
|
still be woken/serviced after a fleet flip. An explicit path always wins;
|
|
112
112
|
otherwise the per-backend env var, then the default path. For claude the
|
|
113
|
-
legacy spellings (`AUTORESEARCH_HARNESS_KEY_FILE`,
|
|
113
|
+
legacy spellings (`AUTORESEARCH_HARNESS_KEY_FILE`, read through its bridged
|
|
114
|
+
`OUTERLOOP_` twin, and `harness_key`) are still
|
|
114
115
|
honored, so a machine set up before the rename keeps working; the new name
|
|
115
116
|
wins when both exist. The result is always ~-expanded, so every caller gets
|
|
116
117
|
a real path (an env value like "~/.config/..." must not reach the token
|
|
117
118
|
provider verbatim)."""
|
|
118
119
|
if not explicit:
|
|
119
120
|
if backend == "codex":
|
|
120
|
-
explicit = os.environ.get("
|
|
121
|
+
explicit = os.environ.get("OUTERLOOP_CODEX_KEY_FILE") or CODEX_KEY_DEFAULT
|
|
121
122
|
else:
|
|
122
123
|
explicit = (
|
|
123
|
-
os.environ.get("
|
|
124
|
-
or os.environ.get("
|
|
124
|
+
os.environ.get("OUTERLOOP_CLAUDE_KEY_FILE")
|
|
125
|
+
or os.environ.get("OUTERLOOP_HARNESS_KEY_FILE")
|
|
125
126
|
or ""
|
|
126
127
|
)
|
|
127
128
|
if not explicit:
|
|
@@ -140,14 +141,14 @@ def codex_author_config_error(backend: str, model: str, image: str) -> str:
|
|
|
140
141
|
a unit and never a fleet backend against a run's model. codex writes+executes,
|
|
141
142
|
so it must be contained (--image) and needs a non-claude model."""
|
|
142
143
|
if backend not in ("claude", "codex"):
|
|
143
|
-
# a typo'd
|
|
144
|
+
# a typo'd OUTERLOOP_AUTHOR_BACKEND passes the env DEFAULT silently
|
|
144
145
|
# (argparse validates the flag, not its default) and the climb rejects it
|
|
145
146
|
# at build_harness — catch it on the tick host so a claimed intake
|
|
146
147
|
# issue never strands on it
|
|
147
148
|
return f"unknown author backend {backend!r} (expected 'claude' or 'codex')"
|
|
148
149
|
if backend == "claude":
|
|
149
150
|
# symmetric to the codex check: a claude harness 404s on a non-claude
|
|
150
|
-
# model (e.g.
|
|
151
|
+
# model (e.g. OUTERLOOP_AUTHOR_MODEL left on a codex id while the
|
|
151
152
|
# backend is claude) — catch that misconfig before spend
|
|
152
153
|
if model and not model.startswith("claude"):
|
|
153
154
|
return f"author-backend claude needs a claude model (got {model!r})"
|
|
@@ -604,8 +605,15 @@ def _make_launcher(
|
|
|
604
605
|
partially-submitted batch is reaped rather than orphaned. `gpus` is the
|
|
605
606
|
benchmark's: an author's experiments run on the same lane as its evals."""
|
|
606
607
|
account, partition = dispatch.placement(gpus)
|
|
608
|
+
# under launch admission a GPU launch enters the queue held; the tick
|
|
609
|
+
# releases it when the user's GPUs fit under the cap (tick.service_admission)
|
|
610
|
+
from outerloop.tick import max_launch_gpus_from_env
|
|
611
|
+
|
|
612
|
+
hold = gpus > 0 and not local_mode() and max_launch_gpus_from_env() > 0
|
|
607
613
|
|
|
608
614
|
def launcher(sha: str, request: SyscallRequest) -> str:
|
|
615
|
+
from dataclasses import replace as _replace
|
|
616
|
+
|
|
609
617
|
from outerloop.dispatch import eval_job_spec, write_eval_job
|
|
610
618
|
from outerloop.syscall import launch_jobs
|
|
611
619
|
|
|
@@ -627,18 +635,15 @@ def _make_launcher(
|
|
|
627
635
|
artifact_max_bytes=MAX_ARTIFACT_BYTES,
|
|
628
636
|
gpus=gpus,
|
|
629
637
|
)
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
eval_minutes=launch.minutes,
|
|
638
|
-
gpus=gpus,
|
|
639
|
-
)
|
|
640
|
-
)
|
|
638
|
+
spec = eval_job_spec(
|
|
639
|
+
script,
|
|
640
|
+
job_name=f"{run_id}-launch-{job_name}",
|
|
641
|
+
account=account,
|
|
642
|
+
partition=partition,
|
|
643
|
+
eval_minutes=launch.minutes,
|
|
644
|
+
gpus=gpus,
|
|
641
645
|
)
|
|
646
|
+
ids.append(dispatch.compute.submit(_replace(spec, hold=hold)))
|
|
642
647
|
except Exception:
|
|
643
648
|
# a partial batch must not orphan: no park record was written yet,
|
|
644
649
|
# so nothing would ever wake or cancel the jobs that DID submit —
|
|
@@ -2171,7 +2176,7 @@ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str
|
|
|
2171
2176
|
if backend == "codex":
|
|
2172
2177
|
lens_key = _judge_lens_key(
|
|
2173
2178
|
backend="codex",
|
|
2174
|
-
key_file_env="
|
|
2179
|
+
key_file_env="OUTERLOOP_PANEL_CODEX_KEY_FILE",
|
|
2175
2180
|
author_backend="codex",
|
|
2176
2181
|
claude_panel_path=claude_panel_path,
|
|
2177
2182
|
image=args.image,
|
|
@@ -2185,7 +2190,7 @@ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str
|
|
|
2185
2190
|
# separate against the codex author key.
|
|
2186
2191
|
lens_key = _judge_lens_key(
|
|
2187
2192
|
backend="hermes",
|
|
2188
|
-
key_file_env="
|
|
2193
|
+
key_file_env="OUTERLOOP_PANEL_HERMES_KEY_FILE",
|
|
2189
2194
|
author_backend="codex",
|
|
2190
2195
|
claude_panel_path=claude_panel_path,
|
|
2191
2196
|
image=args.image,
|
|
@@ -3143,45 +3148,48 @@ def main() -> int:
|
|
|
3143
3148
|
)
|
|
3144
3149
|
parser.add_argument("--base-branch", default="main")
|
|
3145
3150
|
# All three default from the chain env the tick sets on the climb job, so
|
|
3146
|
-
# a contained run with
|
|
3147
|
-
# dispatched measurement without extra flags.
|
|
3148
|
-
# session + inline eval;
|
|
3149
|
-
# regardless of the benchmark's eval hint.
|
|
3151
|
+
# a contained run with OUTERLOOP_{IMAGE,ACCOUNT,PARTITION} set selects
|
|
3152
|
+
# dispatched measurement without extra flags. Only the image is required
|
|
3153
|
+
# for dispatch (it also contains the session + inline eval); without it,
|
|
3154
|
+
# measurement stays inline regardless of the benchmark's eval hint. Empty
|
|
3155
|
+
# account/partition use Slurm's defaults.
|
|
3150
3156
|
parser.add_argument(
|
|
3151
3157
|
"--image",
|
|
3152
|
-
default=os.environ.get("
|
|
3158
|
+
default=os.environ.get("OUTERLOOP_IMAGE", ""),
|
|
3153
3159
|
help="apptainer image for session+eval",
|
|
3154
3160
|
)
|
|
3155
|
-
parser.add_argument("--account", default=os.environ.get("
|
|
3156
|
-
parser.add_argument("--partition", default=os.environ.get("
|
|
3161
|
+
parser.add_argument("--account", default=os.environ.get("OUTERLOOP_ACCOUNT", ""))
|
|
3162
|
+
parser.add_argument("--partition", default=os.environ.get("OUTERLOOP_PARTITION", ""))
|
|
3157
3163
|
# the GPU lane for benchmarks with `gpus > 0` (evals + author launches);
|
|
3158
3164
|
# empty = this deployment cannot place GPU jobs
|
|
3159
|
-
parser.add_argument("--gpu-partition", default=os.environ.get("
|
|
3160
|
-
parser.add_argument("--gpu-account", default=os.environ.get("
|
|
3165
|
+
parser.add_argument("--gpu-partition", default=os.environ.get("OUTERLOOP_GPU_PARTITION", ""))
|
|
3166
|
+
parser.add_argument("--gpu-account", default=os.environ.get("OUTERLOOP_GPU_ACCOUNT", ""))
|
|
3161
3167
|
parser.add_argument(
|
|
3162
3168
|
"--uncontained",
|
|
3163
3169
|
action="store_true",
|
|
3164
3170
|
help="run WITHOUT a container (dev only: sessions can then read "
|
|
3165
3171
|
"same-user files, including credential files)",
|
|
3166
3172
|
)
|
|
3167
|
-
parser.add_argument(
|
|
3173
|
+
parser.add_argument(
|
|
3174
|
+
"--claude-bin",
|
|
3175
|
+
default=default_binary("claude"),
|
|
3176
|
+
help="host claude binary for the claude author (OUTERLOOP_CLAUDE_BIN, which init records)",
|
|
3177
|
+
)
|
|
3168
3178
|
parser.add_argument(
|
|
3169
3179
|
"--codex-bin",
|
|
3170
|
-
default=
|
|
3171
|
-
os.environ.get("AUTORESEARCH_CODEX_BIN") or "~/.local/bin/codex"
|
|
3172
|
-
),
|
|
3180
|
+
default=default_binary("codex"),
|
|
3173
3181
|
help="host codex binary for the codex author; bind-mounted into apptainer "
|
|
3174
3182
|
"(must be an absolute path).",
|
|
3175
3183
|
)
|
|
3176
3184
|
parser.add_argument(
|
|
3177
|
-
"--model", default=os.environ.get("
|
|
3185
|
+
"--model", default=os.environ.get("OUTERLOOP_AUTHOR_MODEL") or "claude-opus-5"
|
|
3178
3186
|
)
|
|
3179
3187
|
parser.add_argument(
|
|
3180
3188
|
"--author-backend",
|
|
3181
3189
|
choices=("claude", "codex"),
|
|
3182
|
-
default=os.environ.get("
|
|
3190
|
+
default=os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude",
|
|
3183
3191
|
help="agent backend for the author/editor role (config-driven: default "
|
|
3184
|
-
"from
|
|
3192
|
+
"from OUTERLOOP_AUTHOR_BACKEND). codex runs contained (apptainer + "
|
|
3185
3193
|
"--sandbox danger-full-access) and REQUIRES --image and a codex/openai "
|
|
3186
3194
|
"--model (e.g. gpt-5.6-terra).",
|
|
3187
3195
|
)
|
|
@@ -3225,7 +3233,7 @@ def main() -> int:
|
|
|
3225
3233
|
parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
|
|
3226
3234
|
parser.add_argument(
|
|
3227
3235
|
"--github-app-file",
|
|
3228
|
-
default=os.environ.get("
|
|
3236
|
+
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
3229
3237
|
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
3230
3238
|
"when set, installation tokens replace the PAT",
|
|
3231
3239
|
)
|
|
@@ -3233,7 +3241,7 @@ def main() -> int:
|
|
|
3233
3241
|
"--key-file",
|
|
3234
3242
|
default="",
|
|
3235
3243
|
help="author key file; default resolves per backend (config-driven): "
|
|
3236
|
-
"
|
|
3244
|
+
"OUTERLOOP_CLAUDE_KEY_FILE for claude, OUTERLOOP_CODEX_KEY_FILE for codex",
|
|
3237
3245
|
)
|
|
3238
3246
|
parser.add_argument("--issue", type=int, default=0)
|
|
3239
3247
|
parser.add_argument(
|
|
@@ -3262,12 +3270,11 @@ def main() -> int:
|
|
|
3262
3270
|
# and re-enter the decision. The wake job the WakeDispatcher submits runs
|
|
3263
3271
|
# exactly this.
|
|
3264
3272
|
if args.resume:
|
|
3265
|
-
|
|
3266
|
-
if not (placed and args.image and Path(args.image).is_file()):
|
|
3273
|
+
if not (args.image and Path(args.image).is_file()):
|
|
3267
3274
|
parser.error(
|
|
3268
|
-
"--resume needs the
|
|
3269
|
-
"
|
|
3270
|
-
"
|
|
3275
|
+
"--resume needs --image to rebuild the dispatched measurer "
|
|
3276
|
+
"(account and partition are optional: empty ones use Slurm's "
|
|
3277
|
+
"defaults, and local compute has no placement)"
|
|
3271
3278
|
)
|
|
3272
3279
|
from outerloop.runstate import load_record
|
|
3273
3280
|
|
|
@@ -3432,13 +3439,12 @@ def main() -> int:
|
|
|
3432
3439
|
except ValueError as exc:
|
|
3433
3440
|
parser.error(str(exc))
|
|
3434
3441
|
|
|
3435
|
-
# Dispatched measurement needs
|
|
3436
|
-
#
|
|
3437
|
-
# sets
|
|
3442
|
+
# Dispatched measurement needs a real image file to bind against; without
|
|
3443
|
+
# one the climb measures inline. Account and partition are optional: empty
|
|
3444
|
+
# ones use Slurm's defaults (the tick sets them on the climb job's env from
|
|
3445
|
+
# the chain's), and local compute has no placement.
|
|
3438
3446
|
dispatch: DispatchSettings | None = None
|
|
3439
|
-
if
|
|
3440
|
-
args.image and Path(args.image).is_file()
|
|
3441
|
-
):
|
|
3447
|
+
if args.image and Path(args.image).is_file():
|
|
3442
3448
|
dispatch = _dispatch_settings(args)
|
|
3443
3449
|
try:
|
|
3444
3450
|
try:
|