outerloop-science 0.1.0.dev4__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +1 -1
- outerloop/attempt.py +34 -7
- outerloop/measure.py +4 -4
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/METADATA +7 -2
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/RECORD +9 -9
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/licenses/NOTICE +0 -0
outerloop/__init__.py
CHANGED
outerloop/attempt.py
CHANGED
|
@@ -19,6 +19,7 @@ import os
|
|
|
19
19
|
import re
|
|
20
20
|
import shutil
|
|
21
21
|
import time
|
|
22
|
+
import traceback
|
|
22
23
|
from collections.abc import Callable, Iterable
|
|
23
24
|
from dataclasses import dataclass
|
|
24
25
|
from dataclasses import replace as dc_replace
|
|
@@ -319,18 +320,35 @@ class AttemptOutcome:
|
|
|
319
320
|
report_path: str = ""
|
|
320
321
|
|
|
321
322
|
|
|
323
|
+
def _attach_run_log(directory: Path) -> None:
|
|
324
|
+
"""Also write the kernel's log lines to `<run dir>/kernel.log`: the scheduler
|
|
325
|
+
discards a wake job's own stderr, so this is the only trace a swallowed
|
|
326
|
+
step leaves. Only for a run that exists — never a stray directory."""
|
|
327
|
+
if not directory.is_dir():
|
|
328
|
+
return
|
|
329
|
+
try:
|
|
330
|
+
handler = logging.FileHandler(directory / "kernel.log", encoding="utf-8")
|
|
331
|
+
except OSError as exc:
|
|
332
|
+
log.warning("run log unavailable: %s", exc)
|
|
333
|
+
return
|
|
334
|
+
handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
|
|
335
|
+
logging.getLogger().addHandler(handler)
|
|
336
|
+
|
|
337
|
+
|
|
322
338
|
def _best_effort(what: str, fn: Callable[[], object], secrets: tuple[str, ...] = ()) -> bool:
|
|
323
|
-
"""One ending step; a failure is logged, never raised.
|
|
339
|
+
"""One ending step; a failure is logged with its traceback, never raised.
|
|
324
340
|
|
|
325
341
|
The terminal sequence (record, report, issue post) must degrade
|
|
326
342
|
independently: a full disk must not block the GitHub post, and a network
|
|
327
|
-
failure must not block the record.
|
|
343
|
+
failure must not block the record. The traceback is the only trace a
|
|
344
|
+
swallowed step leaves, so it is logged whole (redacted).
|
|
328
345
|
"""
|
|
329
346
|
try:
|
|
330
347
|
fn()
|
|
331
348
|
return True
|
|
332
349
|
except Exception as exc:
|
|
333
|
-
|
|
350
|
+
detail = "".join(traceback.format_exception(exc)).rstrip()
|
|
351
|
+
log.warning("%s failed: %s", what, redact(detail, secrets))
|
|
334
352
|
return False
|
|
335
353
|
|
|
336
354
|
|
|
@@ -1025,8 +1043,6 @@ def _wake_author_sleep(
|
|
|
1025
1043
|
# existing wake path decides). Keep the NEW park's snapshot ref; the OLD
|
|
1026
1044
|
# sleep ref is superseded once the new park persists.
|
|
1027
1045
|
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
1028
|
-
import time
|
|
1029
|
-
|
|
1030
1046
|
try:
|
|
1031
1047
|
_park_run(
|
|
1032
1048
|
run_root,
|
|
@@ -3017,8 +3033,6 @@ def live_attempt(
|
|
|
3017
3033
|
# commit); record and keep that same ref, drop the rest. An
|
|
3018
3034
|
# author-sleep's snapshot is the tree the wake re-delivers.
|
|
3019
3035
|
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
3020
|
-
import time
|
|
3021
|
-
|
|
3022
3036
|
# anchor the deadline to the PARK (when the evals were submitted),
|
|
3023
3037
|
# not the run's start `now` — a session lasting hours would otherwise
|
|
3024
3038
|
# eat the queue budget and let the sweep cancel a still-queued eval.
|
|
@@ -3419,11 +3433,22 @@ def main() -> int:
|
|
|
3419
3433
|
)
|
|
3420
3434
|
return value
|
|
3421
3435
|
|
|
3436
|
+
def _run_id(value: str) -> str:
|
|
3437
|
+
# the id names the run directory under runs/: one path segment, so a
|
|
3438
|
+
# value cannot reach outside the run root before the record is read
|
|
3439
|
+
# ("" is the default: a fresh climb, not a resume)
|
|
3440
|
+
if value and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}", value):
|
|
3441
|
+
raise argparse.ArgumentTypeError(
|
|
3442
|
+
f"run id {value!r} is not a run directory name (want [A-Za-z0-9][A-Za-z0-9_.@-]*)"
|
|
3443
|
+
)
|
|
3444
|
+
return value
|
|
3445
|
+
|
|
3422
3446
|
parser.add_argument("--agent-id", default="agent-01", type=_agent_id)
|
|
3423
3447
|
parser.add_argument("--run-root", required=True, type=Path)
|
|
3424
3448
|
parser.add_argument(
|
|
3425
3449
|
"--resume",
|
|
3426
3450
|
default="",
|
|
3451
|
+
type=_run_id,
|
|
3427
3452
|
metavar="RUN_ID",
|
|
3428
3453
|
help="wake a parked dispatched run instead of starting a fresh climb",
|
|
3429
3454
|
)
|
|
@@ -3530,6 +3555,8 @@ def main() -> int:
|
|
|
3530
3555
|
)
|
|
3531
3556
|
args = parser.parse_args()
|
|
3532
3557
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
3558
|
+
if args.resume:
|
|
3559
|
+
_attach_run_log(run_dir_of(args.run_root, args.resume))
|
|
3533
3560
|
if not args.image and not args.uncontained:
|
|
3534
3561
|
parser.error("--image is required (or pass --uncontained explicitly, dev only)")
|
|
3535
3562
|
# NOTE: the codex author is validated on the EFFECTIVE author per path — the
|
outerloop/measure.py
CHANGED
|
@@ -24,7 +24,7 @@ from dataclasses import dataclass
|
|
|
24
24
|
from pathlib import Path
|
|
25
25
|
from typing import Any
|
|
26
26
|
|
|
27
|
-
from outerloop.compute import GONE, Compute, JobSpec, is_terminal
|
|
27
|
+
from outerloop.compute import GONE, Compute, JobSpec, is_terminal, local_mode
|
|
28
28
|
from outerloop.dispatch import (
|
|
29
29
|
eval_job_spec,
|
|
30
30
|
read_eval_result,
|
|
@@ -275,7 +275,9 @@ class DispatchedMeasurer:
|
|
|
275
275
|
seed_cache: Path | None = None
|
|
276
276
|
|
|
277
277
|
def _placement(self, m: Measure) -> tuple[str, str]:
|
|
278
|
-
|
|
278
|
+
# local compute has no lanes: the job is a subprocess on whatever GPUs
|
|
279
|
+
# the machine has (same rule as DispatchSettings.placement)
|
|
280
|
+
if m.gpus <= 0 or local_mode():
|
|
279
281
|
return self.account, self.partition
|
|
280
282
|
if not self.gpu_partition:
|
|
281
283
|
raise ValueError(
|
|
@@ -492,8 +494,6 @@ class DispatchSettings:
|
|
|
492
494
|
GPU job has no lane — a queue that can never run is worse than a
|
|
493
495
|
loud refusal. Local compute has no lanes: jobs are subprocesses on
|
|
494
496
|
whatever GPUs the machine has, so placement is empty by design."""
|
|
495
|
-
from outerloop.compute import local_mode
|
|
496
|
-
|
|
497
497
|
if gpus <= 0 or local_mode():
|
|
498
498
|
return self.account, self.partition
|
|
499
499
|
if not self.gpu_partition:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: outerloop-science
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Autonomous research agents that improve the benchmark you point them at, one verified pull request at a time
|
|
5
5
|
Project-URL: Homepage, https://outerloop.science
|
|
6
6
|
Project-URL: Repository, https://github.com/outerloop-science/outerloop
|
|
@@ -10,6 +10,11 @@ Author: Agentic Learning AI Lab, New York University
|
|
|
10
10
|
License-Expression: Apache-2.0
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
License-File: NOTICE
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
13
18
|
Requires-Python: >=3.12
|
|
14
19
|
Requires-Dist: cryptography>=42
|
|
15
20
|
Requires-Dist: pydantic>=2
|
|
@@ -26,7 +31,7 @@ Description-Content-Type: text/markdown
|
|
|
26
31
|
|
|
27
32
|
[](https://github.com/outerloop-science/outerloop/actions/workflows/ci.yml)
|
|
28
33
|
[](https://pypi.org/project/outerloop-science/)
|
|
29
|
-
[](https://pypi.org/project/outerloop-science/)
|
|
30
35
|
[](LICENSE)
|
|
31
36
|
|
|
32
37
|
**Autoresearch agents that improve your benchmark.**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
outerloop/__init__.py,sha256
|
|
1
|
+
outerloop/__init__.py,sha256=8FDFKlRnWe-vvhGRifpF7WTDHHqFC-EguDhS04L2NVo,631
|
|
2
2
|
outerloop/__main__.py,sha256=Fq2ADl3rjZ4i2m1svON_KO2kJqpwUnohLhdlacAnEpM,57
|
|
3
3
|
outerloop/appauth.py,sha256=0FJD8JdQ3r9DgCTpY7GgOudCB7s7BDbaVqBWpdt02oU,9666
|
|
4
4
|
outerloop/appmanifest.py,sha256=9L4NyUqVJhgE8BlBnK6BTpMdkhpukLtAnezZQcbyBwI,8459
|
|
5
|
-
outerloop/attempt.py,sha256=
|
|
5
|
+
outerloop/attempt.py,sha256=EKsgkyvYpvLOeLnepqsm2kEM_2zf1pNgceD1rWALAaA,172576
|
|
6
6
|
outerloop/brief.py,sha256=XGNrO5VZUASNRu4y_T3VHcGp-PCkpHPV8ACOpzXNT4M,25206
|
|
7
7
|
outerloop/cli.py,sha256=VDp_-ImCFUbXaj_zmjrPE6ceEgstcJR-fi5w9dH8J0s,24952
|
|
8
8
|
outerloop/climbboard.py,sha256=C7jVJp91hain3kxUouL6DK8n4Xpa_j_nyDtBphMP8K0,68452
|
|
@@ -25,7 +25,7 @@ outerloop/maintain.py,sha256=9_4rpOvSMBLKiO9MZnSNkPLg6nu7fv8TVJpwPIZVDB4,15525
|
|
|
25
25
|
outerloop/maintain_agent_cli.py,sha256=QRF6VUDVR05_wxXJyBgvhsUUMqm3NvSIBbwdG9Cfn3Y,3010
|
|
26
26
|
outerloop/maintain_post_cli.py,sha256=0c_i5fa1QEqUKauqyq9hTa6opviIF9Y3v6X5ezvZvyo,5567
|
|
27
27
|
outerloop/markers.py,sha256=Bi_hwWCICW6qPQ8_yB--0IrUaLLh9C3K24aDaCxHNAY,1739
|
|
28
|
-
outerloop/measure.py,sha256=
|
|
28
|
+
outerloop/measure.py,sha256=iIG7rP-rP01Un8AX3_ZCFM36s_fCm-S6jT0keC8MRsQ,22920
|
|
29
29
|
outerloop/orchestrator.py,sha256=I7bYVJYCY4Pus9jqlS4JkVqYiKum5lQc_d43Z0KG6Jg,89372
|
|
30
30
|
outerloop/panel.py,sha256=DcK3E0WdUUZPao0DWFCz3uLtm-VDmhhv1QTQHiEYcEY,7677
|
|
31
31
|
outerloop/paths.py,sha256=SpOt3zU2KS9_KZ6FiNEH9vs9fbQt8y1rWMYbZskwxHA,1501
|
|
@@ -51,9 +51,9 @@ outerloop/verify_agent.py,sha256=x9OINnpmISDIjq5q5wtJ1wUD13Obpc8UXvF18vfkONY,655
|
|
|
51
51
|
outerloop/verify_agent_cli.py,sha256=3QCQTB7MSdtN-k9x9HovLPo8q988H9o9Uus8rmaR444,3908
|
|
52
52
|
outerloop/verify_post_cli.py,sha256=qbQ-tT_MQUzKIKLTrJEgLn0O320JnrJESKyBqqjznU4,4599
|
|
53
53
|
outerloop/watcher.py,sha256=l377ldNgnmDR_jsbQyM-QrExmAwNTha0wEVGDv1TVNc,8319
|
|
54
|
-
outerloop_science-0.1.
|
|
55
|
-
outerloop_science-0.1.
|
|
56
|
-
outerloop_science-0.1.
|
|
57
|
-
outerloop_science-0.1.
|
|
58
|
-
outerloop_science-0.1.
|
|
59
|
-
outerloop_science-0.1.
|
|
54
|
+
outerloop_science-0.1.2.dist-info/METADATA,sha256=POMdPz-8sKt99-F3yNYTtFim511mmOJEmE0n8vn3XRw,7215
|
|
55
|
+
outerloop_science-0.1.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
56
|
+
outerloop_science-0.1.2.dist-info/entry_points.txt,sha256=mLYW8Dxv8FnrYdCkKV8ekwcV3TnfzPYkdbJOkYzF5xE,49
|
|
57
|
+
outerloop_science-0.1.2.dist-info/licenses/LICENSE,sha256=Kqt7czAsZdfZp9B6a3WoTt9CZuy7_0sjPNMW7KHckk8,11375
|
|
58
|
+
outerloop_science-0.1.2.dist-info/licenses/NOTICE,sha256=VTBNz2gnaoM68CoUkQe1G7VbrV-R0WfqKHrZhYYbuqI,193
|
|
59
|
+
outerloop_science-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
{outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{outerloop_science-0.1.0.dev4.dist-info → outerloop_science-0.1.2.dist-info}/licenses/NOTICE
RENAMED
|
File without changes
|