outerloop-science 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +18 -0
- outerloop/__main__.py +3 -0
- outerloop/appauth.py +230 -0
- outerloop/appmanifest.py +203 -0
- outerloop/attempt.py +3784 -0
- outerloop/brief.py +528 -0
- outerloop/cli.py +621 -0
- outerloop/climbboard.py +1395 -0
- outerloop/compute.py +654 -0
- outerloop/contract.py +492 -0
- outerloop/contract_cli.py +63 -0
- outerloop/disk.py +164 -0
- outerloop/dispatch.py +631 -0
- outerloop/evalcache.py +147 -0
- outerloop/followup.py +2172 -0
- outerloop/github.py +1531 -0
- outerloop/harness.py +1435 -0
- outerloop/housekeeping.py +151 -0
- outerloop/image.py +368 -0
- outerloop/init.py +744 -0
- outerloop/intake.py +126 -0
- outerloop/launchlog.py +239 -0
- outerloop/limits.py +80 -0
- outerloop/maintain.py +353 -0
- outerloop/maintain_agent_cli.py +81 -0
- outerloop/maintain_post_cli.py +140 -0
- outerloop/markers.py +48 -0
- outerloop/measure.py +529 -0
- outerloop/orchestrator.py +2011 -0
- outerloop/panel.py +188 -0
- outerloop/paths.py +40 -0
- outerloop/posting.py +160 -0
- outerloop/progress.py +170 -0
- outerloop/py.typed +0 -0
- outerloop/review.py +615 -0
- outerloop/review_agent.py +263 -0
- outerloop/review_agent_cli.py +209 -0
- outerloop/review_post_cli.py +162 -0
- outerloop/review_summarize_cli.py +165 -0
- outerloop/role_runner.py +229 -0
- outerloop/roles.py +274 -0
- outerloop/rolespec.py +91 -0
- outerloop/runstate.py +385 -0
- outerloop/steward.py +845 -0
- outerloop/style.py +12 -0
- outerloop/syscall.py +1192 -0
- outerloop/syscall_cli.py +762 -0
- outerloop/tick.py +3422 -0
- outerloop/verifier.py +403 -0
- outerloop/verify_agent.py +151 -0
- outerloop/verify_agent_cli.py +95 -0
- outerloop/verify_post_cli.py +116 -0
- outerloop/watcher.py +203 -0
- outerloop_science-0.1.0.dist-info/METADATA +152 -0
- outerloop_science-0.1.0.dist-info/RECORD +59 -0
- outerloop_science-0.1.0.dist-info/WHEEL +4 -0
- outerloop_science-0.1.0.dist-info/entry_points.txt +2 -0
- outerloop_science-0.1.0.dist-info/licenses/LICENSE +202 -0
- outerloop_science-0.1.0.dist-info/licenses/NOTICE +5 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Posting half of the verifier's tokenless split (mirrors review_post_cli).
|
|
2
|
+
|
|
3
|
+
Reads the verdict envelope the session job emitted (`VERIFY_EMIT_FILE`),
|
|
4
|
+
re-validates it, and posts through the normal verification path. This job holds
|
|
5
|
+
the write token; NO model session runs next to it, so a shell judge in the
|
|
6
|
+
session job has no write credential to lift. The artifact crosses a job
|
|
7
|
+
boundary, so nothing in it is trusted: the envelope must name this PR, the
|
|
8
|
+
bot-only skip rule is re-checked here, and every string passes the same
|
|
9
|
+
sanitizing render as the single-job path. Exits 0 on every failure — the
|
|
10
|
+
verifier never fails the target repo's CI.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import logging
|
|
17
|
+
import os
|
|
18
|
+
import sys
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from outerloop.github import EnvTokenProvider, GitHubClient
|
|
22
|
+
from outerloop.posting import EXPECTED_FAILURES, post_round, post_skip_stub
|
|
23
|
+
from outerloop.review_agent import pull_request
|
|
24
|
+
from outerloop.verifier import (
|
|
25
|
+
VERIFY_MARKER,
|
|
26
|
+
format_verify_comment,
|
|
27
|
+
verify_result_from_data,
|
|
28
|
+
verify_skip_reason,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
log = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def post_from_file(
|
|
35
|
+
client: GitHubClient, repo: str, number: int, bot_login: str, path: Path
|
|
36
|
+
) -> str | None:
|
|
37
|
+
"""Post the emitted verdict (or skip stub). Returns the round label, or
|
|
38
|
+
None when there was nothing to post or the envelope was refused."""
|
|
39
|
+
try:
|
|
40
|
+
envelope = json.loads(path.read_text())
|
|
41
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
42
|
+
log.warning("verdict file unreadable (%s); nothing posted", exc)
|
|
43
|
+
return None
|
|
44
|
+
if not isinstance(envelope, dict):
|
|
45
|
+
log.warning("verdict file is not an object; nothing posted")
|
|
46
|
+
return None
|
|
47
|
+
if envelope.get("repo") != repo or envelope.get("number") != number:
|
|
48
|
+
log.warning("verdict envelope names a different PR; refused")
|
|
49
|
+
return None
|
|
50
|
+
kind = envelope.get("kind")
|
|
51
|
+
if kind == "skip-clean":
|
|
52
|
+
log.info("session skipped cleanly (%s); nothing to post", envelope.get("detail", ""))
|
|
53
|
+
return None
|
|
54
|
+
if kind not in ("skip-stub", "findings"):
|
|
55
|
+
log.warning("unknown envelope kind %r; nothing posted", kind)
|
|
56
|
+
return None
|
|
57
|
+
try:
|
|
58
|
+
# the write authority re-checks the bot-only rule for EVERY kind: this
|
|
59
|
+
# side of the artifact boundary is the one that must never post on a
|
|
60
|
+
# non-bot PR — a forged envelope is still a post
|
|
61
|
+
pr, pr_data = pull_request(client, repo, number)
|
|
62
|
+
skip = verify_skip_reason(pr, bot_login)
|
|
63
|
+
if skip is not None:
|
|
64
|
+
log.info("skipping post on %s#%s: %s", repo, number, skip)
|
|
65
|
+
return None
|
|
66
|
+
if kind == "skip-stub":
|
|
67
|
+
post_skip_stub(client, repo, number, "verification", RuntimeError("no verdict"))
|
|
68
|
+
return "skip-stub"
|
|
69
|
+
data = envelope.get("data")
|
|
70
|
+
result = verify_result_from_data(data if isinstance(data, dict) else {})
|
|
71
|
+
body = format_verify_comment(result)
|
|
72
|
+
if body is None:
|
|
73
|
+
log.info("nothing to post")
|
|
74
|
+
return None
|
|
75
|
+
round_label = post_round(
|
|
76
|
+
client,
|
|
77
|
+
repo,
|
|
78
|
+
number,
|
|
79
|
+
VERIFY_MARKER,
|
|
80
|
+
body,
|
|
81
|
+
pr_data,
|
|
82
|
+
reviewed_by=str(envelope.get("reviewed_by", "")),
|
|
83
|
+
)
|
|
84
|
+
log.info("posted verification (%s) on %s#%s", round_label, repo, number)
|
|
85
|
+
return round_label
|
|
86
|
+
except EXPECTED_FAILURES as exc: # advisory: never fail the target repo's CI
|
|
87
|
+
log.warning("posting did not complete: %s: %s", type(exc).__name__, exc)
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main() -> int:
|
|
92
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
93
|
+
repo = os.environ.get("PR_REPO", "").strip()
|
|
94
|
+
number_raw = os.environ.get("PR_NUMBER", "").strip()
|
|
95
|
+
if not repo or not number_raw.isdigit():
|
|
96
|
+
log.warning("PR_REPO/PR_NUMBER unset or invalid; skipping")
|
|
97
|
+
return 0
|
|
98
|
+
bot_login = os.environ.get("REVIEW_BOT_LOGIN", "").strip()
|
|
99
|
+
if not bot_login:
|
|
100
|
+
log.warning("REVIEW_BOT_LOGIN is unset; skipping (cannot re-check the bot rule)")
|
|
101
|
+
return 0
|
|
102
|
+
emit_file = os.environ.get("VERIFY_EMIT_FILE", "").strip()
|
|
103
|
+
if not emit_file:
|
|
104
|
+
log.warning("VERIFY_EMIT_FILE is unset; skipping")
|
|
105
|
+
return 0
|
|
106
|
+
path = Path(emit_file).resolve()
|
|
107
|
+
if not path.is_file():
|
|
108
|
+
log.info("no verdict file at %s (clean skip upstream); nothing to post", path)
|
|
109
|
+
return 0
|
|
110
|
+
client = GitHubClient(auth=EnvTokenProvider("GITHUB_TOKEN"))
|
|
111
|
+
post_from_file(client, repo, int(number_raw), bot_login, path)
|
|
112
|
+
return 0
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
sys.exit(main())
|
outerloop/watcher.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""The session watcher: a thread beside a live author session that answers its
|
|
2
|
+
questions through the syscall channel in seconds — `queue` (the kernel's jobs
|
|
3
|
+
in the cluster queue, every agent's) and `history` (this run's launches) —
|
|
4
|
+
where the tick would take a cadence. It changes no lifecycle state: it writes
|
|
5
|
+
answer files into the channel and nothing else; submitting, cancelling,
|
|
6
|
+
sealing and parking stay at the sleep boundary (docs/design/session-watcher.md).
|
|
7
|
+
|
|
8
|
+
Failure never reaches the session: an exception is logged and the request is
|
|
9
|
+
left standing, the tool times out and says so. A watcher that dies leaves the
|
|
10
|
+
session exactly as it is without one."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import re
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from outerloop.climbboard import queue_rows
|
|
24
|
+
from outerloop.launchlog import history, why_by_job
|
|
25
|
+
from outerloop.runstate import run_dir
|
|
26
|
+
from outerloop.syscall import MAX_WHY_CHARS, mark_done, marker_requested, write_channel_json
|
|
27
|
+
|
|
28
|
+
log = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
VERBS = ("queue", "history")
|
|
31
|
+
POLL_S = 2.0
|
|
32
|
+
# one scheduler query per this many seconds however often the marker is
|
|
33
|
+
# rewritten: a session cannot turn the watcher into a squeue loop
|
|
34
|
+
QUERY_GAP_S = 5.0
|
|
35
|
+
MAX_QUEUE_ROWS = 500
|
|
36
|
+
# squeue names a pending array `<id>_[0-15%4]` and a running task `<id>_3`
|
|
37
|
+
_ARRAY_THROTTLE = re.compile(r"_\[[^\]]*%(\d+)\]$")
|
|
38
|
+
# the lane's node states move slowly; one sinfo a minute is plenty
|
|
39
|
+
LANE_GAP_S = 60.0
|
|
40
|
+
# a stopped watcher may still be inside a scheduler query (its own timeout,
|
|
41
|
+
# a minute at most); the attempt waits this long for it, then moves on — the
|
|
42
|
+
# thread is a daemon and writes nothing once stopped
|
|
43
|
+
JOIN_S = 15.0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class WatcherContext:
|
|
48
|
+
"""What the watcher needs to answer: where the channel is, which run it
|
|
49
|
+
serves, and the compute backend to ask (None = no queue, local mode)."""
|
|
50
|
+
|
|
51
|
+
workspace: Path
|
|
52
|
+
run_root: Path
|
|
53
|
+
run_id: str
|
|
54
|
+
target: str
|
|
55
|
+
agent_id: str
|
|
56
|
+
compute: Any = None
|
|
57
|
+
gpu_partition: str = ""
|
|
58
|
+
poll_s: float = POLL_S
|
|
59
|
+
query_gap_s: float = QUERY_GAP_S
|
|
60
|
+
lane_gap_s: float = LANE_GAP_S
|
|
61
|
+
clock: Callable[[], float] = field(default=time.time)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _kind(name: str) -> str:
|
|
65
|
+
"""A short label for a kernel job that is not a launch."""
|
|
66
|
+
for key in ("wake", "followup", "eval", "resident", "tick"):
|
|
67
|
+
if key in name:
|
|
68
|
+
return key
|
|
69
|
+
return name
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SessionWatcher:
|
|
73
|
+
"""`with SessionWatcher(ctx):` around the harness subprocess. `service()`
|
|
74
|
+
is one pass, public so the loop is testable without the thread."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, ctx: WatcherContext) -> None:
|
|
77
|
+
self.ctx = ctx
|
|
78
|
+
self._stop = threading.Event()
|
|
79
|
+
self._thread: threading.Thread | None = None
|
|
80
|
+
self._queue_cache: dict[str, Any] | None = None
|
|
81
|
+
self._queue_at = 0.0
|
|
82
|
+
self._lane_cache: dict[str, Any] | None = None
|
|
83
|
+
self._lane_at = 0.0
|
|
84
|
+
|
|
85
|
+
def __enter__(self) -> SessionWatcher:
|
|
86
|
+
self._thread = threading.Thread(target=self._loop, name="session-watcher", daemon=True)
|
|
87
|
+
self._thread.start()
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
def __exit__(self, *exc: object) -> None:
|
|
91
|
+
self._stop.set()
|
|
92
|
+
if self._thread is not None:
|
|
93
|
+
self._thread.join(timeout=JOIN_S)
|
|
94
|
+
if self._thread.is_alive():
|
|
95
|
+
log.warning("session watcher still inside a scheduler query at session end")
|
|
96
|
+
|
|
97
|
+
def _loop(self) -> None:
|
|
98
|
+
# wait() is False on timeout: poll; True once stopped
|
|
99
|
+
while not self._stop.wait(self.ctx.poll_s):
|
|
100
|
+
self.service()
|
|
101
|
+
|
|
102
|
+
def service(self) -> None:
|
|
103
|
+
"""Answer every verb whose request marker is newer than its done marker."""
|
|
104
|
+
for verb in VERBS:
|
|
105
|
+
if self._stop.is_set():
|
|
106
|
+
return
|
|
107
|
+
try:
|
|
108
|
+
requested = marker_requested(self.ctx.workspace, f"{verb}-request", f"{verb}-done")
|
|
109
|
+
if requested is None:
|
|
110
|
+
continue
|
|
111
|
+
payload = self.queue_view() if verb == "queue" else self.history_view()
|
|
112
|
+
if self._stop.is_set():
|
|
113
|
+
return # the session ended meanwhile: nothing is written after it
|
|
114
|
+
write_channel_json(self.ctx.workspace, f"{verb}.json", payload)
|
|
115
|
+
mark_done(self.ctx.workspace, f"{verb}-done", requested)
|
|
116
|
+
except Exception as exc: # the request stands; the tool times out
|
|
117
|
+
log.warning("session watcher: %s failed: %s", verb, exc)
|
|
118
|
+
|
|
119
|
+
def history_view(self) -> dict[str, Any]:
|
|
120
|
+
ctx = self.ctx
|
|
121
|
+
return {
|
|
122
|
+
"at": ctx.clock(),
|
|
123
|
+
"run_id": ctx.run_id,
|
|
124
|
+
"history": history(run_dir(ctx.run_root, ctx.run_id)),
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
def queue_view(self) -> dict[str, Any]:
|
|
128
|
+
"""The kernel's jobs as the board projects them (attributed to agents),
|
|
129
|
+
each launch labelled with its author's `why`, plus the GPU lane's load.
|
|
130
|
+
Cached for `query_gap_s` — the rate limit on scheduler queries."""
|
|
131
|
+
ctx = self.ctx
|
|
132
|
+
now = ctx.clock()
|
|
133
|
+
if self._queue_cache is not None and now - self._queue_at < ctx.query_gap_s:
|
|
134
|
+
return {**self._queue_cache, "at": now, "cached": True}
|
|
135
|
+
error = ""
|
|
136
|
+
rows: list[dict[str, Any]] = []
|
|
137
|
+
try:
|
|
138
|
+
snapshot = ctx.compute.queue_snapshot() if ctx.compute is not None else []
|
|
139
|
+
rows = [
|
|
140
|
+
dict(r) for r in queue_rows(ctx.run_root, ctx.target, snapshot)[:MAX_QUEUE_ROWS]
|
|
141
|
+
]
|
|
142
|
+
except Exception as exc:
|
|
143
|
+
error = str(exc)[:200]
|
|
144
|
+
labels: dict[str, dict[str, Any]] = {}
|
|
145
|
+
for rid in {str(r.get("run_id") or "") for r in rows} - {""}:
|
|
146
|
+
labels.update(why_by_job(run_dir(ctx.run_root, rid)))
|
|
147
|
+
for r in rows:
|
|
148
|
+
rid = str(r.get("run_id") or "")
|
|
149
|
+
name = str(r.get("name") or "")
|
|
150
|
+
prefix = f"{rid}-launch-" if rid else ""
|
|
151
|
+
r["mine"] = bool(rid) and rid == ctx.run_id
|
|
152
|
+
job_id = str(r.get("id") or "")
|
|
153
|
+
throttle = _ARRAY_THROTTLE.search(job_id)
|
|
154
|
+
if throttle:
|
|
155
|
+
r["concurrency"] = int(throttle.group(1))
|
|
156
|
+
if prefix and name.startswith(prefix):
|
|
157
|
+
r["experiment"] = name[len(prefix) :]
|
|
158
|
+
# the ledger keys the array's id; squeue shows `<id>_<k>` or `<id>_[...]`
|
|
159
|
+
base = job_id.split("_", 1)[0]
|
|
160
|
+
label = labels.get(base, {})
|
|
161
|
+
r["why"] = str(label.get("why", ""))[:MAX_WHY_CHARS]
|
|
162
|
+
r["kind"] = "launch"
|
|
163
|
+
# the pace: the pending range says it; once only running tasks
|
|
164
|
+
# are left, the ledger does (the whole array when unset)
|
|
165
|
+
array = int(label.get("array") or 1)
|
|
166
|
+
if "concurrency" not in r and array > 1:
|
|
167
|
+
r["concurrency"] = int(label.get("concurrency") or array)
|
|
168
|
+
else:
|
|
169
|
+
r["experiment"] = ""
|
|
170
|
+
r["why"] = ""
|
|
171
|
+
r["kind"] = _kind(name)
|
|
172
|
+
view = {
|
|
173
|
+
"at": now,
|
|
174
|
+
"run_id": ctx.run_id,
|
|
175
|
+
"agent": ctx.agent_id,
|
|
176
|
+
"jobs": rows,
|
|
177
|
+
"lane": self._lane_load(),
|
|
178
|
+
"error": error,
|
|
179
|
+
}
|
|
180
|
+
self._queue_cache, self._queue_at = view, now
|
|
181
|
+
return view
|
|
182
|
+
|
|
183
|
+
def _lane_load(self) -> dict[str, Any]:
|
|
184
|
+
"""The GPU lane's node states, refreshed once per `lane_gap_s` (a
|
|
185
|
+
second scheduler query per request would defeat the rate limit). A
|
|
186
|
+
failed sinfo is reported in the answer, never passed off as a lane
|
|
187
|
+
with no state."""
|
|
188
|
+
ctx = self.ctx
|
|
189
|
+
load = getattr(ctx.compute, "lane_load", None)
|
|
190
|
+
if load is None or not ctx.gpu_partition:
|
|
191
|
+
return {}
|
|
192
|
+
now = ctx.clock()
|
|
193
|
+
if self._lane_cache is not None and now - self._lane_at < ctx.lane_gap_s:
|
|
194
|
+
return self._lane_cache
|
|
195
|
+
try:
|
|
196
|
+
lane: dict[str, Any] = {
|
|
197
|
+
"partition": ctx.gpu_partition,
|
|
198
|
+
"nodes": load(ctx.gpu_partition),
|
|
199
|
+
}
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
lane = {"partition": ctx.gpu_partition, "nodes": {}, "error": str(exc)[:200]}
|
|
202
|
+
self._lane_cache, self._lane_at = lane, now
|
|
203
|
+
return lane
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: outerloop-science
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Autonomous research agents that improve the benchmark you point them at, one verified pull request at a time
|
|
5
|
+
Project-URL: Homepage, https://outerloop.science
|
|
6
|
+
Project-URL: Repository, https://github.com/outerloop-science/outerloop
|
|
7
|
+
Project-URL: Changelog, https://github.com/outerloop-science/outerloop/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Issues, https://github.com/outerloop-science/outerloop/issues
|
|
9
|
+
Author: Agentic Learning AI Lab, New York University
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
License-File: NOTICE
|
|
13
|
+
Requires-Python: >=3.12
|
|
14
|
+
Requires-Dist: cryptography>=42
|
|
15
|
+
Requires-Dist: pydantic>=2
|
|
16
|
+
Requires-Dist: pyyaml>=6
|
|
17
|
+
Provides-Extra: app-auth
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
<picture>
|
|
21
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/outerloop-science/outerloop/main/docs/assets/icon-dark.svg">
|
|
22
|
+
<img alt="Outerloop" src="https://raw.githubusercontent.com/outerloop-science/outerloop/main/docs/assets/icon-light.svg" width="132">
|
|
23
|
+
</picture>
|
|
24
|
+
|
|
25
|
+
# Outerloop
|
|
26
|
+
|
|
27
|
+
[](https://github.com/outerloop-science/outerloop/actions/workflows/ci.yml)
|
|
28
|
+
[](https://pypi.org/project/outerloop-science/)
|
|
29
|
+
[](https://pypi.org/project/outerloop-science/)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
|
|
32
|
+
**Autoresearch agents that improve your benchmark.**
|
|
33
|
+
|
|
34
|
+
Outerloop runs AI agents on your own research code. An agent proposes a change,
|
|
35
|
+
runs the experiment on your cluster, and opens a pull request only when your
|
|
36
|
+
benchmark actually improves. Every attempt is written up, including the ones
|
|
37
|
+
that failed.
|
|
38
|
+
|
|
39
|
+
You run it yourself: your keys, your compute, your repos. Nothing reports back
|
|
40
|
+
to us. It is built and used every day by the
|
|
41
|
+
[Agentic Learning AI Lab](https://agenticlearning.ai) at NYU, where it
|
|
42
|
+
co-develops our research codebases.
|
|
43
|
+
|
|
44
|
+
## How it works
|
|
45
|
+
|
|
46
|
+
1. **Propose.** An agent picks a hypothesis and writes the code change.
|
|
47
|
+
2. **Experiment.** It runs the training on your cluster and reads the results.
|
|
48
|
+
3. **Measure.** Outerloop scores the change against the base tree at the same
|
|
49
|
+
seed. Noise does not count as an improvement.
|
|
50
|
+
4. **Review.** Reviewers read the change and the claim. If both hold up, a
|
|
51
|
+
pull request opens.
|
|
52
|
+
5. **Record.** Every attempt gets a short report: hypothesis, change, outcome,
|
|
53
|
+
next step. Negative results included.
|
|
54
|
+
|
|
55
|
+
Agents cannot touch the benchmark, the budgets, or your CI. Your branch
|
|
56
|
+
protection and required checks apply to them as to any contributor. By default
|
|
57
|
+
a pull request waits for a human; a repo can also let clean ones merge
|
|
58
|
+
themselves.
|
|
59
|
+
|
|
60
|
+
## Get started
|
|
61
|
+
|
|
62
|
+
Three commands and one file. You need a repo with a benchmark command, an API
|
|
63
|
+
key for the model that will write the code, and a Slurm cluster or one machine
|
|
64
|
+
with a GPU.
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install outerloop-science
|
|
68
|
+
outerloop init # where the loop runs, which repo, which model and its key, your GitHub identity
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The wizard asks for a GitHub identity for the agents to open pull requests
|
|
72
|
+
as. Pick `app` and it walks you through creating a GitHub App, under your
|
|
73
|
+
account or under an organization you name, and installing it on the repo:
|
|
74
|
+
two browser pages and a code pasted back. It then checks that the App can
|
|
75
|
+
write the repo and tells you if it cannot. Pick `pat` if you already have a
|
|
76
|
+
token. It writes the config and the key files; nothing to edit by hand. Then
|
|
77
|
+
add one file, `.outerloop.yaml`, to the repo you want improved:
|
|
78
|
+
|
|
79
|
+
```yaml
|
|
80
|
+
benchmarks:
|
|
81
|
+
- name: my-benchmark
|
|
82
|
+
command: uv run python -m mypkg.eval --json # prints {"success_rate": 0.42}
|
|
83
|
+
metric: success_rate
|
|
84
|
+
direction: max
|
|
85
|
+
budgets:
|
|
86
|
+
gpu_hours_per_run: 8
|
|
87
|
+
runs_per_week: 10
|
|
88
|
+
scope:
|
|
89
|
+
allowed: [src/] # the only paths an agent may change
|
|
90
|
+
roadmap: docs/roadmap.md # what the agents read for direction; never written
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
outerloop start # on a Slurm login node this submits the loop; without Slurm it runs in the foreground
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Step by step, other model backends included:
|
|
98
|
+
[docs/install.md](https://github.com/outerloop-science/outerloop/blob/main/docs/install.md).
|
|
99
|
+
Everything the contract can say:
|
|
100
|
+
[docs/contract.md](https://github.com/outerloop-science/outerloop/blob/main/docs/contract.md).
|
|
101
|
+
|
|
102
|
+
## Only want pull request reviews?
|
|
103
|
+
|
|
104
|
+
The reviewer works on its own. One workflow file and an API key, about five
|
|
105
|
+
minutes, no bot account and no cluster. It comments on pull requests with
|
|
106
|
+
concrete findings and never approves, blocks, or fails your build. See
|
|
107
|
+
[docs/reviewer.md](https://github.com/outerloop-science/outerloop/blob/main/docs/reviewer.md).
|
|
108
|
+
|
|
109
|
+
## Where it runs
|
|
110
|
+
|
|
111
|
+
The first-class home is a Slurm cluster. There is no daemon: the loop is a
|
|
112
|
+
chain of short jobs that resubmit themselves, so nothing listens and no inbound
|
|
113
|
+
SSH is needed. Experiments and evaluations run inside your container image with
|
|
114
|
+
no credentials, and GPU-hours are metered against the contract's budget. A
|
|
115
|
+
single machine with a GPU works too, for cheap benchmarks. Details:
|
|
116
|
+
[docs/compute.md](https://github.com/outerloop-science/outerloop/blob/main/docs/compute.md).
|
|
117
|
+
|
|
118
|
+
## Safety by design
|
|
119
|
+
|
|
120
|
+
- **Opt-in and contract-bound.** A repo takes part by granting the bot access
|
|
121
|
+
and committing a contract. The contract, your roadmap, and `.github/` are
|
|
122
|
+
never writable by an agent.
|
|
123
|
+
- **Nothing on trust.** Outerloop measures every claim itself, on committed
|
|
124
|
+
trees, and re-verifies before a pull request exists.
|
|
125
|
+
- **Untrusted input.** Pull request text, diffs, issues, web pages, and job
|
|
126
|
+
output are data, never instructions. Agents run without credentials.
|
|
127
|
+
- **Budgets in code.** Launches, GPU-hours, and runs per week are enforced by
|
|
128
|
+
the kernel, not left to the agent.
|
|
129
|
+
- **No model lock-in.** Claude Code, Codex, and hermes-agent are wired today;
|
|
130
|
+
backends are swappable.
|
|
131
|
+
|
|
132
|
+
Full design: [docs/design/architecture.md](https://github.com/outerloop-science/outerloop/blob/main/docs/design/architecture.md) ·
|
|
133
|
+
Roadmap: [docs/roadmap.md](https://github.com/outerloop-science/outerloop/blob/main/docs/roadmap.md)
|
|
134
|
+
|
|
135
|
+
## Developing
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
uv sync
|
|
139
|
+
uv run pre-commit install
|
|
140
|
+
uv run pytest
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
| Path | Purpose |
|
|
144
|
+
| --- | --- |
|
|
145
|
+
| `src/outerloop/` | The kernel: contract, tick (the Slurm chain), attempt/orchestrator (the climb), measure/dispatch (evals as jobs), syscall (the author's tool), panel/verifier/review, github, harness backends |
|
|
146
|
+
| `tests/` | Tiers: unit (default), `slow`, `llm`, `slurm` markers |
|
|
147
|
+
| `scripts/` | Committed operational scripts (the tick chain, provisioning) |
|
|
148
|
+
| `docs/` | Install guide, architecture and design notes, roadmap |
|
|
149
|
+
|
|
150
|
+
## License
|
|
151
|
+
|
|
152
|
+
Apache License 2.0 — see [LICENSE](https://github.com/outerloop-science/outerloop/blob/main/LICENSE) and [NOTICE](https://github.com/outerloop-science/outerloop/blob/main/NOTICE).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
outerloop/__init__.py,sha256=mQ99KR3e8OsTlibN0yyEkwZoMX1aCgzaH4mgYjYO8HA,631
|
|
2
|
+
outerloop/__main__.py,sha256=Fq2ADl3rjZ4i2m1svON_KO2kJqpwUnohLhdlacAnEpM,57
|
|
3
|
+
outerloop/appauth.py,sha256=0FJD8JdQ3r9DgCTpY7GgOudCB7s7BDbaVqBWpdt02oU,9666
|
|
4
|
+
outerloop/appmanifest.py,sha256=9L4NyUqVJhgE8BlBnK6BTpMdkhpukLtAnezZQcbyBwI,8459
|
|
5
|
+
outerloop/attempt.py,sha256=JAuJhpWHppK8uZPeNMfLiCQ6tz-W_Lq00sfSzdHqq0Y,171201
|
|
6
|
+
outerloop/brief.py,sha256=XGNrO5VZUASNRu4y_T3VHcGp-PCkpHPV8ACOpzXNT4M,25206
|
|
7
|
+
outerloop/cli.py,sha256=VDp_-ImCFUbXaj_zmjrPE6ceEgstcJR-fi5w9dH8J0s,24952
|
|
8
|
+
outerloop/climbboard.py,sha256=C7jVJp91hain3kxUouL6DK8n4Xpa_j_nyDtBphMP8K0,68452
|
|
9
|
+
outerloop/compute.py,sha256=aC5yecD4o6C1geA_OAiA4pzts1vXsiskzvLNbPOXPe0,27926
|
|
10
|
+
outerloop/contract.py,sha256=wGeLhFl57wQQi57uZmJYb9M_21H8Dj5hHIOLSs06zOY,25101
|
|
11
|
+
outerloop/contract_cli.py,sha256=Pa-sJe7G8BuLGed5tqnlq6L75fTT1HHO4d-XWZPW8U8,1992
|
|
12
|
+
outerloop/disk.py,sha256=FAgR1FJVPTu1YCcfNxcJtUFxJsBFcOaenF8lYAVPL1Y,5853
|
|
13
|
+
outerloop/dispatch.py,sha256=hRW8K_OxFR9cmWcUg89XZSc_t3UGCY5X2AxRhuGOehk,29207
|
|
14
|
+
outerloop/evalcache.py,sha256=uVCcG3HjHNpmFRuQbMyQDVYE_-W1a3v-vxN3mDGZoNY,5045
|
|
15
|
+
outerloop/followup.py,sha256=AYD1ir-PydgK1ItSZCqA2w-rZQ1lxMn5Ymr1tkry9cg,95200
|
|
16
|
+
outerloop/github.py,sha256=4Z0hFjuhWcWuIgM8VBabh218AWPKmYWORYFyejnC0w4,67854
|
|
17
|
+
outerloop/harness.py,sha256=2UH2WZOKgQaeERVsIx3KX3muZRarYMOC4f25SLRw7qA,66092
|
|
18
|
+
outerloop/housekeeping.py,sha256=w1GWSIRZq3yBgxmIi__f0f_OF8TtKqaR-WtLvsWEjHI,6001
|
|
19
|
+
outerloop/image.py,sha256=bXCB-VqFfzJnnYVgclC0JuCN-AXJZkq938z4vV8Bvoo,15330
|
|
20
|
+
outerloop/init.py,sha256=IwxAEG8mvESP__Z8A_tqnPG5c2cAoMcRf6W3gFodq_g,34393
|
|
21
|
+
outerloop/intake.py,sha256=UxJddNYfB2IuV57mcUZ_7ShfJTJXavfUiFm5xo7g9lg,5243
|
|
22
|
+
outerloop/launchlog.py,sha256=oJf8ADmptkB_yz8-Dp6smkPjgn3Yl3oKpEEFAlNflN0,8975
|
|
23
|
+
outerloop/limits.py,sha256=ubdBqhtN8SN4r5RoDhuptoolqtq_m-1KZPdW3iXaml8,3385
|
|
24
|
+
outerloop/maintain.py,sha256=9_4rpOvSMBLKiO9MZnSNkPLg6nu7fv8TVJpwPIZVDB4,15525
|
|
25
|
+
outerloop/maintain_agent_cli.py,sha256=QRF6VUDVR05_wxXJyBgvhsUUMqm3NvSIBbwdG9Cfn3Y,3010
|
|
26
|
+
outerloop/maintain_post_cli.py,sha256=0c_i5fa1QEqUKauqyq9hTa6opviIF9Y3v6X5ezvZvyo,5567
|
|
27
|
+
outerloop/markers.py,sha256=Bi_hwWCICW6qPQ8_yB--0IrUaLLh9C3K24aDaCxHNAY,1739
|
|
28
|
+
outerloop/measure.py,sha256=hUCy17FcWyd6yIFhRyNrLfLfNWtSs1yasFVYJKUbVd0,22795
|
|
29
|
+
outerloop/orchestrator.py,sha256=I7bYVJYCY4Pus9jqlS4JkVqYiKum5lQc_d43Z0KG6Jg,89372
|
|
30
|
+
outerloop/panel.py,sha256=DcK3E0WdUUZPao0DWFCz3uLtm-VDmhhv1QTQHiEYcEY,7677
|
|
31
|
+
outerloop/paths.py,sha256=SpOt3zU2KS9_KZ6FiNEH9vs9fbQt8y1rWMYbZskwxHA,1501
|
|
32
|
+
outerloop/posting.py,sha256=dUi7P46HzsQmVgs5BXy5GXGOkcTNjOZDO-x_y0yAaaU,6632
|
|
33
|
+
outerloop/progress.py,sha256=9vCKsO8_NGrw3-hhSif5QA6rLoqNOrQ8ICIpVBqD2VI,6133
|
|
34
|
+
outerloop/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
|
+
outerloop/review.py,sha256=rpqdqhNjlvmbRln_zNbhrGyNmpt39Pxu5Oq7jEVcYtI,27416
|
|
36
|
+
outerloop/review_agent.py,sha256=sPN2-mAfe1e3uLn2ogJrppIMAvGuS2NTgiitCQHlaRw,9999
|
|
37
|
+
outerloop/review_agent_cli.py,sha256=qWE79-rP8HqNPLFAgyuNJxoHCtxowV-Ee6GKJib644U,8371
|
|
38
|
+
outerloop/review_post_cli.py,sha256=VzZunoNOzNTDIR710hDJ2n3xE2lNtLGL3KFgwV-1Bmo,6264
|
|
39
|
+
outerloop/review_summarize_cli.py,sha256=f1MK9tMo6P4SfrwJS0CFbwV35XaACL_WyzhZCgT53rs,6862
|
|
40
|
+
outerloop/role_runner.py,sha256=4l4zw0TtiFrGDmzHW8T6vZ7OLq7vVD0cN4U4BoNbgUs,9681
|
|
41
|
+
outerloop/roles.py,sha256=PyhWGhZm7tEr_t5sfiPJplxO4BK3JGGB5lGgQFMKIcI,12253
|
|
42
|
+
outerloop/rolespec.py,sha256=Y9vsUsPLtSc1ricvFSaA2qmOMkOg76TPnHsdGZOi0ME,3935
|
|
43
|
+
outerloop/runstate.py,sha256=5bEczCz1KTbVI1cTwNvd4Oc2RXIX-9piWD5-g_S6tMo,17976
|
|
44
|
+
outerloop/steward.py,sha256=iv6OUpZ-QypJbQZJaoYKz89FfpmFH6Xy53ADyCVLF5s,34107
|
|
45
|
+
outerloop/style.py,sha256=icZQNq0hOLjhoXStpH7rxZm24fsAh4uL8NMH72nz9S0,497
|
|
46
|
+
outerloop/syscall.py,sha256=XsjS0TkzvdNt2HjOSiVxgcbwf8QKIXibfLq2DbDiQcI,54317
|
|
47
|
+
outerloop/syscall_cli.py,sha256=srjqlNDhSmrPJI4haYRxVsGwzy4uO-dx_Uhu5fneLk0,31768
|
|
48
|
+
outerloop/tick.py,sha256=95gHa44SWIldM9RHSkiD4vXJCPQm6K2PTTDYaMVNtbI,154872
|
|
49
|
+
outerloop/verifier.py,sha256=BeizeiAqXGiuL-pmGptedtFo_GWAxfguwZhoTdAn1qg,17793
|
|
50
|
+
outerloop/verify_agent.py,sha256=x9OINnpmISDIjq5q5wtJ1wUD13Obpc8UXvF18vfkONY,6550
|
|
51
|
+
outerloop/verify_agent_cli.py,sha256=3QCQTB7MSdtN-k9x9HovLPo8q988H9o9Uus8rmaR444,3908
|
|
52
|
+
outerloop/verify_post_cli.py,sha256=qbQ-tT_MQUzKIKLTrJEgLn0O320JnrJESKyBqqjznU4,4599
|
|
53
|
+
outerloop/watcher.py,sha256=l377ldNgnmDR_jsbQyM-QrExmAwNTha0wEVGDv1TVNc,8319
|
|
54
|
+
outerloop_science-0.1.0.dist-info/METADATA,sha256=KZqpVJ7s8-PvwtFeOlfMv7uiT7ucj00_CTH_BDKoREg,6962
|
|
55
|
+
outerloop_science-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
56
|
+
outerloop_science-0.1.0.dist-info/entry_points.txt,sha256=mLYW8Dxv8FnrYdCkKV8ekwcV3TnfzPYkdbJOkYzF5xE,49
|
|
57
|
+
outerloop_science-0.1.0.dist-info/licenses/LICENSE,sha256=Kqt7czAsZdfZp9B6a3WoTt9CZuy7_0sjPNMW7KHckk8,11375
|
|
58
|
+
outerloop_science-0.1.0.dist-info/licenses/NOTICE,sha256=VTBNz2gnaoM68CoUkQe1G7VbrV-R0WfqKHrZhYYbuqI,193
|
|
59
|
+
outerloop_science-0.1.0.dist-info/RECORD,,
|