proteus-evolve 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proteus/__init__.py +2 -0
- proteus/adapters/__init__.py +3 -0
- proteus/adapters/aki.py +231 -0
- proteus/adapters/dsh.py +381 -0
- proteus/adapters/instructions.py +40 -0
- proteus/adapters/llm.py +172 -0
- proteus/adapters/minimal.py +141 -0
- proteus/adapters/pi.py +317 -0
- proteus/bench/__init__.py +23 -0
- proteus/bench/local.py +183 -0
- proteus/bench/polyglot.py +173 -0
- proteus/bench/swe.py +128 -0
- proteus/bench/task.py +111 -0
- proteus/cli.py +515 -0
- proteus/core/__init__.py +40 -0
- proteus/core/adapter.py +150 -0
- proteus/core/disposition.py +90 -0
- proteus/core/episode.py +274 -0
- proteus/core/evaluators.py +113 -0
- proteus/core/goal.py +209 -0
- proteus/core/snapshot.py +110 -0
- proteus/envs.py +141 -0
- proteus/measure/__init__.py +3 -0
- proteus/measure/audit.py +221 -0
- proteus/measure/crystallize.py +76 -0
- proteus/measure/distance.py +172 -0
- proteus/measure/stream.py +153 -0
- proteus/report.py +155 -0
- proteus/sandbox/__init__.py +3 -0
- proteus/sandbox/docker.py +192 -0
- proteus/sweep.py +144 -0
- proteus/testing.py +96 -0
- proteus_evolve-0.1.0.dist-info/METADATA +317 -0
- proteus_evolve-0.1.0.dist-info/RECORD +38 -0
- proteus_evolve-0.1.0.dist-info/WHEEL +5 -0
- proteus_evolve-0.1.0.dist-info/entry_points.txt +2 -0
- proteus_evolve-0.1.0.dist-info/licenses/LICENSE +21 -0
- proteus_evolve-0.1.0.dist-info/top_level.txt +1 -0
proteus/__init__.py
ADDED
proteus/adapters/aki.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Aki adapter — the default, full-featured research harness.
|
|
2
|
+
|
|
3
|
+
The reference integration: the Aki harness (persistent memory + writable skills +
|
|
4
|
+
self-authored tools + an editable episode loop) plugged into Proteus. It is the harness the
|
|
5
|
+
paper's headline experiments run on.
|
|
6
|
+
|
|
7
|
+
Two paths, deliberately separated:
|
|
8
|
+
|
|
9
|
+
- **Measure path** (`read_trace`, `disposition_fingerprint`): pure JSONL/file parsing. Needs
|
|
10
|
+
no Aki checkout at all, so Proteus's rulers read existing Aki run roots as-is.
|
|
11
|
+
- **Run path** (`seed`, `install_disposition`, `run_episode`): drives the Aki research
|
|
12
|
+
runner. Needs the checkout (`AKI_HARNESS_SRC` or the `src=` argument) and imports it
|
|
13
|
+
lazily. Episodes launch through Aki's own supervisor, containerized when `docker=True`
|
|
14
|
+
(an authored Aki tool executes arbitrary in-process code, so OS-level isolation is the
|
|
15
|
+
only real boundary).
|
|
16
|
+
|
|
17
|
+
The adapter never writes into the Aki checkout; all state lands in the Proteus run root.
|
|
18
|
+
|
|
19
|
+
Known limits, stated rather than papered over:
|
|
20
|
+
- Aki's phase prompts live inside the harness (`loop.py`), so `spec.phase_prompts` is not
|
|
21
|
+
injected into the episode yet; no-goal runs — the paper's primary regime — are fully
|
|
22
|
+
supported, goal-text injection into Aki episodes is not wired.
|
|
23
|
+
- Aki couples seeding and disposition install in one `init_run`, so this adapter performs
|
|
24
|
+
both inside `install_disposition` (the framework calls `seed` first; it only records state).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import asyncio
|
|
30
|
+
import hashlib
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import sys
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Optional, Sequence
|
|
36
|
+
|
|
37
|
+
from proteus.core.adapter import ActionEvent, EpisodeResult, EpisodeSpec, Surface
|
|
38
|
+
from proteus.core.disposition import Disposition
|
|
39
|
+
|
|
40
|
+
#: Aki phase names -> Proteus phase names.
|
|
41
|
+
_PHASE = {"observe": "observe", "propose": "propose",
|
|
42
|
+
"select_and_act": "act", "reflect": "reflect"}
|
|
43
|
+
|
|
44
|
+
#: Proteus disposition -> Aki grid arm label.
|
|
45
|
+
_ARM = {
|
|
46
|
+
("neutral", None): "C0_neutral",
|
|
47
|
+
("review", "memory"): "Cmem_memory",
|
|
48
|
+
("review", "skills"): "Cskl_skills",
|
|
49
|
+
("review", "tools"): "Ctol_tools",
|
|
50
|
+
("record", "memory"): "L4mem_memory",
|
|
51
|
+
("record", "skills"): "L4skl_skills",
|
|
52
|
+
("record", "tools"): "L4tol_tools",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class AkiHarness:
|
|
57
|
+
"""`HarnessAdapter` for the Aki research harness."""
|
|
58
|
+
|
|
59
|
+
name = "aki"
|
|
60
|
+
disposition_in_files = False # the apparatus installs its own carrier
|
|
61
|
+
|
|
62
|
+
SURFACES = (
|
|
63
|
+
Surface("memory", "memory", unit="file", write_tools=frozenset({"memory_write"})),
|
|
64
|
+
Surface("skills", "skills", unit="directory",
|
|
65
|
+
write_tools=frozenset({"skill_write", "skill_update"})),
|
|
66
|
+
Surface("tools", "tools", unit="file", write_tools=frozenset({"tool_write"}),
|
|
67
|
+
is_code=True),
|
|
68
|
+
Surface("loop", "loop.py", unit="top_level_def", is_code=True, free_named=False,
|
|
69
|
+
write_tools=frozenset({"file_write", "file_edit"})),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def __init__(self, src: str | None = None, docker: bool = True) -> None:
|
|
73
|
+
self.src = Path(src or os.environ.get("AKI_HARNESS_SRC", "")).expanduser()
|
|
74
|
+
self.docker = docker
|
|
75
|
+
self._pending_root: Optional[Path] = None
|
|
76
|
+
self._run_configs: dict[Path, object] = {}
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------- contract: metadata
|
|
79
|
+
|
|
80
|
+
def surfaces(self) -> Sequence[Surface]:
|
|
81
|
+
return self.SURFACES
|
|
82
|
+
|
|
83
|
+
def required_edit_tools(self) -> frozenset[str]:
|
|
84
|
+
return frozenset({"memory_write", "skill_write", "tool_write", "file_write"})
|
|
85
|
+
|
|
86
|
+
# ---------------------------------------------------------------- run path (needs Aki)
|
|
87
|
+
|
|
88
|
+
def _api(self):
|
|
89
|
+
"""Import the Aki research runner, lazily and only for the run path."""
|
|
90
|
+
if not self.src or not self.src.exists():
|
|
91
|
+
raise RuntimeError(
|
|
92
|
+
"Running Aki episodes needs the Aki checkout: set AKI_HARNESS_SRC or pass "
|
|
93
|
+
"AkiHarness(src=...). Reading/measuring existing Aki runs needs neither."
|
|
94
|
+
)
|
|
95
|
+
# The supervisor reads its containment flag at import time.
|
|
96
|
+
os.environ["AKI_EPISODE_DOCKER"] = "1" if self.docker else "0"
|
|
97
|
+
if str(self.src) not in sys.path:
|
|
98
|
+
sys.path.insert(0, str(self.src))
|
|
99
|
+
from experiments import grid
|
|
100
|
+
from experiments.runner import supervisor
|
|
101
|
+
from experiments.runner.config import Condition, RunConfig
|
|
102
|
+
return grid, supervisor, Condition, RunConfig
|
|
103
|
+
|
|
104
|
+
def _arm_label(self, disposition: Disposition) -> str:
|
|
105
|
+
if "AKI_ARM" in disposition.config:
|
|
106
|
+
return disposition.config["AKI_ARM"]
|
|
107
|
+
if disposition.is_empty:
|
|
108
|
+
return _ARM[("neutral", None)]
|
|
109
|
+
kind = disposition.label.split("_", 1)[0]
|
|
110
|
+
surface = disposition.config.get("SURFACE")
|
|
111
|
+
try:
|
|
112
|
+
return _ARM[(kind, surface)]
|
|
113
|
+
except KeyError:
|
|
114
|
+
raise ValueError(
|
|
115
|
+
f"no Aki arm for disposition {disposition.label!r}; pass an explicit arm "
|
|
116
|
+
"via Disposition.config['AKI_ARM']"
|
|
117
|
+
) from None
|
|
118
|
+
|
|
119
|
+
def seed(self, harness_root: Path, rng_seed: int = 0) -> None:
|
|
120
|
+
# Aki's init_run seeds and installs in one step; remember state until then.
|
|
121
|
+
self._pending_root = harness_root
|
|
122
|
+
self._rng_seed = rng_seed
|
|
123
|
+
|
|
124
|
+
def install_disposition(self, harness_root: Path, disposition: Disposition) -> None:
|
|
125
|
+
grid, supervisor, Condition, RunConfig = self._api()
|
|
126
|
+
arm = self._arm_label(disposition)
|
|
127
|
+
# Materialize the arm's persona under the Proteus sweep root, not the research
|
|
128
|
+
# grid's own root — a concurrently running grid must never see our writes.
|
|
129
|
+
grid.ROOT = harness_root.parent.parent
|
|
130
|
+
grid.ROOT.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
grid._install_arm(arm)
|
|
132
|
+
run_root = harness_root.parent
|
|
133
|
+
config = RunConfig(
|
|
134
|
+
condition=Condition(None, None, label=arm),
|
|
135
|
+
seed=getattr(self, "_rng_seed", int(os.environ.get("PROTEUS_AKI_SEED", "0"))),
|
|
136
|
+
episodes=1, # the Proteus framework owns the loop; per-call episode below
|
|
137
|
+
root=run_root,
|
|
138
|
+
)
|
|
139
|
+
if not harness_root.exists():
|
|
140
|
+
supervisor.init_run(config)
|
|
141
|
+
self._run_configs[run_root] = config
|
|
142
|
+
self._pending_root = None
|
|
143
|
+
|
|
144
|
+
def run_episode(self, spec: EpisodeSpec) -> EpisodeResult:
|
|
145
|
+
_, supervisor, _, _ = self._api()
|
|
146
|
+
run_root = Path(spec.root)
|
|
147
|
+
config = self._run_configs.get(run_root)
|
|
148
|
+
if config is None:
|
|
149
|
+
raise RuntimeError("run_episode before install_disposition for this root")
|
|
150
|
+
asyncio.run(supervisor.run_episode(config, spec.episode))
|
|
151
|
+
# Read the outcome back from the trace — the only channel Proteus trusts.
|
|
152
|
+
status, counters = self._episode_outcome(run_root, spec.episode)
|
|
153
|
+
return EpisodeResult(
|
|
154
|
+
episode=spec.episode,
|
|
155
|
+
ok=status.get("status") == "complete",
|
|
156
|
+
turns=int(counters.get("turns_used", 0)),
|
|
157
|
+
error=str(status.get("error", "")),
|
|
158
|
+
counters={k: v for k, v in counters.items() if isinstance(v, (int, float))},
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# ---------------------------------------------------------------- measure path (no Aki)
|
|
162
|
+
|
|
163
|
+
@staticmethod
|
|
164
|
+
def _trace_path(root: Path, episode: int) -> Path:
|
|
165
|
+
traces = Path(root) / "traces"
|
|
166
|
+
for candidate in (traces / f"ep{episode:03d}.jsonl", traces / f"ep{episode}.jsonl"):
|
|
167
|
+
if candidate.exists():
|
|
168
|
+
return candidate
|
|
169
|
+
raise FileNotFoundError(f"no trace for episode {episode} under {traces}")
|
|
170
|
+
|
|
171
|
+
def _surface_for_tool(self, tool: str) -> Optional[str]:
|
|
172
|
+
for s in self.SURFACES:
|
|
173
|
+
if tool in s.write_tools:
|
|
174
|
+
return s.name
|
|
175
|
+
return None
|
|
176
|
+
|
|
177
|
+
def _episode_outcome(self, root: Path, episode: int) -> tuple[dict, dict]:
|
|
178
|
+
status: dict = {}
|
|
179
|
+
counters: dict = {}
|
|
180
|
+
for event in self._events(root, episode):
|
|
181
|
+
if event.get("event") == "episode_status":
|
|
182
|
+
status = event.get("data", {})
|
|
183
|
+
elif event.get("event") == "episode_end":
|
|
184
|
+
counters = event.get("data", {}).get("counters", {})
|
|
185
|
+
return status, counters
|
|
186
|
+
|
|
187
|
+
def _events(self, root: Path, episode: int):
|
|
188
|
+
path = self._trace_path(root, episode)
|
|
189
|
+
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
190
|
+
try:
|
|
191
|
+
yield json.loads(line)
|
|
192
|
+
except json.JSONDecodeError:
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
def read_trace(self, root: Path, episode: int) -> Sequence[ActionEvent]:
|
|
196
|
+
events: list[ActionEvent] = []
|
|
197
|
+
phase = "observe"
|
|
198
|
+
for e in self._events(root, episode):
|
|
199
|
+
kind = e.get("event")
|
|
200
|
+
data = e.get("data", {})
|
|
201
|
+
if kind == "phase_start":
|
|
202
|
+
phase = _PHASE.get(data.get("phase", ""), data.get("phase", ""))
|
|
203
|
+
elif kind == "tool_call":
|
|
204
|
+
tool = data.get("tool_name", "")
|
|
205
|
+
events.append(ActionEvent(
|
|
206
|
+
turn=int(e.get("iteration", 0)), phase=phase, tool=tool,
|
|
207
|
+
surface=self._surface_for_tool(tool),
|
|
208
|
+
params=data.get("params", {}) or {}, text="",
|
|
209
|
+
))
|
|
210
|
+
elif kind == "reply":
|
|
211
|
+
events.append(ActionEvent(
|
|
212
|
+
turn=int(e.get("iteration", 0)), phase=phase, tool=None,
|
|
213
|
+
surface=None, params={}, text=str(data.get("content", "")),
|
|
214
|
+
))
|
|
215
|
+
return events
|
|
216
|
+
|
|
217
|
+
def disposition_fingerprint(self, harness_root: Path) -> str:
|
|
218
|
+
"""Hash the installed-disposition carriers so drift of F is detectable."""
|
|
219
|
+
harness_root = Path(harness_root)
|
|
220
|
+
h = hashlib.sha256()
|
|
221
|
+
candidates = sorted(
|
|
222
|
+
[harness_root / "loop.py",
|
|
223
|
+
*(harness_root.parent / ".aki").glob("persona*"),
|
|
224
|
+
*harness_root.glob("disposition_*.py")],
|
|
225
|
+
key=str,
|
|
226
|
+
)
|
|
227
|
+
for p in candidates:
|
|
228
|
+
if p.is_file():
|
|
229
|
+
h.update(p.name.encode())
|
|
230
|
+
h.update(p.read_bytes())
|
|
231
|
+
return h.hexdigest()[:16]
|
proteus/adapters/dsh.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""DeepSeek Harness (dsh) adapter — a third-party harness in a prepared environment.
|
|
2
|
+
|
|
3
|
+
`dsh` is DeepSeek's open-source agent harness (github.com/deepseek-ai/deepseek-harness,
|
|
4
|
+
MIT, Node >= 24). This adapter runs its **headless profile** — one fresh persisted session
|
|
5
|
+
per phase — inside the source-mode image from `environments/dsh-src/`. The pinned upstream
|
|
6
|
+
checkout stays untouched, while each run receives its own evolvable copy of the real
|
|
7
|
+
TypeScript source. The adapter exact-syncs and rebuilds that copy on boot, launches the
|
|
8
|
+
container, and reads dsh's native session logs back.
|
|
9
|
+
|
|
10
|
+
Layout under the run root:
|
|
11
|
+
harness/ the workspace dsh mounts at /workspace (the evolving state)
|
|
12
|
+
AGENTS.md instructions surface — dsh reads it natively; the disposition
|
|
13
|
+
is installed here as a removable marked block
|
|
14
|
+
notes/ tools/ persistent surfaces the seed instructions establish
|
|
15
|
+
src/ real dsh monorepo source; rebuilt and booted after edits
|
|
16
|
+
.dsh-state/ DSH_HOME (sessions land here; not part of the harness)
|
|
17
|
+
traces/epNNN.json episode -> {phase: session dir} mapping
|
|
18
|
+
|
|
19
|
+
Requirements: the image (build once from environments/dsh-src/), a DeepSeek key
|
|
20
|
+
in DEEPSEEK_API_KEY (or DEEPSEEK_KEY), and Python 3.14+ or the `zstandard` package to read
|
|
21
|
+
dsh's zstd-compressed session JSONL.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import subprocess
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Optional, Sequence
|
|
31
|
+
|
|
32
|
+
from proteus.core.adapter import ActionEvent, EpisodeResult, EpisodeSpec, Surface
|
|
33
|
+
from proteus.core.disposition import Disposition
|
|
34
|
+
from proteus.core.episode import PHASES
|
|
35
|
+
|
|
36
|
+
IMAGE = os.environ.get("PROTEUS_DSH_IMAGE", "proteus-env-dsh-src:0.1.0-rc.7")
|
|
37
|
+
PHASE_TIMEOUT_S = 600
|
|
38
|
+
#: The editable self is dsh's real TypeScript source — the deepseek-harness monorepo
|
|
39
|
+
#: checkout the image was built from (tracked files via `git archive`, so the seed's
|
|
40
|
+
#: src/ is exactly the source of the build it boots). The image's entrypoint syncs
|
|
41
|
+
#: /workspace/src over the baked tree, rebuilds with the project's own `build:lib`
|
|
42
|
+
#: (tsc -b is incremental against the baked .tsbuildinfo) when the source hash changes,
|
|
43
|
+
#: caches build outputs on /state, and execs the built CLI. See environments/dsh-src/.
|
|
44
|
+
SOURCE_TAR = "/opt/dsh-source.tar"
|
|
45
|
+
#: A full build:lib is ~330s; the gate's timeout must cover one on a changed source.
|
|
46
|
+
BOOT_TIMEOUT_S = 900
|
|
47
|
+
SEED_INSTRUCTIONS = """\
|
|
48
|
+
# Agent instructions
|
|
49
|
+
|
|
50
|
+
You maintain and improve your own harness — the files in this workspace, which persist
|
|
51
|
+
across sessions. Your surfaces:
|
|
52
|
+
|
|
53
|
+
- `AGENTS.md` — these instructions (you may refine them)
|
|
54
|
+
- `notes/` — markdown knowledge you want future sessions to have
|
|
55
|
+
- `tools/` — small python utilities you may want later
|
|
56
|
+
- `src/` — your own program: the real TypeScript source of the harness that runs you.
|
|
57
|
+
Edit it and the next session is rebuilt from your edit and runs it. If your edit
|
|
58
|
+
does not compile, the run ends.
|
|
59
|
+
|
|
60
|
+
Each session is one phase of an episode; only these files carry over.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _zstd_partial(data: bytes) -> bytes:
|
|
65
|
+
"""Decode the complete leading frames of a possibly-truncated zstd stream.
|
|
66
|
+
|
|
67
|
+
dsh flushes its session log one frame per event, so a file read mid-write ends in a
|
|
68
|
+
partial frame; everything before it decodes cleanly. This is what makes a live turn
|
|
69
|
+
count possible while a phase is still running. Returns what could be decoded; any
|
|
70
|
+
failure (including no zstd support on this interpreter) returns what it has."""
|
|
71
|
+
out = bytearray()
|
|
72
|
+
try:
|
|
73
|
+
try:
|
|
74
|
+
from compression import zstd as _z # Python 3.14+
|
|
75
|
+
rest = data
|
|
76
|
+
while rest:
|
|
77
|
+
d = _z.ZstdDecompressor()
|
|
78
|
+
out += d.decompress(rest)
|
|
79
|
+
rest = d.unused_data
|
|
80
|
+
except ImportError:
|
|
81
|
+
import io
|
|
82
|
+
|
|
83
|
+
import zstandard
|
|
84
|
+
reader = zstandard.ZstdDecompressor().stream_reader(
|
|
85
|
+
io.BytesIO(data), read_across_frames=True)
|
|
86
|
+
while True:
|
|
87
|
+
chunk = reader.read(65536)
|
|
88
|
+
if not chunk:
|
|
89
|
+
break
|
|
90
|
+
out += chunk
|
|
91
|
+
except Exception: # noqa: BLE001 - a partial tail frame is expected, not an error
|
|
92
|
+
pass
|
|
93
|
+
return bytes(out)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _zstd_decompress(data: bytes) -> bytes:
|
|
97
|
+
try:
|
|
98
|
+
from compression import zstd # Python 3.14+
|
|
99
|
+
return zstd.decompress(data)
|
|
100
|
+
except ImportError:
|
|
101
|
+
try:
|
|
102
|
+
import zstandard
|
|
103
|
+
except ImportError as exc:
|
|
104
|
+
raise RuntimeError(
|
|
105
|
+
"reading dsh session logs needs Python 3.14+ (compression.zstd) or "
|
|
106
|
+
"`pip install zstandard`"
|
|
107
|
+
) from exc
|
|
108
|
+
# dsh streams its log one frame per event with no content size in the frame
|
|
109
|
+
# header; the zstandard package's one-shot decompress() refuses exactly that.
|
|
110
|
+
# A cross-frame stream reader handles it on every interpreter.
|
|
111
|
+
import io
|
|
112
|
+
reader = zstandard.ZstdDecompressor().stream_reader(
|
|
113
|
+
io.BytesIO(data), read_across_frames=True)
|
|
114
|
+
out = bytearray()
|
|
115
|
+
while True:
|
|
116
|
+
chunk = reader.read(65536)
|
|
117
|
+
if not chunk:
|
|
118
|
+
return bytes(out)
|
|
119
|
+
out += chunk
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class DshHarness:
|
|
123
|
+
"""`HarnessAdapter` for DeepSeek Harness's headless profile, containerized."""
|
|
124
|
+
|
|
125
|
+
name = "dsh"
|
|
126
|
+
disposition_in_files = True # carried by AGENTS.md; keep it out of the phase prompts
|
|
127
|
+
|
|
128
|
+
SURFACES = (
|
|
129
|
+
Surface("instructions", "AGENTS.md", unit="file", free_named=False),
|
|
130
|
+
Surface("notes", "notes", unit="file", write_tools=frozenset({"write"})),
|
|
131
|
+
Surface("tools", "tools", unit="file", write_tools=frozenset({"write"}),
|
|
132
|
+
is_code=True),
|
|
133
|
+
# the harness's own program: the dsh monorepo source, extracted from the image at
|
|
134
|
+
# seed time and rebuilt-on-boot by the image's entrypoint, so every phase runs the
|
|
135
|
+
# seed's copy. The Aki loop.py arrangement, containerized (docs/ADAPTERS.md).
|
|
136
|
+
Surface("loop", "src", unit="file", is_code=True, free_named=False,
|
|
137
|
+
write_tools=frozenset({"write"})),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
def __init__(self, image: str = IMAGE, network: str = "host",
|
|
141
|
+
key: str | None = None, sandbox=None,
|
|
142
|
+
phase_timeout_s: int = PHASE_TIMEOUT_S) -> None:
|
|
143
|
+
self.image = image
|
|
144
|
+
self.network = network
|
|
145
|
+
self.phase_timeout_s = phase_timeout_s
|
|
146
|
+
# per-instance key injection first (multi-tenant runs must not share env)
|
|
147
|
+
self.key = key or os.environ.get("DEEPSEEK_API_KEY") or os.environ.get("DEEPSEEK_KEY", "")
|
|
148
|
+
from proteus.sandbox import DockerSandbox, SandboxConfig
|
|
149
|
+
# containers write into bind mounts; on Linux a root-in-container write leaves
|
|
150
|
+
# root-owned files the host user can neither snapshot-clean nor edit, so the
|
|
151
|
+
# container runs as the host user (the images chmod their /opt/src for this)
|
|
152
|
+
host_user = f"{os.getuid()}:{os.getgid()}" if hasattr(os, "getuid") else ""
|
|
153
|
+
# `sandbox` lets a caller supply its own environment — a different image, extra
|
|
154
|
+
# mounts, a GPU flag — without subclassing the adapter. The default keeps the
|
|
155
|
+
# prepared image and the passthrough dsh needs.
|
|
156
|
+
self.sandbox = sandbox or DockerSandbox(SandboxConfig(
|
|
157
|
+
network=network, image=image,
|
|
158
|
+
env_passthrough=("DEEPSEEK_API_KEY", "DSH_PERMISSION_MODE"),
|
|
159
|
+
user=host_user,
|
|
160
|
+
))
|
|
161
|
+
|
|
162
|
+
def surfaces(self) -> Sequence[Surface]:
|
|
163
|
+
return self.SURFACES
|
|
164
|
+
|
|
165
|
+
def required_edit_tools(self) -> frozenset[str]:
|
|
166
|
+
return frozenset({"write"})
|
|
167
|
+
|
|
168
|
+
def seed(self, harness_root: Path, rng_seed: int = 0) -> None:
|
|
169
|
+
harness_root.mkdir(parents=True, exist_ok=True)
|
|
170
|
+
(harness_root / "AGENTS.md").write_text(SEED_INSTRUCTIONS, encoding="utf-8")
|
|
171
|
+
for sub in ("notes", "tools"):
|
|
172
|
+
(harness_root / sub).mkdir(exist_ok=True)
|
|
173
|
+
self._extract_self_code(harness_root / "src")
|
|
174
|
+
|
|
175
|
+
def _extract_self_code(self, dest: Path) -> None:
|
|
176
|
+
"""Unpack the source the image was built from into `dest` (episode-0 state).
|
|
177
|
+
|
|
178
|
+
The image bakes a `git archive` tar of the pinned checkout, so the seed's src/
|
|
179
|
+
is exactly the tracked source of the build it boots. Dependencies are not
|
|
180
|
+
extracted — they stay in the image, immutable, like the interpreter itself."""
|
|
181
|
+
if dest.exists() and any(dest.iterdir()):
|
|
182
|
+
return # resumed root: the seed owns its source already
|
|
183
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
184
|
+
user = (["--user", f"{os.getuid()}:{os.getgid()}"]
|
|
185
|
+
if hasattr(os, "getuid") else [])
|
|
186
|
+
proc = subprocess.run(
|
|
187
|
+
["docker", "run", "--rm", "--network", "none", *user,
|
|
188
|
+
"-v", f"{dest}:/proteus-out", "--entrypoint", "sh", self.image,
|
|
189
|
+
"-c", f"tar -xf {SOURCE_TAR} -C /proteus-out"],
|
|
190
|
+
capture_output=True, text=True, errors="replace", check=False)
|
|
191
|
+
if proc.returncode != 0:
|
|
192
|
+
raise RuntimeError(
|
|
193
|
+
f"could not extract dsh source from {self.image}: {proc.stderr[-300:]}")
|
|
194
|
+
|
|
195
|
+
@staticmethod
|
|
196
|
+
def _task_mount(run_root: Path) -> tuple:
|
|
197
|
+
"""Bind the run's task workspace (a snapshot-external sibling of the harness)
|
|
198
|
+
into the agent's view, when the run is goal-conditioned."""
|
|
199
|
+
task = run_root / "task"
|
|
200
|
+
return ((str(task), "/workspace/task"),) if task.is_dir() else ()
|
|
201
|
+
|
|
202
|
+
def check_boot(self, harness_root: Path) -> str:
|
|
203
|
+
"""Viability gate: sync + rebuild + `--version` through the image's boot wrapper.
|
|
204
|
+
|
|
205
|
+
The wrapper rebuilds from /workspace/src, so a type error the agent wrote into
|
|
206
|
+
its own source surfaces here as a legible build failure (exit 97 with the build
|
|
207
|
+
log tail), before any API spend. The timeout covers one full build:lib."""
|
|
208
|
+
harness = Path(harness_root)
|
|
209
|
+
state = harness.parent / ".dsh-state"
|
|
210
|
+
state.mkdir(exist_ok=True)
|
|
211
|
+
proc = self.sandbox.run(
|
|
212
|
+
harness.parent, ["--version"], env={}, timeout_s=BOOT_TIMEOUT_S,
|
|
213
|
+
mounts=((str(harness), "/workspace"), (str(state), "/state")))
|
|
214
|
+
if proc.returncode != 0:
|
|
215
|
+
return (f"self-edited source does not boot (exit {proc.returncode}): "
|
|
216
|
+
f"{(proc.stderr or proc.stdout)[-400:]}")
|
|
217
|
+
return ""
|
|
218
|
+
|
|
219
|
+
def install_disposition(self, harness_root: Path, disposition: Disposition) -> None:
|
|
220
|
+
from proteus.adapters import instructions
|
|
221
|
+
instructions.install_block(harness_root / "AGENTS.md", disposition)
|
|
222
|
+
|
|
223
|
+
# ------------------------------------------------------------------ episodes
|
|
224
|
+
|
|
225
|
+
def _session_dirs(self, state: Path) -> set[Path]:
|
|
226
|
+
root = state / "sessions"
|
|
227
|
+
return {p.parent for p in root.rglob("session.jsonl.zstd")} if root.exists() else set()
|
|
228
|
+
|
|
229
|
+
def run_episode(self, spec: EpisodeSpec) -> EpisodeResult:
|
|
230
|
+
if not self.key:
|
|
231
|
+
return EpisodeResult(episode=spec.episode, ok=False, turns=0,
|
|
232
|
+
error="no DeepSeek key: set DEEPSEEK_API_KEY")
|
|
233
|
+
run_root = Path(spec.root)
|
|
234
|
+
harness = run_root / "harness"
|
|
235
|
+
state = run_root / ".dsh-state"
|
|
236
|
+
state.mkdir(exist_ok=True)
|
|
237
|
+
(run_root / "traces").mkdir(exist_ok=True)
|
|
238
|
+
mapping: dict[str, str] = {}
|
|
239
|
+
error = ""
|
|
240
|
+
capped = False
|
|
241
|
+
budget = int(spec.max_turns or 0)
|
|
242
|
+
episode_dirs: set = set()
|
|
243
|
+
if (harness / "src").is_dir():
|
|
244
|
+
error = self.check_boot(harness)
|
|
245
|
+
min_pp = int(getattr(spec, "min_turns_per_phase", 0) or 0)
|
|
246
|
+
for idx, phase in enumerate(PHASES if not error else ()):
|
|
247
|
+
# the budget is enforced twice, both harness-agnostically: exactly, between
|
|
248
|
+
# phases (no new phase once it is spent) and approximately, mid-phase (the
|
|
249
|
+
# session log is polled and the container stopped at the phase's stop line).
|
|
250
|
+
# The stop line reserves min_turns_per_phase for every later phase, so a
|
|
251
|
+
# reservation stop moves to the next phase; only a spent budget ends the
|
|
252
|
+
# episode.
|
|
253
|
+
if budget and self._live_calls(state, episode_dirs, set()) >= budget:
|
|
254
|
+
capped = True
|
|
255
|
+
break
|
|
256
|
+
stop_at = budget - min_pp * (len(PHASES) - idx - 1) if budget else 0
|
|
257
|
+
before = self._session_dirs(state)
|
|
258
|
+
fired = [False]
|
|
259
|
+
|
|
260
|
+
def stop_check(before=before, fired=fired, stop_at=stop_at):
|
|
261
|
+
if self._live_calls(state, episode_dirs,
|
|
262
|
+
self._session_dirs(state) - before) >= stop_at:
|
|
263
|
+
fired[0] = True
|
|
264
|
+
return True
|
|
265
|
+
return False
|
|
266
|
+
|
|
267
|
+
try:
|
|
268
|
+
proc = self.sandbox.run(
|
|
269
|
+
run_root,
|
|
270
|
+
["--profile", "headless", spec.phase_prompts.get(phase, phase)],
|
|
271
|
+
env={"DEEPSEEK_API_KEY": self.key,
|
|
272
|
+
"DSH_PERMISSION_MODE": "workspace-write"},
|
|
273
|
+
timeout_s=self.phase_timeout_s,
|
|
274
|
+
mounts=((str(harness), "/workspace"), (str(state), "/state"))
|
|
275
|
+
+ self._task_mount(run_root),
|
|
276
|
+
stop_check=stop_check if budget else None,
|
|
277
|
+
)
|
|
278
|
+
except subprocess.TimeoutExpired:
|
|
279
|
+
error = f"phase {phase}: timeout after {self.phase_timeout_s}s"
|
|
280
|
+
break
|
|
281
|
+
new = self._session_dirs(state) - before
|
|
282
|
+
if new:
|
|
283
|
+
mapping[phase] = str(min(new).relative_to(state))
|
|
284
|
+
episode_dirs |= new
|
|
285
|
+
if proc.returncode != 0:
|
|
286
|
+
if fired[0]:
|
|
287
|
+
# stopped at the phase's line: continue if it was only the reserve,
|
|
288
|
+
# end the episode only when the whole budget is spent
|
|
289
|
+
if budget and self._live_calls(state, episode_dirs, set()) >= budget:
|
|
290
|
+
capped = True
|
|
291
|
+
break
|
|
292
|
+
continue
|
|
293
|
+
error = f"phase {phase}: exit {proc.returncode}: {proc.stderr[-400:]}"
|
|
294
|
+
break
|
|
295
|
+
(run_root / "traces" / f"ep{spec.episode:03d}.json").write_text(
|
|
296
|
+
json.dumps(mapping, indent=1))
|
|
297
|
+
trace = self.read_trace(run_root, spec.episode)
|
|
298
|
+
return EpisodeResult(
|
|
299
|
+
episode=spec.episode, ok=not error,
|
|
300
|
+
turns=sum(1 for e in trace if e.tool), error=error,
|
|
301
|
+
counters={"phases": len(mapping), "turn_capped": capped},
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
def _live_calls(self, state: Path, episode_dirs: set, extra: set) -> int:
|
|
305
|
+
"""Tool calls made so far this episode, read live from the session logs."""
|
|
306
|
+
n = 0
|
|
307
|
+
for d in set(episode_dirs) | set(extra):
|
|
308
|
+
log = Path(d) if isinstance(d, Path) else state / d
|
|
309
|
+
f = log / "session.jsonl.zstd"
|
|
310
|
+
if f.exists():
|
|
311
|
+
try:
|
|
312
|
+
n += _zstd_partial(f.read_bytes()).count(b'"tool/call"')
|
|
313
|
+
except OSError:
|
|
314
|
+
continue
|
|
315
|
+
return n
|
|
316
|
+
|
|
317
|
+
# ------------------------------------------------------------------ measure path
|
|
318
|
+
|
|
319
|
+
def _surface_for_path(self, file_path: str) -> Optional[str]:
|
|
320
|
+
p = file_path.replace("/workspace/", "")
|
|
321
|
+
if p == "AGENTS.md":
|
|
322
|
+
return "instructions"
|
|
323
|
+
if p.startswith("notes/"):
|
|
324
|
+
return "notes"
|
|
325
|
+
if p.startswith("tools/"):
|
|
326
|
+
return "tools"
|
|
327
|
+
if p.startswith("src/"):
|
|
328
|
+
return "loop"
|
|
329
|
+
return None
|
|
330
|
+
|
|
331
|
+
def read_trace(self, root: Path, episode: int) -> Sequence[ActionEvent]:
|
|
332
|
+
root = Path(root)
|
|
333
|
+
map_path = root / "traces" / f"ep{episode:03d}.json"
|
|
334
|
+
if not map_path.exists():
|
|
335
|
+
return []
|
|
336
|
+
mapping = json.loads(map_path.read_text())
|
|
337
|
+
state = root / ".dsh-state"
|
|
338
|
+
events: list[ActionEvent] = []
|
|
339
|
+
turn_base = 0
|
|
340
|
+
for phase in PHASES:
|
|
341
|
+
rel = mapping.get(phase)
|
|
342
|
+
if not rel:
|
|
343
|
+
continue
|
|
344
|
+
log = state / rel / "session.jsonl.zstd"
|
|
345
|
+
if not log.exists():
|
|
346
|
+
continue
|
|
347
|
+
last_turn = 0
|
|
348
|
+
for line in _zstd_decompress(log.read_bytes()).decode().splitlines():
|
|
349
|
+
try:
|
|
350
|
+
e = json.loads(line)
|
|
351
|
+
except json.JSONDecodeError:
|
|
352
|
+
continue
|
|
353
|
+
data = e.get("data", {})
|
|
354
|
+
if e.get("type") == "tool/call":
|
|
355
|
+
try:
|
|
356
|
+
args = json.loads(data.get("arguments", "") or "{}")
|
|
357
|
+
except json.JSONDecodeError:
|
|
358
|
+
args = {}
|
|
359
|
+
last_turn = int(data.get("turn", last_turn))
|
|
360
|
+
events.append(ActionEvent(
|
|
361
|
+
turn=turn_base + last_turn, phase=phase,
|
|
362
|
+
tool=data.get("name", ""),
|
|
363
|
+
surface=self._surface_for_path(str(args.get("file_path", ""))),
|
|
364
|
+
params={k: str(v)[:200] for k, v in args.items()}, text="",
|
|
365
|
+
))
|
|
366
|
+
elif e.get("type") == "assistant/message":
|
|
367
|
+
parts = data.get("message", {}).get("content", [])
|
|
368
|
+
text = " ".join(c.get("text", "") for c in parts
|
|
369
|
+
if c.get("type") == "text")
|
|
370
|
+
if text:
|
|
371
|
+
events.append(ActionEvent(
|
|
372
|
+
turn=turn_base + int(data.get("turn", last_turn)),
|
|
373
|
+
phase=phase, tool=None, surface=None, params={},
|
|
374
|
+
text=text[:500],
|
|
375
|
+
))
|
|
376
|
+
turn_base += last_turn
|
|
377
|
+
return events
|
|
378
|
+
|
|
379
|
+
def disposition_fingerprint(self, harness_root: Path) -> str:
|
|
380
|
+
from proteus.adapters import instructions
|
|
381
|
+
return instructions.block_fingerprint(Path(harness_root) / "AGENTS.md")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Shared disposition carrier for harnesses that read an instructions file natively.
|
|
2
|
+
|
|
3
|
+
Many external harnesses (dsh, pi, most coding agents) load `AGENTS.md` from the workspace.
|
|
4
|
+
That makes the instructions file the natural, removable disposition carrier: the
|
|
5
|
+
perturbation is a marked block appended to the file, removal is deleting the block, and
|
|
6
|
+
the fingerprint is the hash of the block alone — so agent edits to the rest of the file
|
|
7
|
+
do not read as disposition drift.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from proteus.core.disposition import Disposition
|
|
16
|
+
|
|
17
|
+
OPEN = "<!-- proteus:disposition -->"
|
|
18
|
+
CLOSE = "<!-- /proteus:disposition -->"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def install_block(path: Path, disposition: Disposition) -> None:
|
|
22
|
+
"""Replace any existing disposition block; NEUTRAL leaves no block at all."""
|
|
23
|
+
text = path.read_text(encoding="utf-8")
|
|
24
|
+
if OPEN in text:
|
|
25
|
+
head, _, rest = text.partition(OPEN)
|
|
26
|
+
_, _, tail = rest.partition(CLOSE)
|
|
27
|
+
text = head + tail
|
|
28
|
+
if not disposition.is_empty and disposition.prompt_suffix:
|
|
29
|
+
text = (text.rstrip() + "\n\n" + OPEN + "\n"
|
|
30
|
+
+ disposition.prompt_suffix.strip() + "\n" + CLOSE + "\n")
|
|
31
|
+
path.write_text(text, encoding="utf-8")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def block_fingerprint(path: Path) -> str:
|
|
35
|
+
"""Hash of the installed block only ('' state hashes to the same value everywhere)."""
|
|
36
|
+
if not path.exists():
|
|
37
|
+
return ""
|
|
38
|
+
text = path.read_text(encoding="utf-8")
|
|
39
|
+
block = text.split(OPEN, 1)[1].split(CLOSE, 1)[0] if OPEN in text else ""
|
|
40
|
+
return hashlib.sha256(block.encode()).hexdigest()[:16]
|