proteus-evolve 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proteus_evolve-0.1.0/LICENSE +21 -0
- proteus_evolve-0.1.0/PKG-INFO +317 -0
- proteus_evolve-0.1.0/README.md +284 -0
- proteus_evolve-0.1.0/proteus/__init__.py +2 -0
- proteus_evolve-0.1.0/proteus/adapters/__init__.py +3 -0
- proteus_evolve-0.1.0/proteus/adapters/aki.py +231 -0
- proteus_evolve-0.1.0/proteus/adapters/dsh.py +381 -0
- proteus_evolve-0.1.0/proteus/adapters/instructions.py +40 -0
- proteus_evolve-0.1.0/proteus/adapters/llm.py +172 -0
- proteus_evolve-0.1.0/proteus/adapters/minimal.py +141 -0
- proteus_evolve-0.1.0/proteus/adapters/pi.py +317 -0
- proteus_evolve-0.1.0/proteus/bench/__init__.py +23 -0
- proteus_evolve-0.1.0/proteus/bench/local.py +183 -0
- proteus_evolve-0.1.0/proteus/bench/polyglot.py +173 -0
- proteus_evolve-0.1.0/proteus/bench/swe.py +128 -0
- proteus_evolve-0.1.0/proteus/bench/task.py +111 -0
- proteus_evolve-0.1.0/proteus/cli.py +515 -0
- proteus_evolve-0.1.0/proteus/core/__init__.py +40 -0
- proteus_evolve-0.1.0/proteus/core/adapter.py +150 -0
- proteus_evolve-0.1.0/proteus/core/disposition.py +90 -0
- proteus_evolve-0.1.0/proteus/core/episode.py +274 -0
- proteus_evolve-0.1.0/proteus/core/evaluators.py +113 -0
- proteus_evolve-0.1.0/proteus/core/goal.py +209 -0
- proteus_evolve-0.1.0/proteus/core/snapshot.py +110 -0
- proteus_evolve-0.1.0/proteus/envs.py +141 -0
- proteus_evolve-0.1.0/proteus/measure/__init__.py +3 -0
- proteus_evolve-0.1.0/proteus/measure/audit.py +221 -0
- proteus_evolve-0.1.0/proteus/measure/crystallize.py +76 -0
- proteus_evolve-0.1.0/proteus/measure/distance.py +172 -0
- proteus_evolve-0.1.0/proteus/measure/stream.py +153 -0
- proteus_evolve-0.1.0/proteus/report.py +155 -0
- proteus_evolve-0.1.0/proteus/sandbox/__init__.py +3 -0
- proteus_evolve-0.1.0/proteus/sandbox/docker.py +192 -0
- proteus_evolve-0.1.0/proteus/sweep.py +144 -0
- proteus_evolve-0.1.0/proteus/testing.py +96 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/PKG-INFO +317 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/SOURCES.txt +48 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/dependency_links.txt +1 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/entry_points.txt +2 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/requires.txt +16 -0
- proteus_evolve-0.1.0/proteus_evolve.egg-info/top_level.txt +1 -0
- proteus_evolve-0.1.0/pyproject.toml +55 -0
- proteus_evolve-0.1.0/setup.cfg +4 -0
- proteus_evolve-0.1.0/tests/test_aki_adapter.py +53 -0
- proteus_evolve-0.1.0/tests/test_bench.py +80 -0
- proteus_evolve-0.1.0/tests/test_goals.py +233 -0
- proteus_evolve-0.1.0/tests/test_instrument.py +386 -0
- proteus_evolve-0.1.0/tests/test_polyglot.py +83 -0
- proteus_evolve-0.1.0/tests/test_selfcode.py +286 -0
- proteus_evolve-0.1.0/tests/test_smoke.py +123 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Proteus Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: proteus-evolve
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A harness-agnostic driver for measuring agent self-evolution: plug in any harness × model, evolve it under goal or no-goal conditions, and measure how the harness changes.
|
|
5
|
+
Author: Proteus Authors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://proteus-evolve.github.io/
|
|
8
|
+
Project-URL: Repository, https://github.com/proteus-evolve/Proteus
|
|
9
|
+
Project-URL: Documentation, https://github.com/proteus-evolve/Proteus/tree/main/docs
|
|
10
|
+
Project-URL: Issues, https://github.com/proteus-evolve/Proteus/issues
|
|
11
|
+
Keywords: agents,self-evolution,harness,llm,self-improving
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
25
|
+
Provides-Extra: llm
|
|
26
|
+
Provides-Extra: sandbox
|
|
27
|
+
Provides-Extra: dsh
|
|
28
|
+
Requires-Dist: zstandard>=0.21; python_version < "3.14" and extra == "dsh"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff==0.14.14; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<p align="center">
|
|
35
|
+
<img src="docs/assets/proteus-logo.png" alt="Proteus logo" width="340">
|
|
36
|
+
</p>
|
|
37
|
+
|
|
38
|
+
<h3 align="center">Self-evolution for any agent harness.</h3>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<b>Plug in. Evolve. Measure.</b>
|
|
42
|
+
</p>
|
|
43
|
+
|
|
44
|
+
<p align="center">
|
|
45
|
+
<a href="https://github.com/proteus-evolve/Proteus/actions/workflows/ci.yml"><img src="https://github.com/proteus-evolve/Proteus/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
46
|
+
<a href="https://github.com/proteus-evolve/Proteus/actions/workflows/release-smoke.yml"><img src="https://github.com/proteus-evolve/Proteus/actions/workflows/release-smoke.yml/badge.svg" alt="release smoke"></a>
|
|
47
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License"></a>
|
|
48
|
+
<img src="https://img.shields.io/badge/python-3.10%2B-blue.svg" alt="Python 3.10+">
|
|
49
|
+
<img src="https://img.shields.io/badge/version-0.1.0-informational.svg" alt="v0.1.0">
|
|
50
|
+
<img src="https://img.shields.io/badge/status-research%20preview-orange.svg" alt="research preview">
|
|
51
|
+
</p>
|
|
52
|
+
|
|
53
|
+
<p align="center">
|
|
54
|
+
<a href="#-60-second-demo-no-api-key-no-docker">Quick Start</a> •
|
|
55
|
+
<a href="#-harnesses-in-the-box">Harnesses</a> •
|
|
56
|
+
<a href="#%EF%B8%8F-how-it-works">How It Works</a> •
|
|
57
|
+
<a href="docs/EPISODE.md">The Episode Loop</a> •
|
|
58
|
+
<a href="docs/ADAPTERS.md">Onboard Your Harness</a> •
|
|
59
|
+
<a href="docs/RECIPES.md">Recipes</a> •
|
|
60
|
+
<a href="docs/BENCHMARKS.md">Bring a Benchmark</a> •
|
|
61
|
+
<a href="docs/MEASUREMENTS.md">Add a Measurement</a> •
|
|
62
|
+
<a href="environments/README.md">Environments</a> •
|
|
63
|
+
<a href="#-measurement">Measurement</a>
|
|
64
|
+
</p>
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
Plug in *any* agent harness × *any* model, let it rewrite its own harness over many
|
|
69
|
+
context-fresh episodes, and measure **how the harness changes** — under a goal, many goals,
|
|
70
|
+
or no goal at all.
|
|
71
|
+
|
|
72
|
+
> Named for the sea-god who changes shape at will: Proteus watches a harness reshape
|
|
73
|
+
> itself, and gives you the ruler to measure the change.
|
|
74
|
+
|
|
75
|
+
## 🔭 Why Proteus is different
|
|
76
|
+
|
|
77
|
+
Agent self-improvement is moving from the weights to the **harness** — the prompts, memory,
|
|
78
|
+
skills, tools, and control loop the model runs on. Recent systems evolve a harness to raise
|
|
79
|
+
a benchmark score. Proteus asks a different, complementary question: **what does a
|
|
80
|
+
self-evolving harness actually *do*, and does an initial condition leave a permanent mark?**
|
|
81
|
+
|
|
82
|
+
Three things set it apart from every existing harness-evolution system:
|
|
83
|
+
|
|
84
|
+
1. **Harness-agnostic.** Others evolve harnesses built from their *own* primitives. Proteus
|
|
85
|
+
evolves *yours*: implement one small `HarnessAdapter` and your agent — the bundled
|
|
86
|
+
offline `minimal` harness (the CLI default), DeepSeek Harness, Pi, Aki, or your own —
|
|
87
|
+
plugs into the same framework, sandbox, and measurement.
|
|
88
|
+
2. **Goal *and* no-goal, with visible or hidden evaluators.** Others hard-code a single
|
|
89
|
+
regime: one benchmark verifier, agent blind to the score, goal mandatory. Proteus spans
|
|
90
|
+
the space — `no-goal | one goal | many goals`, and evaluators the agent either **sees**
|
|
91
|
+
(in the observe phase) or **never sees**. No-goal, unpressured evolution is a
|
|
92
|
+
first-class mode.
|
|
93
|
+
3. **A measurement instrument, not just a score.** Others report task pass-rates. Proteus
|
|
94
|
+
ships the ruler for the harness itself: **structural distance** between harness states
|
|
95
|
+
(per surface, path length), a **crystallization / swap** test (remove the disposition,
|
|
96
|
+
read the harness back), and **behavioural distance** with a permutation test (the
|
|
97
|
+
action-preference statistic). Every condition is read with the same ruler.
|
|
98
|
+
|
|
99
|
+
## 🚀 60-second demo (no API key, no Docker)
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install proteus-evolve # no model SDK; Python 3.10 adds only a TOML compatibility package
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The bundled `minimal` harness runs fully offline, so you can see the whole pipeline before
|
|
106
|
+
wiring up a real agent:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
proteus run --harness minimal \
|
|
110
|
+
--arm neutral --arm review:notes --arm review:tools \
|
|
111
|
+
--seeds 4 --episodes 8 --out runs/demo
|
|
112
|
+
proteus measure --harness minimal --out runs/demo
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
arm seeds notes tools (mean units built)
|
|
117
|
+
neutral 4 3.5 4.0
|
|
118
|
+
review_notes 4 13.0 0.0
|
|
119
|
+
review_tools 4 3.8 8.0
|
|
120
|
+
|
|
121
|
+
behavioural R (between/within arms, last episode): 3.075 p=0.0150
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
An installed action preference measurably shifts what the harness grows — and the same
|
|
125
|
+
`measure` reads a no-goal run and a goal run identically.
|
|
126
|
+
|
|
127
|
+
## 🧩 Harnesses in the box
|
|
128
|
+
|
|
129
|
+
| adapter | what it is | needs |
|
|
130
|
+
|---|---|---|
|
|
131
|
+
| `minimal` | offline reference harness (mock policy) | nothing |
|
|
132
|
+
| `llm` | the same harness driven by a live model — any OpenAI-compatible endpoint, DeepSeek by default | an API key |
|
|
133
|
+
| `dsh` | [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness), headless profile, in a prepared container | Docker + a DeepSeek key |
|
|
134
|
+
| `pi` | [Pi](https://github.com/badlogic/pi-mono) — Mario Zechner's minimal coding harness (4 tools, native AGENTS.md + skills) | Docker + a DeepSeek key |
|
|
135
|
+
| `aki` | the Aki research harness (the paper's apparatus) | the research checkout |
|
|
136
|
+
| yours | `--harness <module>:<Class>` — no registration | your adapter |
|
|
137
|
+
|
|
138
|
+
`dsh` and `pi` are the source-evolving third-party integrations. At seed time each adapter
|
|
139
|
+
extracts the pinned harness's real TypeScript source into `harness/src/`; every later phase
|
|
140
|
+
boots that copy, rebuilding it when its content changes. The source is therefore a measured,
|
|
141
|
+
snapshotted `loop` surface alongside instructions, notes, tools, and skills. The adapters
|
|
142
|
+
still leave the upstream repositories untouched: they arrange the run copy, launch one
|
|
143
|
+
prepared container per phase, and parse the harness's own session logs.
|
|
144
|
+
|
|
145
|
+
## 🏗️ How it works
|
|
146
|
+
|
|
147
|
+
```mermaid
|
|
148
|
+
flowchart LR
|
|
149
|
+
U["Run config<br/>harness × model<br/>goal + evaluators<br/>arms + seeds"] --> F["Proteus framework<br/>assemble phase prompts"]
|
|
150
|
+
F --> A["HarnessAdapter<br/>run one episode"]
|
|
151
|
+
A --> H["harness/<br/>evolving, snapshotted subject"]
|
|
152
|
+
A --> T["task/<br/>optional benchmark workspace<br/>outside the snapshot"]
|
|
153
|
+
A --> L["native harness logs"]
|
|
154
|
+
L --> E["evaluators<br/>hidden or observe-visible"]
|
|
155
|
+
E --> S["selection + snapshot<br/>accept or preserve-and-restore"]
|
|
156
|
+
S --> F
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
Every seed runs `N` context-fresh **episodes**; only files cross the episode boundary. One
|
|
160
|
+
episode is four phases:
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
observe → propose → act → reflect
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
- **observe** — take stock; if you configured a *visible* evaluator, its score on the last
|
|
167
|
+
episode is shown here.
|
|
168
|
+
- **propose** — list ways to improve your own harness.
|
|
169
|
+
- **act** — carry one out by editing the harness (the goal, if any, is announced here).
|
|
170
|
+
- **reflect** — decide what to keep.
|
|
171
|
+
|
|
172
|
+
The **framework** owns everything that is not the harness (prompts, goal text, evaluator
|
|
173
|
+
routing, snapshotting, selection, measurement). The **adapter** owns everything that is
|
|
174
|
+
(how the four phases actually execute). That split is what makes Proteus harness-agnostic.
|
|
175
|
+
|
|
176
|
+
### The core objects
|
|
177
|
+
|
|
178
|
+
| Concept | What it is |
|
|
179
|
+
|---|---|
|
|
180
|
+
| `HarnessAdapter` | the contract a harness implements: its surfaces, how to run an episode, how to read the action trace, how to install/remove a disposition |
|
|
181
|
+
| `Surface` | one editable, persistent region (memory / skills / tools / code / …), declared as data so the measurement layer needs no hard-coded names |
|
|
182
|
+
| `Disposition` | the action-preference perturbation — a **single, removable** change at t=0 (prompt suffix, config value, or code patch) |
|
|
183
|
+
| `GoalConfig` | goal / no-goal / multi-goal, each evaluator `HIDDEN` or `OBSERVE`-visible, plus outer-loop selection (`accept_reject`) |
|
|
184
|
+
| `Sandbox` | where an episode runs; `LocalSandbox` (trusted) or `DockerSandbox` (OS-level isolation, tunable network) |
|
|
185
|
+
|
|
186
|
+
### Action preference
|
|
187
|
+
|
|
188
|
+
An action preference is installed as a `Disposition` and is guaranteed **removable**, so the
|
|
189
|
+
crystallization test can take it away and read what the harness built on its own:
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from proteus.core import review, record, NEUTRAL
|
|
193
|
+
review("memory") # each phase: review your memory, act or not
|
|
194
|
+
record("tools") # keep your tools current as you work
|
|
195
|
+
NEUTRAL # the control, F0 — no perturbation
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### Goals and evaluators
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
from proteus.core import EvaluatorSpec, GoalConfig, Visibility
|
|
202
|
+
|
|
203
|
+
GoalConfig.no_goal() # unpressured evolution
|
|
204
|
+
GoalConfig.of(text="Become more reliable.") # stated goal, no evaluator
|
|
205
|
+
GoalConfig.of(
|
|
206
|
+
text="Become more reliable.",
|
|
207
|
+
evaluators=(EvaluatorSpec("reliability", my_eval,
|
|
208
|
+
visibility=Visibility.OBSERVE),),
|
|
209
|
+
) # agent sees the score next episode
|
|
210
|
+
GoalConfig.of(text="Pursue A and B together.",
|
|
211
|
+
evaluators=(EvaluatorSpec("a", eval_a),
|
|
212
|
+
EvaluatorSpec("b", eval_b)),
|
|
213
|
+
selection="accept_reject") # outer loop rejects regressions
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
An evaluator is any callable `(trace, ctx) -> EvalResult`; bring a benchmark verifier, an
|
|
217
|
+
LLM judge, or one of the built-ins (`proteus.core.evaluators`).
|
|
218
|
+
|
|
219
|
+
### Sandbox
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
from proteus.sandbox import SandboxConfig, DockerSandbox
|
|
223
|
+
DockerSandbox(SandboxConfig(network="none")) # no egress
|
|
224
|
+
DockerSandbox(SandboxConfig(network="host", # needs an LLM endpoint
|
|
225
|
+
env_passthrough=("OPENAI_API_KEY",),
|
|
226
|
+
mem_limit="4g"))
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
A self-editing agent writes and runs its own code, so an application-level file sandbox
|
|
230
|
+
cannot contain it — Proteus runs real harnesses in a container whose filesystem holds the
|
|
231
|
+
harness and nothing else.
|
|
232
|
+
|
|
233
|
+
## 🔌 Onboard your harness
|
|
234
|
+
|
|
235
|
+
The input is a **repository** — a git URL or local path:
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
proteus env scaffold --from https://github.com/org/their-harness --name theirs --ref v1.2.0
|
|
239
|
+
proteus env build theirs # pinned image, resolved sha recorded in the manifest
|
|
240
|
+
# write the adapter (7 methods), then:
|
|
241
|
+
proteus check --harness mypkg.theirs_adapter:TheirsHarness --episode
|
|
242
|
+
proteus run --harness mypkg.theirs_adapter:TheirsHarness --arm neutral ...
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
`proteus check` machine-verifies the contract (removable disposition via fingerprint
|
|
246
|
+
round-trip, snapshot-ability, trace shape). The full guide: [docs/ADAPTERS.md](docs/ADAPTERS.md).
|
|
247
|
+
|
|
248
|
+
## 📦 Prepared environments
|
|
249
|
+
|
|
250
|
+
`environments/` contains two environment shapes. Manifest-backed environments pair a
|
|
251
|
+
`Dockerfile` or prebuilt image with `environment.toml`; the built-in `dsh-src/` and
|
|
252
|
+
`pi-src/` images are instead built from pinned upstream source checkouts, because the image
|
|
253
|
+
must contain the exact source and toolchain that the adapter later extracts and rebuilds.
|
|
254
|
+
In both shapes evolving state lives in mounts, never in a per-run image. Conventions:
|
|
255
|
+
[environments/README.md](environments/README.md); design notes:
|
|
256
|
+
[docs/ENVIRONMENTS.md](docs/ENVIRONMENTS.md).
|
|
257
|
+
|
|
258
|
+
## 📏 Measurement
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
from proteus.measure import distance, stream, crystallize
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
- `distance` — structural distance per surface (added / dropped / revised), path length
|
|
265
|
+
(`proteus measure --travel`).
|
|
266
|
+
- `stream` — behavioural distance (frequency / order / procedure) and the between/within
|
|
267
|
+
permutation test `R`.
|
|
268
|
+
- `crystallize` — mount an evolved state under a neutral disposition and test whether it
|
|
269
|
+
reads back as its own endpoint (two-stage fidelity + arm-shift).
|
|
270
|
+
|
|
271
|
+
To add a per-episode measurement evaluator, a post-run statistic, or an adapter-native
|
|
272
|
+
counter, see [docs/MEASUREMENTS.md](docs/MEASUREMENTS.md). The guide covers their different
|
|
273
|
+
contracts, artifact boundaries, CLI/report integration, statistical rules, and tests.
|
|
274
|
+
|
|
275
|
+
## 📤 Outputs
|
|
276
|
+
|
|
277
|
+
Every run's primary artifact is its **evolution history as a git repository** — one commit
|
|
278
|
+
per episode. Keep it local, browse it, or push it wherever you like (never automatic):
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
proteus repo export runs/demo/runs/run-<id> my-evolution # normal repo, `git log` = the trajectory
|
|
282
|
+
proteus repo push runs/demo/runs/run-<id> git@github.com:you/my-evolution.git
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Every sweep also ships a **live tracking page** — per-run progress, per-surface growth
|
|
286
|
+
curves, evaluator scores — updating while the sweep runs:
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
proteus watch --out runs/demo # http://localhost:8300/report.html
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
Tracking data (condition labels, hidden scores) lives at the sweep level, outside run
|
|
293
|
+
roots, so the evolving agent can never read its own condition.
|
|
294
|
+
|
|
295
|
+
## 📊 Status
|
|
296
|
+
|
|
297
|
+
`v0.1` (research preview). Working today: the offline `minimal` harness; the live `llm`
|
|
298
|
+
harness; pinned, source-evolving DeepSeek Harness and Pi adapters with exact-tree boot,
|
|
299
|
+
rebuild caching, viability gates, turn budgets, and task mounts; the Aki research adapter;
|
|
300
|
+
local, Polyglot, and SWE-bench task integrations; resume-safe sweeps; the full measurement,
|
|
301
|
+
audit, reliability, report, and repository-export paths; and adapter/environment tooling.
|
|
302
|
+
CI covers Python 3.10–3.14. The separate release-smoke workflow runs two episodes across
|
|
303
|
+
the public release set (`minimal`, `llm`, `dsh`, `pi`), exercises the benchmark path, and
|
|
304
|
+
requires both container harnesses to edit their own source and boot the edit; releases use
|
|
305
|
+
pinned upstream versions, while the weekly upstream canary is advisory. As a
|
|
306
|
+
cross-implementation check, Proteus's
|
|
307
|
+
behavioural ruler applied to the research runs independently reproduces their headline
|
|
308
|
+
dynamics: arms separate at episode 1 (R = 1.63) and converge by episode 30 (R = 0.93).
|
|
309
|
+
|
|
310
|
+
## 📖 Citation
|
|
311
|
+
|
|
312
|
+
See [CITATION.cff](CITATION.cff). A paper reference will be added when the preprint is
|
|
313
|
+
public.
|
|
314
|
+
|
|
315
|
+
## License
|
|
316
|
+
|
|
317
|
+
MIT.
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/proteus-logo.png" alt="Proteus logo" width="340">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h3 align="center">Self-evolution for any agent harness.</h3>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<b>Plug in. Evolve. Measure.</b>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<a href="https://github.com/proteus-evolve/Proteus/actions/workflows/ci.yml"><img src="https://github.com/proteus-evolve/Proteus/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
13
|
+
<a href="https://github.com/proteus-evolve/Proteus/actions/workflows/release-smoke.yml"><img src="https://github.com/proteus-evolve/Proteus/actions/workflows/release-smoke.yml/badge.svg" alt="release smoke"></a>
|
|
14
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License"></a>
|
|
15
|
+
<img src="https://img.shields.io/badge/python-3.10%2B-blue.svg" alt="Python 3.10+">
|
|
16
|
+
<img src="https://img.shields.io/badge/version-0.1.0-informational.svg" alt="v0.1.0">
|
|
17
|
+
<img src="https://img.shields.io/badge/status-research%20preview-orange.svg" alt="research preview">
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
<p align="center">
|
|
21
|
+
<a href="#-60-second-demo-no-api-key-no-docker">Quick Start</a> •
|
|
22
|
+
<a href="#-harnesses-in-the-box">Harnesses</a> •
|
|
23
|
+
<a href="#%EF%B8%8F-how-it-works">How It Works</a> •
|
|
24
|
+
<a href="docs/EPISODE.md">The Episode Loop</a> •
|
|
25
|
+
<a href="docs/ADAPTERS.md">Onboard Your Harness</a> •
|
|
26
|
+
<a href="docs/RECIPES.md">Recipes</a> •
|
|
27
|
+
<a href="docs/BENCHMARKS.md">Bring a Benchmark</a> •
|
|
28
|
+
<a href="docs/MEASUREMENTS.md">Add a Measurement</a> •
|
|
29
|
+
<a href="environments/README.md">Environments</a> •
|
|
30
|
+
<a href="#-measurement">Measurement</a>
|
|
31
|
+
</p>
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
Plug in *any* agent harness × *any* model, let it rewrite its own harness over many
|
|
36
|
+
context-fresh episodes, and measure **how the harness changes** — under a goal, many goals,
|
|
37
|
+
or no goal at all.
|
|
38
|
+
|
|
39
|
+
> Named for the sea-god who changes shape at will: Proteus watches a harness reshape
|
|
40
|
+
> itself, and gives you the ruler to measure the change.
|
|
41
|
+
|
|
42
|
+
## 🔭 Why Proteus is different
|
|
43
|
+
|
|
44
|
+
Agent self-improvement is moving from the weights to the **harness** — the prompts, memory,
|
|
45
|
+
skills, tools, and control loop the model runs on. Recent systems evolve a harness to raise
|
|
46
|
+
a benchmark score. Proteus asks a different, complementary question: **what does a
|
|
47
|
+
self-evolving harness actually *do*, and does an initial condition leave a permanent mark?**
|
|
48
|
+
|
|
49
|
+
Three things set it apart from every existing harness-evolution system:
|
|
50
|
+
|
|
51
|
+
1. **Harness-agnostic.** Others evolve harnesses built from their *own* primitives. Proteus
|
|
52
|
+
evolves *yours*: implement one small `HarnessAdapter` and your agent — the bundled
|
|
53
|
+
offline `minimal` harness (the CLI default), DeepSeek Harness, Pi, Aki, or your own —
|
|
54
|
+
plugs into the same framework, sandbox, and measurement.
|
|
55
|
+
2. **Goal *and* no-goal, with visible or hidden evaluators.** Others hard-code a single
|
|
56
|
+
regime: one benchmark verifier, agent blind to the score, goal mandatory. Proteus spans
|
|
57
|
+
the space — `no-goal | one goal | many goals`, and evaluators the agent either **sees**
|
|
58
|
+
(in the observe phase) or **never sees**. No-goal, unpressured evolution is a
|
|
59
|
+
first-class mode.
|
|
60
|
+
3. **A measurement instrument, not just a score.** Others report task pass-rates. Proteus
|
|
61
|
+
ships the ruler for the harness itself: **structural distance** between harness states
|
|
62
|
+
(per surface, path length), a **crystallization / swap** test (remove the disposition,
|
|
63
|
+
read the harness back), and **behavioural distance** with a permutation test (the
|
|
64
|
+
action-preference statistic). Every condition is read with the same ruler.
|
|
65
|
+
|
|
66
|
+
## 🚀 60-second demo (no API key, no Docker)
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install proteus-evolve # no model SDK; Python 3.10 adds only a TOML compatibility package
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The bundled `minimal` harness runs fully offline, so you can see the whole pipeline before
|
|
73
|
+
wiring up a real agent:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
proteus run --harness minimal \
|
|
77
|
+
--arm neutral --arm review:notes --arm review:tools \
|
|
78
|
+
--seeds 4 --episodes 8 --out runs/demo
|
|
79
|
+
proteus measure --harness minimal --out runs/demo
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
arm seeds notes tools (mean units built)
|
|
84
|
+
neutral 4 3.5 4.0
|
|
85
|
+
review_notes 4 13.0 0.0
|
|
86
|
+
review_tools 4 3.8 8.0
|
|
87
|
+
|
|
88
|
+
behavioural R (between/within arms, last episode): 3.075 p=0.0150
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
An installed action preference measurably shifts what the harness grows — and the same
|
|
92
|
+
`measure` reads a no-goal run and a goal run identically.
|
|
93
|
+
|
|
94
|
+
## 🧩 Harnesses in the box
|
|
95
|
+
|
|
96
|
+
| adapter | what it is | needs |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| `minimal` | offline reference harness (mock policy) | nothing |
|
|
99
|
+
| `llm` | the same harness driven by a live model — any OpenAI-compatible endpoint, DeepSeek by default | an API key |
|
|
100
|
+
| `dsh` | [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness), headless profile, in a prepared container | Docker + a DeepSeek key |
|
|
101
|
+
| `pi` | [Pi](https://github.com/badlogic/pi-mono) — Mario Zechner's minimal coding harness (4 tools, native AGENTS.md + skills) | Docker + a DeepSeek key |
|
|
102
|
+
| `aki` | the Aki research harness (the paper's apparatus) | the research checkout |
|
|
103
|
+
| yours | `--harness <module>:<Class>` — no registration | your adapter |
|
|
104
|
+
|
|
105
|
+
`dsh` and `pi` are the source-evolving third-party integrations. At seed time each adapter
|
|
106
|
+
extracts the pinned harness's real TypeScript source into `harness/src/`; every later phase
|
|
107
|
+
boots that copy, rebuilding it when its content changes. The source is therefore a measured,
|
|
108
|
+
snapshotted `loop` surface alongside instructions, notes, tools, and skills. The adapters
|
|
109
|
+
still leave the upstream repositories untouched: they arrange the run copy, launch one
|
|
110
|
+
prepared container per phase, and parse the harness's own session logs.
|
|
111
|
+
|
|
112
|
+
## 🏗️ How it works
|
|
113
|
+
|
|
114
|
+
```mermaid
|
|
115
|
+
flowchart LR
|
|
116
|
+
U["Run config<br/>harness × model<br/>goal + evaluators<br/>arms + seeds"] --> F["Proteus framework<br/>assemble phase prompts"]
|
|
117
|
+
F --> A["HarnessAdapter<br/>run one episode"]
|
|
118
|
+
A --> H["harness/<br/>evolving, snapshotted subject"]
|
|
119
|
+
A --> T["task/<br/>optional benchmark workspace<br/>outside the snapshot"]
|
|
120
|
+
A --> L["native harness logs"]
|
|
121
|
+
L --> E["evaluators<br/>hidden or observe-visible"]
|
|
122
|
+
E --> S["selection + snapshot<br/>accept or preserve-and-restore"]
|
|
123
|
+
S --> F
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Every seed runs `N` context-fresh **episodes**; only files cross the episode boundary. One
|
|
127
|
+
episode is four phases:
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
observe → propose → act → reflect
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
- **observe** — take stock; if you configured a *visible* evaluator, its score on the last
|
|
134
|
+
episode is shown here.
|
|
135
|
+
- **propose** — list ways to improve your own harness.
|
|
136
|
+
- **act** — carry one out by editing the harness (the goal, if any, is announced here).
|
|
137
|
+
- **reflect** — decide what to keep.
|
|
138
|
+
|
|
139
|
+
The **framework** owns everything that is not the harness (prompts, goal text, evaluator
|
|
140
|
+
routing, snapshotting, selection, measurement). The **adapter** owns everything that is
|
|
141
|
+
(how the four phases actually execute). That split is what makes Proteus harness-agnostic.
|
|
142
|
+
|
|
143
|
+
### The core objects
|
|
144
|
+
|
|
145
|
+
| Concept | What it is |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `HarnessAdapter` | the contract a harness implements: its surfaces, how to run an episode, how to read the action trace, how to install/remove a disposition |
|
|
148
|
+
| `Surface` | one editable, persistent region (memory / skills / tools / code / …), declared as data so the measurement layer needs no hard-coded names |
|
|
149
|
+
| `Disposition` | the action-preference perturbation — a **single, removable** change at t=0 (prompt suffix, config value, or code patch) |
|
|
150
|
+
| `GoalConfig` | goal / no-goal / multi-goal, each evaluator `HIDDEN` or `OBSERVE`-visible, plus outer-loop selection (`accept_reject`) |
|
|
151
|
+
| `Sandbox` | where an episode runs; `LocalSandbox` (trusted) or `DockerSandbox` (OS-level isolation, tunable network) |
|
|
152
|
+
|
|
153
|
+
### Action preference
|
|
154
|
+
|
|
155
|
+
An action preference is installed as a `Disposition` and is guaranteed **removable**, so the
|
|
156
|
+
crystallization test can take it away and read what the harness built on its own:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from proteus.core import review, record, NEUTRAL
|
|
160
|
+
review("memory") # each phase: review your memory, act or not
|
|
161
|
+
record("tools") # keep your tools current as you work
|
|
162
|
+
NEUTRAL # the control, F0 — no perturbation
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### Goals and evaluators
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from proteus.core import EvaluatorSpec, GoalConfig, Visibility
|
|
169
|
+
|
|
170
|
+
GoalConfig.no_goal() # unpressured evolution
|
|
171
|
+
GoalConfig.of(text="Become more reliable.") # stated goal, no evaluator
|
|
172
|
+
GoalConfig.of(
|
|
173
|
+
text="Become more reliable.",
|
|
174
|
+
evaluators=(EvaluatorSpec("reliability", my_eval,
|
|
175
|
+
visibility=Visibility.OBSERVE),),
|
|
176
|
+
) # agent sees the score next episode
|
|
177
|
+
GoalConfig.of(text="Pursue A and B together.",
|
|
178
|
+
evaluators=(EvaluatorSpec("a", eval_a),
|
|
179
|
+
EvaluatorSpec("b", eval_b)),
|
|
180
|
+
selection="accept_reject") # outer loop rejects regressions
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
An evaluator is any callable `(trace, ctx) -> EvalResult`; bring a benchmark verifier, an
|
|
184
|
+
LLM judge, or one of the built-ins (`proteus.core.evaluators`).
|
|
185
|
+
|
|
186
|
+
### Sandbox
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from proteus.sandbox import SandboxConfig, DockerSandbox
|
|
190
|
+
DockerSandbox(SandboxConfig(network="none")) # no egress
|
|
191
|
+
DockerSandbox(SandboxConfig(network="host", # needs an LLM endpoint
|
|
192
|
+
env_passthrough=("OPENAI_API_KEY",),
|
|
193
|
+
mem_limit="4g"))
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
A self-editing agent writes and runs its own code, so an application-level file sandbox
|
|
197
|
+
cannot contain it — Proteus runs real harnesses in a container whose filesystem holds the
|
|
198
|
+
harness and nothing else.
|
|
199
|
+
|
|
200
|
+
## 🔌 Onboard your harness
|
|
201
|
+
|
|
202
|
+
The input is a **repository** — a git URL or local path:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
proteus env scaffold --from https://github.com/org/their-harness --name theirs --ref v1.2.0
|
|
206
|
+
proteus env build theirs # pinned image, resolved sha recorded in the manifest
|
|
207
|
+
# write the adapter (7 methods), then:
|
|
208
|
+
proteus check --harness mypkg.theirs_adapter:TheirsHarness --episode
|
|
209
|
+
proteus run --harness mypkg.theirs_adapter:TheirsHarness --arm neutral ...
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
`proteus check` machine-verifies the contract (removable disposition via fingerprint
|
|
213
|
+
round-trip, snapshot-ability, trace shape). The full guide: [docs/ADAPTERS.md](docs/ADAPTERS.md).
|
|
214
|
+
|
|
215
|
+
## 📦 Prepared environments
|
|
216
|
+
|
|
217
|
+
`environments/` contains two environment shapes. Manifest-backed environments pair a
|
|
218
|
+
`Dockerfile` or prebuilt image with `environment.toml`; the built-in `dsh-src/` and
|
|
219
|
+
`pi-src/` images are instead built from pinned upstream source checkouts, because the image
|
|
220
|
+
must contain the exact source and toolchain that the adapter later extracts and rebuilds.
|
|
221
|
+
In both shapes evolving state lives in mounts, never in a per-run image. Conventions:
|
|
222
|
+
[environments/README.md](environments/README.md); design notes:
|
|
223
|
+
[docs/ENVIRONMENTS.md](docs/ENVIRONMENTS.md).
|
|
224
|
+
|
|
225
|
+
## 📏 Measurement
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
from proteus.measure import distance, stream, crystallize
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
- `distance` — structural distance per surface (added / dropped / revised), path length
|
|
232
|
+
(`proteus measure --travel`).
|
|
233
|
+
- `stream` — behavioural distance (frequency / order / procedure) and the between/within
|
|
234
|
+
permutation test `R`.
|
|
235
|
+
- `crystallize` — mount an evolved state under a neutral disposition and test whether it
|
|
236
|
+
reads back as its own endpoint (two-stage fidelity + arm-shift).
|
|
237
|
+
|
|
238
|
+
To add a per-episode measurement evaluator, a post-run statistic, or an adapter-native
|
|
239
|
+
counter, see [docs/MEASUREMENTS.md](docs/MEASUREMENTS.md). The guide covers their different
|
|
240
|
+
contracts, artifact boundaries, CLI/report integration, statistical rules, and tests.
|
|
241
|
+
|
|
242
|
+
## 📤 Outputs
|
|
243
|
+
|
|
244
|
+
Every run's primary artifact is its **evolution history as a git repository** — one commit
|
|
245
|
+
per episode. Keep it local, browse it, or push it wherever you like (never automatic):
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
proteus repo export runs/demo/runs/run-<id> my-evolution # normal repo, `git log` = the trajectory
|
|
249
|
+
proteus repo push runs/demo/runs/run-<id> git@github.com:you/my-evolution.git
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
Every sweep also ships a **live tracking page** — per-run progress, per-surface growth
|
|
253
|
+
curves, evaluator scores — updating while the sweep runs:
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
proteus watch --out runs/demo # http://localhost:8300/report.html
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Tracking data (condition labels, hidden scores) lives at the sweep level, outside run
|
|
260
|
+
roots, so the evolving agent can never read its own condition.
|
|
261
|
+
|
|
262
|
+
## 📊 Status
|
|
263
|
+
|
|
264
|
+
`v0.1` (research preview). Working today: the offline `minimal` harness; the live `llm`
|
|
265
|
+
harness; pinned, source-evolving DeepSeek Harness and Pi adapters with exact-tree boot,
|
|
266
|
+
rebuild caching, viability gates, turn budgets, and task mounts; the Aki research adapter;
|
|
267
|
+
local, Polyglot, and SWE-bench task integrations; resume-safe sweeps; the full measurement,
|
|
268
|
+
audit, reliability, report, and repository-export paths; and adapter/environment tooling.
|
|
269
|
+
CI covers Python 3.10–3.14. The separate release-smoke workflow runs two episodes across
|
|
270
|
+
the public release set (`minimal`, `llm`, `dsh`, `pi`), exercises the benchmark path, and
|
|
271
|
+
requires both container harnesses to edit their own source and boot the edit; releases use
|
|
272
|
+
pinned upstream versions, while the weekly upstream canary is advisory. As a
|
|
273
|
+
cross-implementation check, Proteus's
|
|
274
|
+
behavioural ruler applied to the research runs independently reproduces their headline
|
|
275
|
+
dynamics: arms separate at episode 1 (R = 1.63) and converge by episode 30 (R = 0.93).
|
|
276
|
+
|
|
277
|
+
## 📖 Citation
|
|
278
|
+
|
|
279
|
+
See [CITATION.cff](CITATION.cff). A paper reference will be added when the preprint is
|
|
280
|
+
public.
|
|
281
|
+
|
|
282
|
+
## License
|
|
283
|
+
|
|
284
|
+
MIT.
|