fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""The study of milestone M4, episode by episode: `python -m fivefold.harness.study`.
|
|
2
|
+
|
|
3
|
+
M4 pilots 10 to 20 live episodes to measure the real cost, sets the spend cap, then runs
|
|
4
|
+
B0, G1 and G2 across the contract levels (docs/plan.md; spec, Testing). A study runs a
|
|
5
|
+
matrix of episodes on one scenario and saves each run's result JSON in an output
|
|
6
|
+
directory, for `python -m fivefold.evaluator report`. An episode is one run (0009): a
|
|
7
|
+
cell, a seed and the study's model.
|
|
8
|
+
|
|
9
|
+
python -m fivefold.harness.study --out study --cells F2 --contracts C0 C2 --episodes 2
|
|
10
|
+
python -m fivefold.evaluator report study --csv study.csv --markdown study.md
|
|
11
|
+
|
|
12
|
+
Its inputs:
|
|
13
|
+
|
|
14
|
+
- `--scenario`: the scenario file, dev-001 by default.
|
|
15
|
+
- `--configs` and `--contracts`: the configurations and contract levels, by default B0,
|
|
16
|
+
G1 and G2, and C0 to C3. Each level must be one the scenario lists.
|
|
17
|
+
- `--cells`: the cells to run in those configurations and levels. The cells are those of
|
|
18
|
+
`faults/expected.yaml` (0006), and `all`, the default, names every one. A fault, such
|
|
19
|
+
as `F2`, names its cells; a fault and variant, such as `F11-after_tool_call`, names
|
|
20
|
+
that variant's; a cell's own name, such as `F2-C2-G2`, names that cell. `none` names
|
|
21
|
+
the cells with no fault, one in each configuration and level: the scenario's task with
|
|
22
|
+
an empty fault plan. Each cell's fault plan, and any change it makes to the scenario,
|
|
23
|
+
come from `fivefold.harness.cells`.
|
|
24
|
+
- `--episodes N` and `--seed S`: each cell runs N times, with the seeds S, S+1, ...,
|
|
25
|
+
S+N-1. S defaults to the seed of `faults/expected.yaml`.
|
|
26
|
+
- The model: the scripted one by default, or `--live PROVIDER:MODEL` with
|
|
27
|
+
`--spend-cap-usd`, `--prices` and any `--param`, as `record --live` takes them
|
|
28
|
+
(`fivefold.harness.live_options`).
|
|
29
|
+
- `--out DIR`: the output directory. It must be new or empty, so a study never replaces
|
|
30
|
+
another's files.
|
|
31
|
+
- `--database-prefix` and `--keep-databases`: episode N runs in a fresh copy of the
|
|
32
|
+
migrated template database (`fivefold.harness.migrations.create_run_database`), named
|
|
33
|
+
`PREFIX_NNNN` (default prefix `fivefold_study`), dropped once the episode ends unless
|
|
34
|
+
`--keep-databases` is given. A kept database stays in the run registry, so the viewer
|
|
35
|
+
lists it (0012); a dropped one leaves it. The admin connection comes from `--dsn`, or
|
|
36
|
+
`FIVEFOLD_ADMIN_DSN`.
|
|
37
|
+
|
|
38
|
+
A study never drops a database it did not create. Before it starts, it refuses names that
|
|
39
|
+
are taken. Each episode then creates its database without dropping anything first, and a
|
|
40
|
+
name another study took since stops the study in one line, naming the database; the
|
|
41
|
+
episodes already run keep their results. Cleanup drops only the database the episode
|
|
42
|
+
created, the one with the OID PostgreSQL gave it, and removes only that database's row
|
|
43
|
+
from the run registry. The drop and the row's removal happen under the database lock that
|
|
44
|
+
every create takes (`fivefold.harness.migrations.database_lock`), so no other study can
|
|
45
|
+
create the name and register its run between the two.
|
|
46
|
+
|
|
47
|
+
Episodes run in a fixed order: by seed, then by cell. Cells go by fault (no fault first,
|
|
48
|
+
then F1 to F12), variant, contract level and configuration. So a study that stops partway
|
|
49
|
+
has run every cell once before it runs any cell twice.
|
|
50
|
+
|
|
51
|
+
Its outputs, in DIR:
|
|
52
|
+
|
|
53
|
+
- `CELL-seed-SEED.json`: each episode's result JSON, such as `F2-C2-G2-seed-7.json` or
|
|
54
|
+
`none-C2-B0-seed-8.json`. A cell's name holds its fault, variant, contract level and
|
|
55
|
+
configuration, so no two episodes share a file. The same inputs with the scripted model
|
|
56
|
+
give the same bytes, but for the wall-clock latencies of recovery (0009, 0014).
|
|
57
|
+
- `cassettes/CELL-seed-SEED.json`: each live episode's cassette (0009). It replays the
|
|
58
|
+
run with no network. `python -m fivefold.recorder play` compares the replay with the
|
|
59
|
+
original run's database, so it needs the episode's database kept. The report reads only
|
|
60
|
+
the `.json` files directly in DIR, so it never reads a cassette.
|
|
61
|
+
- `spend-cap/CELL-seed-SEED.json`: the result of an episode the spend cap cut short.
|
|
62
|
+
|
|
63
|
+
A live study is refused before it touches any database unless the environment sets
|
|
64
|
+
`FIVEFOLD_LIVE=1`, the spend cap is above 0 USD, and the price file prices the model. One
|
|
65
|
+
spend meter serves every episode, so the cap holds across the study. Once the spend
|
|
66
|
+
reaches the cap, the next model call is refused, and its worker ends as `budget`
|
|
67
|
+
(`fivefold.agents.live`). An episode with a refused call is cut short, and its verdict
|
|
68
|
+
tells more about the cap than about the stack: its result goes to `spend-cap/`, which a
|
|
69
|
+
report over DIR leaves out, and its cassette goes to `cassettes/` as usual. The study
|
|
70
|
+
stops once the spend reaches the cap, since every later episode's first call would be
|
|
71
|
+
refused.
|
|
72
|
+
|
|
73
|
+
The command prints a line for each episode: its verdict, the verdict `faults/expected.yaml`
|
|
74
|
+
expects when it covers the scenario, how each agent worker ended, and in a live study the
|
|
75
|
+
episode's model calls and the spend so far. It exits with status 0 when it ran every
|
|
76
|
+
episode, or when the spend cap stopped it. A refused input, such as an unknown cell or an
|
|
77
|
+
output directory that is not empty, ends the command with one line on standard error and
|
|
78
|
+
exit status 1, before any database is touched.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
from __future__ import annotations
|
|
82
|
+
|
|
83
|
+
import argparse
|
|
84
|
+
import asyncio
|
|
85
|
+
import os
|
|
86
|
+
import re
|
|
87
|
+
import sys
|
|
88
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
89
|
+
from dataclasses import dataclass
|
|
90
|
+
from pathlib import Path
|
|
91
|
+
from typing import Final
|
|
92
|
+
|
|
93
|
+
import psycopg
|
|
94
|
+
|
|
95
|
+
from fivefold.agents.live import SPEND_CAP, refusal
|
|
96
|
+
from fivefold.agents.worker import WorkerEnd
|
|
97
|
+
from fivefold.domain.contracts import Contract
|
|
98
|
+
from fivefold.domain.scenario import Scenario, load_scenario_file
|
|
99
|
+
from fivefold.evaluator import RunResult as Evaluation
|
|
100
|
+
from fivefold.evaluator import result_json
|
|
101
|
+
from fivefold.harness.cells import Cell, Expected, cell_plan, cell_scenario, load_expected
|
|
102
|
+
from fivefold.harness.live_options import add_live_options, check_live_options, live_from_args
|
|
103
|
+
from fivefold.harness.migrations import (
|
|
104
|
+
DEFAULT_ADMIN_DSN,
|
|
105
|
+
CreatedDatabase,
|
|
106
|
+
DatabaseExists,
|
|
107
|
+
MigrationError,
|
|
108
|
+
bootstrap_server,
|
|
109
|
+
check_database_name,
|
|
110
|
+
create_run_database,
|
|
111
|
+
database_lock,
|
|
112
|
+
drop_database_locked,
|
|
113
|
+
)
|
|
114
|
+
from fivefold.harness.models import SCRIPTED, Live, ModelSpec
|
|
115
|
+
from fivefold.harness.registry import forget
|
|
116
|
+
from fivefold.harness.run import CONFIGS, Config, InprocRun, RunSpec, run_plan
|
|
117
|
+
from fivefold.paths import DB_DIR, DEV_001
|
|
118
|
+
from fivefold.recorder.cassette import Cassette, response_message
|
|
119
|
+
|
|
120
|
+
ALL: Final = "all"
|
|
121
|
+
"""The `--cells` name of every cell of `faults/expected.yaml`."""
|
|
122
|
+
|
|
123
|
+
NO_FAULT: Final = "none"
|
|
124
|
+
"""The `--cells` name of the cells with no fault, and the start of each one's name."""
|
|
125
|
+
|
|
126
|
+
CASSETTES: Final = "cassettes"
|
|
127
|
+
"""The subdirectory of the output directory that holds live episodes' cassettes."""
|
|
128
|
+
|
|
129
|
+
SPEND_CAP_DIR: Final = "spend-cap"
|
|
130
|
+
"""The subdirectory that holds the results of episodes the spend cap cut short."""
|
|
131
|
+
|
|
132
|
+
DEFAULT_PREFIX: Final = "fivefold_study"
|
|
133
|
+
"""The default start of each episode's database name."""
|
|
134
|
+
|
|
135
|
+
MAX_SEED: Final = 2**63 - 1
|
|
136
|
+
"""The largest seed `harness.run` can record: its seed column is a bigint."""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# Cells ---------------------------------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass(frozen=True)
|
|
143
|
+
class NoFault:
|
|
144
|
+
"""A cell with no fault: the scenario's task, with an empty fault plan, in one
|
|
145
|
+
contract level and configuration."""
|
|
146
|
+
|
|
147
|
+
contract: Contract
|
|
148
|
+
config: Config
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def name(self) -> str:
|
|
152
|
+
"""A short id such as `none-C2-G2`."""
|
|
153
|
+
return f"{NO_FAULT}-{self.contract}-{self.config}"
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
StudyCell = Cell | NoFault
|
|
157
|
+
"""A cell of a study: a cell of `faults/expected.yaml`, or a cell with no fault."""
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def as_config(name: str) -> Config:
|
|
161
|
+
"""The configuration `name`; raises `ValueError` for one this build does not run."""
|
|
162
|
+
for config in CONFIGS:
|
|
163
|
+
if config == name:
|
|
164
|
+
return config
|
|
165
|
+
raise ValueError(f"no configuration {name!r}; this build runs {', '.join(CONFIGS)}")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def selector_names(cell: StudyCell) -> frozenset[str]:
|
|
169
|
+
"""The `--cells` names that name `cell`. A cell of `faults/expected.yaml` has `all`,
|
|
170
|
+
its fault, its fault and variant when it has a variant, and its own name. A cell with
|
|
171
|
+
no fault has `none` and its own name."""
|
|
172
|
+
if isinstance(cell, NoFault):
|
|
173
|
+
return frozenset({NO_FAULT, cell.name})
|
|
174
|
+
names = {ALL, cell.fault, cell.name}
|
|
175
|
+
if cell.variant is not None:
|
|
176
|
+
names.add(f"{cell.fault}-{cell.variant}")
|
|
177
|
+
return frozenset(names)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def cell_order(cell: StudyCell) -> tuple[int, int, str, str, int, int]:
|
|
181
|
+
"""A cell's place in the study's order: no fault first, then by fault number,
|
|
182
|
+
variant, contract level and configuration."""
|
|
183
|
+
contract = list(Contract).index(cell.contract)
|
|
184
|
+
config = CONFIGS.index(as_config(cell.config))
|
|
185
|
+
if isinstance(cell, NoFault):
|
|
186
|
+
return (0, 0, "", "", contract, config)
|
|
187
|
+
number = re.fullmatch(r"F(\d+)", cell.fault)
|
|
188
|
+
return (1, int(number[1]) if number else 0, cell.fault, cell.variant or "", contract, config)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def select_cells(
|
|
192
|
+
expected: Iterable[Cell],
|
|
193
|
+
names: Sequence[str],
|
|
194
|
+
*,
|
|
195
|
+
configs: Sequence[Config],
|
|
196
|
+
contracts: Sequence[Contract],
|
|
197
|
+
) -> tuple[StudyCell, ...]:
|
|
198
|
+
"""The cells that `names` name (`selector_names`) in `configs` and `contracts`, each
|
|
199
|
+
once, in the study's order (`cell_order`).
|
|
200
|
+
|
|
201
|
+
Raises `ValueError` for a name that names no cell there.
|
|
202
|
+
"""
|
|
203
|
+
candidates: list[StudyCell] = [
|
|
204
|
+
cell for cell in expected if cell.config in configs and cell.contract in contracts
|
|
205
|
+
]
|
|
206
|
+
candidates += [NoFault(contract, config) for contract in contracts for config in configs]
|
|
207
|
+
chosen: dict[str, StudyCell] = {}
|
|
208
|
+
for name in names:
|
|
209
|
+
matched = [cell for cell in candidates if name in selector_names(cell)]
|
|
210
|
+
if not matched:
|
|
211
|
+
raise ValueError(
|
|
212
|
+
f"no cell {name!r} in {', '.join(configs)} at {', '.join(map(str, contracts))}:"
|
|
213
|
+
f" give {ALL}, {NO_FAULT}, a fault such as F2, a fault and variant such as"
|
|
214
|
+
" F11-after_tool_call, or a cell such as F2-C2-G2 (faults/expected.yaml)"
|
|
215
|
+
)
|
|
216
|
+
for cell in matched:
|
|
217
|
+
chosen.setdefault(cell.name, cell)
|
|
218
|
+
return tuple(sorted(chosen.values(), key=cell_order))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def expected_outcome(cell: StudyCell, expected: Expected, scenario: Scenario) -> str | None:
|
|
222
|
+
"""The outcome `faults/expected.yaml` expects of `cell`, such as `fail (1, 2, 3)`; None
|
|
223
|
+
for a cell with no fault, or when the file is for another scenario."""
|
|
224
|
+
if isinstance(cell, NoFault) or expected.scenario != scenario.id:
|
|
225
|
+
return None
|
|
226
|
+
return outcome_text(cell.verdict, cell.failing)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def outcome_text(verdict: str, failing: Iterable[int]) -> str:
|
|
230
|
+
"""A verdict and its failing properties, as `faults/expected.yaml` gives them."""
|
|
231
|
+
numbers = ", ".join(str(number) for number in failing)
|
|
232
|
+
return f"{verdict} ({numbers})" if numbers else verdict
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# Episodes ------------------------------------------------------------------------------------
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@dataclass(frozen=True)
|
|
239
|
+
class Episode:
|
|
240
|
+
"""One run of the study: a cell and a seed (0009). `number` is its place in the
|
|
241
|
+
study's order, from 1."""
|
|
242
|
+
|
|
243
|
+
number: int
|
|
244
|
+
cell: StudyCell
|
|
245
|
+
seed: int
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def name(self) -> str:
|
|
249
|
+
"""The name of its files: the cell's name and the seed, such as `F2-C2-G2-seed-7`."""
|
|
250
|
+
return f"{self.cell.name}-seed-{self.seed}"
|
|
251
|
+
|
|
252
|
+
def spec(self, scenario: Scenario, model: ModelSpec = SCRIPTED) -> RunSpec:
|
|
253
|
+
"""The run: the cell's scenario and fault plan, in its configuration and contract
|
|
254
|
+
level, with the episode's seed and `model`."""
|
|
255
|
+
config = as_config(self.cell.config)
|
|
256
|
+
if isinstance(self.cell, NoFault):
|
|
257
|
+
return RunSpec(scenario, config, self.cell.contract, self.seed, plan=[], model=model)
|
|
258
|
+
return RunSpec(
|
|
259
|
+
cell_scenario(self.cell, scenario),
|
|
260
|
+
config,
|
|
261
|
+
self.cell.contract,
|
|
262
|
+
self.seed,
|
|
263
|
+
plan=cell_plan(self.cell),
|
|
264
|
+
model=model,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def study_seeds(first: int, count: int) -> tuple[int, ...]:
|
|
269
|
+
"""The seeds of each cell's episodes: `first`, `first + 1`, ..., `count` of them."""
|
|
270
|
+
if count < 1:
|
|
271
|
+
raise ValueError(f"a study runs at least 1 episode per cell, not {count}")
|
|
272
|
+
if first < 0 or first + count - 1 > MAX_SEED:
|
|
273
|
+
raise ValueError(
|
|
274
|
+
f"a seed is from 0 to {MAX_SEED}, so the seeds cannot run from {first}"
|
|
275
|
+
f" to {first + count - 1}"
|
|
276
|
+
)
|
|
277
|
+
return tuple(range(first, first + count))
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def study_episodes(cells: Sequence[StudyCell], seeds: Sequence[int]) -> tuple[Episode, ...]:
|
|
281
|
+
"""Every cell with every seed, in the study's order: by seed, then by cell, numbered
|
|
282
|
+
from 1. Raises `ValueError` if two episodes would share a name."""
|
|
283
|
+
pairs = [(cell, seed) for seed in seeds for cell in cells]
|
|
284
|
+
episodes = tuple(
|
|
285
|
+
Episode(number, cell, seed) for number, (cell, seed) in enumerate(pairs, start=1)
|
|
286
|
+
)
|
|
287
|
+
names = [episode.name for episode in episodes]
|
|
288
|
+
if len(set(names)) != len(names):
|
|
289
|
+
raise ValueError("two episodes of the study would share a name")
|
|
290
|
+
return episodes
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def database_name(prefix: str, episode: Episode) -> str:
|
|
294
|
+
"""The database episode runs in: the prefix and its number, such as `fivefold_study_0001`."""
|
|
295
|
+
return f"{prefix}_{episode.number:04d}"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True)
|
|
299
|
+
class Study:
|
|
300
|
+
"""What a study runs: the scenario, the episodes in order, and the model."""
|
|
301
|
+
|
|
302
|
+
scenario: Scenario
|
|
303
|
+
episodes: tuple[Episode, ...]
|
|
304
|
+
model: ModelSpec
|
|
305
|
+
expected: Expected
|
|
306
|
+
"""The expected outcomes, which name the cells; they apply when they cover the scenario."""
|
|
307
|
+
|
|
308
|
+
def describe(self) -> str:
|
|
309
|
+
"""The study in one line: its scenario, model, episodes, cells and seeds."""
|
|
310
|
+
if isinstance(self.model, Live):
|
|
311
|
+
model = f"the live model {self.model.name}"
|
|
312
|
+
else:
|
|
313
|
+
model = "the scripted model"
|
|
314
|
+
cells = len({episode.cell.name for episode in self.episodes})
|
|
315
|
+
seeds = sorted({episode.seed for episode in self.episodes})
|
|
316
|
+
seed_text = f"seed {seeds[0]}" if len(seeds) == 1 else f"seeds {seeds[0]} to {seeds[-1]}"
|
|
317
|
+
return (
|
|
318
|
+
f"Study of {self.scenario.id} with {model}: {len(self.episodes)} episodes,"
|
|
319
|
+
f" {cells} cells, {seed_text}."
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def check_study(study: Study, prefix: str) -> None:
|
|
324
|
+
"""Refuse a study that could not run to its end, before any database is touched.
|
|
325
|
+
|
|
326
|
+
Each episode's contract level must be one the scenario lists, its fault plan must fit
|
|
327
|
+
its configuration and timing as a run checks them (`fivefold.harness.run.run_plan`),
|
|
328
|
+
and its database name must be valid. A cell with a fault runs only on the scenario
|
|
329
|
+
`faults/expected.yaml` is written for: its plan targets that scenario's operation and
|
|
330
|
+
facts, and on another scenario it could fail partway through the study. Raises
|
|
331
|
+
`ValueError`.
|
|
332
|
+
"""
|
|
333
|
+
scenario = study.scenario
|
|
334
|
+
faulted = [episode for episode in study.episodes if not isinstance(episode.cell, NoFault)]
|
|
335
|
+
if faulted and study.expected.scenario != scenario.id:
|
|
336
|
+
raise ValueError(
|
|
337
|
+
f"the cells of faults/expected.yaml are written for {study.expected.scenario},"
|
|
338
|
+
f" not {scenario.id}; with another scenario, give --cells {NO_FAULT}"
|
|
339
|
+
)
|
|
340
|
+
for episode in study.episodes:
|
|
341
|
+
spec = episode.spec(scenario)
|
|
342
|
+
if spec.contract not in scenario.contracts:
|
|
343
|
+
levels = ", ".join(map(str, scenario.contracts))
|
|
344
|
+
raise ValueError(f"{scenario.id} runs at {levels}, not at {spec.contract}")
|
|
345
|
+
try:
|
|
346
|
+
run_plan(spec)
|
|
347
|
+
check_database_name(database_name(prefix, episode))
|
|
348
|
+
except (ValueError, MigrationError) as error:
|
|
349
|
+
raise ValueError(f"episode {episode.name}: {error}") from None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def check_databases_free(admin_dsn: str, study: Study, prefix: str) -> None:
|
|
353
|
+
"""Refuse a study whose episode databases already exist, such as the episodes another
|
|
354
|
+
study kept, rather than drop them. Raises `ValueError`."""
|
|
355
|
+
names = [database_name(prefix, episode) for episode in study.episodes]
|
|
356
|
+
with psycopg.connect(admin_dsn) as conn:
|
|
357
|
+
rows = conn.execute(
|
|
358
|
+
"SELECT datname FROM pg_database WHERE datname = ANY(%s) ORDER BY datname", (names,)
|
|
359
|
+
).fetchall()
|
|
360
|
+
if rows:
|
|
361
|
+
raise ValueError(taken(str(rows[0][0])))
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def taken(name: str) -> str:
|
|
365
|
+
"""Why a study refuses the episode database `name`: it already exists."""
|
|
366
|
+
return (
|
|
367
|
+
f"the database {name} already exists, perhaps an episode another study ran or kept;"
|
|
368
|
+
" give --database-prefix another prefix, or drop it"
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def drop_own_database(admin_dsn: str, created: CreatedDatabase) -> None:
|
|
373
|
+
"""Drop the database an episode created, if the name is still that database (its
|
|
374
|
+
OID), and then remove its row from the run registry.
|
|
375
|
+
|
|
376
|
+
Both happen under the database lock (`fivefold.harness.migrations.database_lock`),
|
|
377
|
+
which every create takes: a study that creates the name again does it after the row
|
|
378
|
+
is gone, so its own row stays.
|
|
379
|
+
"""
|
|
380
|
+
with database_lock(admin_dsn) as conn:
|
|
381
|
+
if drop_database_locked(conn, created.name, oid=created.oid):
|
|
382
|
+
forget(admin_dsn, created.name)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def check_out(out: Path) -> None:
|
|
386
|
+
"""Refuse an output directory that is a file, or a directory that holds anything."""
|
|
387
|
+
if out.exists() and (not out.is_dir() or any(out.iterdir())):
|
|
388
|
+
raise ValueError(f"{out} is not a new or empty directory; give the study its own")
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
# Running -------------------------------------------------------------------------------------
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
@dataclass(frozen=True)
|
|
395
|
+
class Outcome:
|
|
396
|
+
"""How an episode ended, and where its files went."""
|
|
397
|
+
|
|
398
|
+
episode: Episode
|
|
399
|
+
evaluation: Evaluation
|
|
400
|
+
ends: dict[str, tuple[WorkerEnd, str]]
|
|
401
|
+
"""How each agent worker ended, and why."""
|
|
402
|
+
result: Path
|
|
403
|
+
cassette: Path | None
|
|
404
|
+
"""The live episode's cassette; None with the scripted model."""
|
|
405
|
+
calls: int | None
|
|
406
|
+
"""The live episode's model calls, refused ones included; None with the scripted model."""
|
|
407
|
+
refused: int
|
|
408
|
+
"""The model calls the spend cap refused; each ends its worker as `budget`."""
|
|
409
|
+
database: str | None
|
|
410
|
+
"""The episode's database, when it is kept."""
|
|
411
|
+
|
|
412
|
+
@property
|
|
413
|
+
def cut(self) -> bool:
|
|
414
|
+
"""Whether the spend cap cut the episode short: it refused one of its model calls."""
|
|
415
|
+
return self.refused > 0
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def refused_calls(cassette: Cassette) -> int:
|
|
419
|
+
"""How many of the cassette's calls the spend cap refused (`fivefold.agents.live`)."""
|
|
420
|
+
return sum(
|
|
421
|
+
1 for call in cassette.calls if refusal(response_message(call.response)) == SPEND_CAP
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def run_episode(
|
|
426
|
+
admin_dsn: str, study: Study, episode: Episode, *, out: Path, database: str, keep: bool
|
|
427
|
+
) -> Outcome:
|
|
428
|
+
"""Run `episode` in `database`, a fresh copy of the migrated template database, and
|
|
429
|
+
save its result, and its cassette when the model is live, in `out`. The database is
|
|
430
|
+
dropped once the run ends, unless `keep`; a database the episode dropped leaves the
|
|
431
|
+
run registry too.
|
|
432
|
+
|
|
433
|
+
Raises `ValueError` when `database` already exists: it is someone else's, so it is
|
|
434
|
+
neither replaced nor dropped, and its registry row stays.
|
|
435
|
+
"""
|
|
436
|
+
spec = episode.spec(study.scenario, study.model)
|
|
437
|
+
try:
|
|
438
|
+
created = create_run_database(admin_dsn, database, DB_DIR / "migrations")
|
|
439
|
+
except DatabaseExists:
|
|
440
|
+
raise ValueError(taken(database)) from None
|
|
441
|
+
try:
|
|
442
|
+
run = InprocRun(spec, created.dsn)
|
|
443
|
+
asyncio.run(run.execute())
|
|
444
|
+
finally:
|
|
445
|
+
if not keep:
|
|
446
|
+
drop_own_database(admin_dsn, created)
|
|
447
|
+
evaluation = run.result.evaluation
|
|
448
|
+
if evaluation is None:
|
|
449
|
+
raise RuntimeError(f"episode {episode.name} was not evaluated")
|
|
450
|
+
ends = dict(run.result.ends)
|
|
451
|
+
cassette: Cassette | None = None
|
|
452
|
+
if isinstance(spec.model, Live):
|
|
453
|
+
cassette = run.models.cassette
|
|
454
|
+
if cassette is None:
|
|
455
|
+
raise RuntimeError(f"episode {episode.name} recorded no cassette")
|
|
456
|
+
refused = 0 if cassette is None else refused_calls(cassette)
|
|
457
|
+
folder = out / SPEND_CAP_DIR if refused else out
|
|
458
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
459
|
+
result = folder / f"{episode.name}.json"
|
|
460
|
+
result.write_bytes(result_json(evaluation).encode("utf-8"))
|
|
461
|
+
saved: Path | None = None
|
|
462
|
+
if cassette is not None:
|
|
463
|
+
(out / CASSETTES).mkdir(parents=True, exist_ok=True)
|
|
464
|
+
saved = out / CASSETTES / f"{episode.name}.json"
|
|
465
|
+
cassette.save(saved)
|
|
466
|
+
return Outcome(
|
|
467
|
+
episode=episode,
|
|
468
|
+
evaluation=evaluation,
|
|
469
|
+
ends=ends,
|
|
470
|
+
result=result,
|
|
471
|
+
cassette=saved,
|
|
472
|
+
calls=None if cassette is None else len(cassette.calls),
|
|
473
|
+
refused=refused,
|
|
474
|
+
database=database if keep else None,
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def run_study(
|
|
479
|
+
admin_dsn: str, study: Study, *, out: Path, prefix: str, keep: bool = False
|
|
480
|
+
) -> Iterator[Outcome]:
|
|
481
|
+
"""Run the study's episodes in order, each in its own database, and yield how each
|
|
482
|
+
ended. A live study stops once its spend reaches the cap: every later episode's first
|
|
483
|
+
model call would be refused."""
|
|
484
|
+
meter = study.model.spend if isinstance(study.model, Live) else None
|
|
485
|
+
for episode in study.episodes:
|
|
486
|
+
if meter is not None and meter.reached():
|
|
487
|
+
return
|
|
488
|
+
database = database_name(prefix, episode)
|
|
489
|
+
try:
|
|
490
|
+
outcome = run_episode(admin_dsn, study, episode, out=out, database=database, keep=keep)
|
|
491
|
+
except Exception as error:
|
|
492
|
+
error.add_note(f"in episode {episode.number}, {episode.name}")
|
|
493
|
+
raise
|
|
494
|
+
yield outcome
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
# The command ---------------------------------------------------------------------------------
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def episode_line(study: Study, outcome: Outcome) -> str:
|
|
501
|
+
"""One line on how an episode ended."""
|
|
502
|
+
episode, evaluation = outcome.episode, outcome.evaluation
|
|
503
|
+
failing = [prop.property for prop in evaluation.properties if str(prop.verdict) == "fail"]
|
|
504
|
+
line = f"[{episode.number}/{len(study.episodes)}] {episode.name}: "
|
|
505
|
+
line += outcome_text(str(evaluation.verdict), failing)
|
|
506
|
+
expected = expected_outcome(episode.cell, study.expected, study.scenario)
|
|
507
|
+
if expected is not None:
|
|
508
|
+
line += f", expected {expected}"
|
|
509
|
+
ends = ", ".join(
|
|
510
|
+
f"{worker} {end}" + ("" if end is WorkerEnd.FINISHED else f" ({text})")
|
|
511
|
+
for worker, (end, text) in sorted(outcome.ends.items())
|
|
512
|
+
)
|
|
513
|
+
line += f"; {ends}"
|
|
514
|
+
meter = study.model.spend if isinstance(study.model, Live) else None
|
|
515
|
+
if outcome.calls is not None and meter is not None:
|
|
516
|
+
line += f"; {outcome.calls} model calls"
|
|
517
|
+
line += f", {outcome.refused} refused" if outcome.refused else ""
|
|
518
|
+
line += f", {meter.spent_usd} USD spent so far"
|
|
519
|
+
if outcome.cut:
|
|
520
|
+
line += f"; cut short by the spend cap, so its result is in {SPEND_CAP_DIR}/"
|
|
521
|
+
if outcome.database is not None:
|
|
522
|
+
line += f"; kept in the database {outcome.database}"
|
|
523
|
+
return line
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def plan_study(args: argparse.Namespace, model: ModelSpec) -> Study:
|
|
527
|
+
"""The study the command's arguments describe, checked before anything runs."""
|
|
528
|
+
expected = load_expected()
|
|
529
|
+
scenario = load_scenario_file(args.scenario)
|
|
530
|
+
configs = [config for config in CONFIGS if config in args.configs]
|
|
531
|
+
contracts = [contract for contract in Contract if str(contract) in args.contracts]
|
|
532
|
+
cells = select_cells(expected.cells, args.cells, configs=configs, contracts=contracts)
|
|
533
|
+
first = expected.seed if args.seed is None else args.seed
|
|
534
|
+
episodes = study_episodes(cells, study_seeds(first, args.episodes))
|
|
535
|
+
study = Study(scenario=scenario, episodes=episodes, model=model, expected=expected)
|
|
536
|
+
check_study(study, args.database_prefix)
|
|
537
|
+
return study
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
541
|
+
"""Run a study. Returns the process exit status."""
|
|
542
|
+
parser = argparse.ArgumentParser(
|
|
543
|
+
prog="python -m fivefold.harness.study",
|
|
544
|
+
description="Run a matrix of episodes and save each run's result for the report.",
|
|
545
|
+
)
|
|
546
|
+
parser.add_argument(
|
|
547
|
+
"--out", type=Path, required=True, metavar="DIR", help="a new or empty directory"
|
|
548
|
+
)
|
|
549
|
+
parser.add_argument(
|
|
550
|
+
"--scenario", type=Path, default=DEV_001, help="the scenario file (default: dev-001)"
|
|
551
|
+
)
|
|
552
|
+
parser.add_argument(
|
|
553
|
+
"--configs",
|
|
554
|
+
nargs="+",
|
|
555
|
+
choices=CONFIGS,
|
|
556
|
+
default=list(CONFIGS),
|
|
557
|
+
metavar="CONFIG",
|
|
558
|
+
help=f"configurations among {', '.join(CONFIGS)} (default: all three)",
|
|
559
|
+
)
|
|
560
|
+
levels = [str(contract) for contract in Contract]
|
|
561
|
+
parser.add_argument(
|
|
562
|
+
"--contracts",
|
|
563
|
+
nargs="+",
|
|
564
|
+
choices=levels,
|
|
565
|
+
default=levels,
|
|
566
|
+
metavar="LEVEL",
|
|
567
|
+
help=f"contract levels among {', '.join(levels)} (default: all four)",
|
|
568
|
+
)
|
|
569
|
+
parser.add_argument(
|
|
570
|
+
"--cells",
|
|
571
|
+
nargs="+",
|
|
572
|
+
default=[ALL],
|
|
573
|
+
metavar="CELL",
|
|
574
|
+
help=f"{ALL}, {NO_FAULT}, a fault (F2), a fault and variant (F11-after_tool_call),"
|
|
575
|
+
f" or a cell (F2-C2-G2) of faults/expected.yaml (default: {ALL})",
|
|
576
|
+
)
|
|
577
|
+
parser.add_argument(
|
|
578
|
+
"--episodes", type=int, default=1, metavar="N", help="episodes per cell (default 1)"
|
|
579
|
+
)
|
|
580
|
+
parser.add_argument(
|
|
581
|
+
"--seed",
|
|
582
|
+
type=int,
|
|
583
|
+
metavar="S",
|
|
584
|
+
help="the first seed; each cell runs with S, S+1, ... (default: faults/expected.yaml's)",
|
|
585
|
+
)
|
|
586
|
+
parser.add_argument(
|
|
587
|
+
"--database-prefix",
|
|
588
|
+
default=DEFAULT_PREFIX,
|
|
589
|
+
metavar="PREFIX",
|
|
590
|
+
help=f"episode N runs in the database PREFIX_NNNN (default {DEFAULT_PREFIX})",
|
|
591
|
+
)
|
|
592
|
+
parser.add_argument(
|
|
593
|
+
"--keep-databases", action="store_true", help="keep each episode's database"
|
|
594
|
+
)
|
|
595
|
+
parser.add_argument(
|
|
596
|
+
"--dsn",
|
|
597
|
+
default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
|
|
598
|
+
help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
|
|
599
|
+
)
|
|
600
|
+
add_live_options(parser)
|
|
601
|
+
args = parser.parse_args(argv)
|
|
602
|
+
check_live_options(parser, args)
|
|
603
|
+
out: Path = args.out
|
|
604
|
+
try:
|
|
605
|
+
live = live_from_args(args, os.environ)
|
|
606
|
+
study = plan_study(args, SCRIPTED if live is None else live)
|
|
607
|
+
check_out(out)
|
|
608
|
+
check_databases_free(args.dsn, study, args.database_prefix)
|
|
609
|
+
bootstrap_server(args.dsn, DB_DIR)
|
|
610
|
+
print(study.describe(), flush=True)
|
|
611
|
+
ran = 0
|
|
612
|
+
try:
|
|
613
|
+
for outcome in run_study(
|
|
614
|
+
args.dsn, study, out=out, prefix=args.database_prefix, keep=args.keep_databases
|
|
615
|
+
):
|
|
616
|
+
ran += 1
|
|
617
|
+
print(episode_line(study, outcome), flush=True)
|
|
618
|
+
finally:
|
|
619
|
+
if live is not None and live.spend is not None:
|
|
620
|
+
print(f"Model {live.name}; {live.spend.summary()}.")
|
|
621
|
+
print(end_line(study, ran, out))
|
|
622
|
+
except (OSError, ValueError, MigrationError) as error:
|
|
623
|
+
notes = "".join(f" ({note})" for note in getattr(error, "__notes__", ()))
|
|
624
|
+
print(f"study failed: {error}{notes}", file=sys.stderr)
|
|
625
|
+
return 1
|
|
626
|
+
return 0
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def end_line(study: Study, ran: int, out: Path) -> str:
|
|
630
|
+
"""How many episodes ran, where their results are, and how to report them."""
|
|
631
|
+
total = len(study.episodes)
|
|
632
|
+
if ran < total:
|
|
633
|
+
text = f"The spend cap stopped the study after {ran} of {total} episodes."
|
|
634
|
+
else:
|
|
635
|
+
text = f"Ran all {total} episodes."
|
|
636
|
+
return (
|
|
637
|
+
f"{text} Results are in {out}. Report them with: python -m fivefold.evaluator"
|
|
638
|
+
f" report {out} --csv runs.csv --markdown report.md"
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
if __name__ == "__main__":
|
|
643
|
+
sys.exit(main())
|