fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,643 @@
1
+ """The study of milestone M4, episode by episode: `python -m fivefold.harness.study`.
2
+
3
+ M4 pilots 10 to 20 live episodes to measure the real cost, sets the spend cap, then runs
4
+ B0, G1 and G2 across the contract levels (docs/plan.md; spec, Testing). A study runs a
5
+ matrix of episodes on one scenario and saves each run's result JSON in an output
6
+ directory, for `python -m fivefold.evaluator report`. An episode is one run (0009): a
7
+ cell, a seed and the study's model.
8
+
9
+ python -m fivefold.harness.study --out study --cells F2 --contracts C0 C2 --episodes 2
10
+ python -m fivefold.evaluator report study --csv study.csv --markdown study.md
11
+
12
+ Its inputs:
13
+
14
+ - `--scenario`: the scenario file, dev-001 by default.
15
+ - `--configs` and `--contracts`: the configurations and contract levels, by default B0,
16
+ G1 and G2, and C0 to C3. Each level must be one the scenario lists.
17
+ - `--cells`: the cells to run in those configurations and levels. The cells are those of
18
+ `faults/expected.yaml` (0006), and `all`, the default, names every one. A fault, such
19
+ as `F2`, names its cells; a fault and variant, such as `F11-after_tool_call`, names
20
+ that variant's; a cell's own name, such as `F2-C2-G2`, names that cell. `none` names
21
+ the cells with no fault, one in each configuration and level: the scenario's task with
22
+ an empty fault plan. Each cell's fault plan, and any change it makes to the scenario,
23
+ come from `fivefold.harness.cells`.
24
+ - `--episodes N` and `--seed S`: each cell runs N times, with the seeds S, S+1, ...,
25
+ S+N-1. S defaults to the seed of `faults/expected.yaml`.
26
+ - The model: the scripted one by default, or `--live PROVIDER:MODEL` with
27
+ `--spend-cap-usd`, `--prices` and any `--param`, as `record --live` takes them
28
+ (`fivefold.harness.live_options`).
29
+ - `--out DIR`: the output directory. It must be new or empty, so a study never replaces
30
+ another's files.
31
+ - `--database-prefix` and `--keep-databases`: episode N runs in a fresh copy of the
32
+ migrated template database (`fivefold.harness.migrations.create_run_database`), named
33
+ `PREFIX_NNNN` (default prefix `fivefold_study`), dropped once the episode ends unless
34
+ `--keep-databases` is given. A kept database stays in the run registry, so the viewer
35
+ lists it (0012); a dropped one leaves it. The admin connection comes from `--dsn`, or
36
+ `FIVEFOLD_ADMIN_DSN`.
37
+
38
+ A study never drops a database it did not create. Before it starts, it refuses names that
39
+ are taken. Each episode then creates its database without dropping anything first, and a
40
+ name another study took since stops the study in one line, naming the database; the
41
+ episodes already run keep their results. Cleanup drops only the database the episode
42
+ created, the one with the OID PostgreSQL gave it, and removes only that database's row
43
+ from the run registry. The drop and the row's removal happen under the database lock that
44
+ every create takes (`fivefold.harness.migrations.database_lock`), so no other study can
45
+ create the name and register its run between the two.
46
+
47
+ Episodes run in a fixed order: by seed, then by cell. Cells go by fault (no fault first,
48
+ then F1 to F12), variant, contract level and configuration. So a study that stops partway
49
+ has run every cell once before it runs any cell twice.
50
+
51
+ Its outputs, in DIR:
52
+
53
+ - `CELL-seed-SEED.json`: each episode's result JSON, such as `F2-C2-G2-seed-7.json` or
54
+ `none-C2-B0-seed-8.json`. A cell's name holds its fault, variant, contract level and
55
+ configuration, so no two episodes share a file. The same inputs with the scripted model
56
+ give the same bytes, but for the wall-clock latencies of recovery (0009, 0014).
57
+ - `cassettes/CELL-seed-SEED.json`: each live episode's cassette (0009). It replays the
58
+ run with no network. `python -m fivefold.recorder play` compares the replay with the
59
+ original run's database, so it needs the episode's database kept. The report reads only
60
+ the `.json` files directly in DIR, so it never reads a cassette.
61
+ - `spend-cap/CELL-seed-SEED.json`: the result of an episode the spend cap cut short.
62
+
63
+ A live study is refused before it touches any database unless the environment sets
64
+ `FIVEFOLD_LIVE=1`, the spend cap is above 0 USD, and the price file prices the model. One
65
+ spend meter serves every episode, so the cap holds across the study. Once the spend
66
+ reaches the cap, the next model call is refused, and its worker ends as `budget`
67
+ (`fivefold.agents.live`). An episode with a refused call is cut short, and its verdict
68
+ tells more about the cap than about the stack: its result goes to `spend-cap/`, which a
69
+ report over DIR leaves out, and its cassette goes to `cassettes/` as usual. The study
70
+ stops once the spend reaches the cap, since every later episode's first call would be
71
+ refused.
72
+
73
+ The command prints a line for each episode: its verdict, the verdict `faults/expected.yaml`
74
+ expects when it covers the scenario, how each agent worker ended, and in a live study the
75
+ episode's model calls and the spend so far. It exits with status 0 when it ran every
76
+ episode, or when the spend cap stopped it. A refused input, such as an unknown cell or an
77
+ output directory that is not empty, ends the command with one line on standard error and
78
+ exit status 1, before any database is touched.
79
+ """
80
+
81
+ from __future__ import annotations
82
+
83
+ import argparse
84
+ import asyncio
85
+ import os
86
+ import re
87
+ import sys
88
+ from collections.abc import Iterable, Iterator, Sequence
89
+ from dataclasses import dataclass
90
+ from pathlib import Path
91
+ from typing import Final
92
+
93
+ import psycopg
94
+
95
+ from fivefold.agents.live import SPEND_CAP, refusal
96
+ from fivefold.agents.worker import WorkerEnd
97
+ from fivefold.domain.contracts import Contract
98
+ from fivefold.domain.scenario import Scenario, load_scenario_file
99
+ from fivefold.evaluator import RunResult as Evaluation
100
+ from fivefold.evaluator import result_json
101
+ from fivefold.harness.cells import Cell, Expected, cell_plan, cell_scenario, load_expected
102
+ from fivefold.harness.live_options import add_live_options, check_live_options, live_from_args
103
+ from fivefold.harness.migrations import (
104
+ DEFAULT_ADMIN_DSN,
105
+ CreatedDatabase,
106
+ DatabaseExists,
107
+ MigrationError,
108
+ bootstrap_server,
109
+ check_database_name,
110
+ create_run_database,
111
+ database_lock,
112
+ drop_database_locked,
113
+ )
114
+ from fivefold.harness.models import SCRIPTED, Live, ModelSpec
115
+ from fivefold.harness.registry import forget
116
+ from fivefold.harness.run import CONFIGS, Config, InprocRun, RunSpec, run_plan
117
+ from fivefold.paths import DB_DIR, DEV_001
118
+ from fivefold.recorder.cassette import Cassette, response_message
119
+
120
+ ALL: Final = "all"
121
+ """The `--cells` name of every cell of `faults/expected.yaml`."""
122
+
123
+ NO_FAULT: Final = "none"
124
+ """The `--cells` name of the cells with no fault, and the start of each one's name."""
125
+
126
+ CASSETTES: Final = "cassettes"
127
+ """The subdirectory of the output directory that holds live episodes' cassettes."""
128
+
129
+ SPEND_CAP_DIR: Final = "spend-cap"
130
+ """The subdirectory that holds the results of episodes the spend cap cut short."""
131
+
132
+ DEFAULT_PREFIX: Final = "fivefold_study"
133
+ """The default start of each episode's database name."""
134
+
135
+ MAX_SEED: Final = 2**63 - 1
136
+ """The largest seed `harness.run` can record: its seed column is a bigint."""
137
+
138
+
139
+ # Cells ---------------------------------------------------------------------------------------
140
+
141
+
142
+ @dataclass(frozen=True)
143
+ class NoFault:
144
+ """A cell with no fault: the scenario's task, with an empty fault plan, in one
145
+ contract level and configuration."""
146
+
147
+ contract: Contract
148
+ config: Config
149
+
150
+ @property
151
+ def name(self) -> str:
152
+ """A short id such as `none-C2-G2`."""
153
+ return f"{NO_FAULT}-{self.contract}-{self.config}"
154
+
155
+
156
+ StudyCell = Cell | NoFault
157
+ """A cell of a study: a cell of `faults/expected.yaml`, or a cell with no fault."""
158
+
159
+
160
+ def as_config(name: str) -> Config:
161
+ """The configuration `name`; raises `ValueError` for one this build does not run."""
162
+ for config in CONFIGS:
163
+ if config == name:
164
+ return config
165
+ raise ValueError(f"no configuration {name!r}; this build runs {', '.join(CONFIGS)}")
166
+
167
+
168
+ def selector_names(cell: StudyCell) -> frozenset[str]:
169
+ """The `--cells` names that name `cell`. A cell of `faults/expected.yaml` has `all`,
170
+ its fault, its fault and variant when it has a variant, and its own name. A cell with
171
+ no fault has `none` and its own name."""
172
+ if isinstance(cell, NoFault):
173
+ return frozenset({NO_FAULT, cell.name})
174
+ names = {ALL, cell.fault, cell.name}
175
+ if cell.variant is not None:
176
+ names.add(f"{cell.fault}-{cell.variant}")
177
+ return frozenset(names)
178
+
179
+
180
+ def cell_order(cell: StudyCell) -> tuple[int, int, str, str, int, int]:
181
+ """A cell's place in the study's order: no fault first, then by fault number,
182
+ variant, contract level and configuration."""
183
+ contract = list(Contract).index(cell.contract)
184
+ config = CONFIGS.index(as_config(cell.config))
185
+ if isinstance(cell, NoFault):
186
+ return (0, 0, "", "", contract, config)
187
+ number = re.fullmatch(r"F(\d+)", cell.fault)
188
+ return (1, int(number[1]) if number else 0, cell.fault, cell.variant or "", contract, config)
189
+
190
+
191
+ def select_cells(
192
+ expected: Iterable[Cell],
193
+ names: Sequence[str],
194
+ *,
195
+ configs: Sequence[Config],
196
+ contracts: Sequence[Contract],
197
+ ) -> tuple[StudyCell, ...]:
198
+ """The cells that `names` name (`selector_names`) in `configs` and `contracts`, each
199
+ once, in the study's order (`cell_order`).
200
+
201
+ Raises `ValueError` for a name that names no cell there.
202
+ """
203
+ candidates: list[StudyCell] = [
204
+ cell for cell in expected if cell.config in configs and cell.contract in contracts
205
+ ]
206
+ candidates += [NoFault(contract, config) for contract in contracts for config in configs]
207
+ chosen: dict[str, StudyCell] = {}
208
+ for name in names:
209
+ matched = [cell for cell in candidates if name in selector_names(cell)]
210
+ if not matched:
211
+ raise ValueError(
212
+ f"no cell {name!r} in {', '.join(configs)} at {', '.join(map(str, contracts))}:"
213
+ f" give {ALL}, {NO_FAULT}, a fault such as F2, a fault and variant such as"
214
+ " F11-after_tool_call, or a cell such as F2-C2-G2 (faults/expected.yaml)"
215
+ )
216
+ for cell in matched:
217
+ chosen.setdefault(cell.name, cell)
218
+ return tuple(sorted(chosen.values(), key=cell_order))
219
+
220
+
221
+ def expected_outcome(cell: StudyCell, expected: Expected, scenario: Scenario) -> str | None:
222
+ """The outcome `faults/expected.yaml` expects of `cell`, such as `fail (1, 2, 3)`; None
223
+ for a cell with no fault, or when the file is for another scenario."""
224
+ if isinstance(cell, NoFault) or expected.scenario != scenario.id:
225
+ return None
226
+ return outcome_text(cell.verdict, cell.failing)
227
+
228
+
229
+ def outcome_text(verdict: str, failing: Iterable[int]) -> str:
230
+ """A verdict and its failing properties, as `faults/expected.yaml` gives them."""
231
+ numbers = ", ".join(str(number) for number in failing)
232
+ return f"{verdict} ({numbers})" if numbers else verdict
233
+
234
+
235
+ # Episodes ------------------------------------------------------------------------------------
236
+
237
+
238
+ @dataclass(frozen=True)
239
+ class Episode:
240
+ """One run of the study: a cell and a seed (0009). `number` is its place in the
241
+ study's order, from 1."""
242
+
243
+ number: int
244
+ cell: StudyCell
245
+ seed: int
246
+
247
+ @property
248
+ def name(self) -> str:
249
+ """The name of its files: the cell's name and the seed, such as `F2-C2-G2-seed-7`."""
250
+ return f"{self.cell.name}-seed-{self.seed}"
251
+
252
+ def spec(self, scenario: Scenario, model: ModelSpec = SCRIPTED) -> RunSpec:
253
+ """The run: the cell's scenario and fault plan, in its configuration and contract
254
+ level, with the episode's seed and `model`."""
255
+ config = as_config(self.cell.config)
256
+ if isinstance(self.cell, NoFault):
257
+ return RunSpec(scenario, config, self.cell.contract, self.seed, plan=[], model=model)
258
+ return RunSpec(
259
+ cell_scenario(self.cell, scenario),
260
+ config,
261
+ self.cell.contract,
262
+ self.seed,
263
+ plan=cell_plan(self.cell),
264
+ model=model,
265
+ )
266
+
267
+
268
+ def study_seeds(first: int, count: int) -> tuple[int, ...]:
269
+ """The seeds of each cell's episodes: `first`, `first + 1`, ..., `count` of them."""
270
+ if count < 1:
271
+ raise ValueError(f"a study runs at least 1 episode per cell, not {count}")
272
+ if first < 0 or first + count - 1 > MAX_SEED:
273
+ raise ValueError(
274
+ f"a seed is from 0 to {MAX_SEED}, so the seeds cannot run from {first}"
275
+ f" to {first + count - 1}"
276
+ )
277
+ return tuple(range(first, first + count))
278
+
279
+
280
+ def study_episodes(cells: Sequence[StudyCell], seeds: Sequence[int]) -> tuple[Episode, ...]:
281
+ """Every cell with every seed, in the study's order: by seed, then by cell, numbered
282
+ from 1. Raises `ValueError` if two episodes would share a name."""
283
+ pairs = [(cell, seed) for seed in seeds for cell in cells]
284
+ episodes = tuple(
285
+ Episode(number, cell, seed) for number, (cell, seed) in enumerate(pairs, start=1)
286
+ )
287
+ names = [episode.name for episode in episodes]
288
+ if len(set(names)) != len(names):
289
+ raise ValueError("two episodes of the study would share a name")
290
+ return episodes
291
+
292
+
293
+ def database_name(prefix: str, episode: Episode) -> str:
294
+ """The database episode runs in: the prefix and its number, such as `fivefold_study_0001`."""
295
+ return f"{prefix}_{episode.number:04d}"
296
+
297
+
298
+ @dataclass(frozen=True)
299
+ class Study:
300
+ """What a study runs: the scenario, the episodes in order, and the model."""
301
+
302
+ scenario: Scenario
303
+ episodes: tuple[Episode, ...]
304
+ model: ModelSpec
305
+ expected: Expected
306
+ """The expected outcomes, which name the cells; they apply when they cover the scenario."""
307
+
308
+ def describe(self) -> str:
309
+ """The study in one line: its scenario, model, episodes, cells and seeds."""
310
+ if isinstance(self.model, Live):
311
+ model = f"the live model {self.model.name}"
312
+ else:
313
+ model = "the scripted model"
314
+ cells = len({episode.cell.name for episode in self.episodes})
315
+ seeds = sorted({episode.seed for episode in self.episodes})
316
+ seed_text = f"seed {seeds[0]}" if len(seeds) == 1 else f"seeds {seeds[0]} to {seeds[-1]}"
317
+ return (
318
+ f"Study of {self.scenario.id} with {model}: {len(self.episodes)} episodes,"
319
+ f" {cells} cells, {seed_text}."
320
+ )
321
+
322
+
323
+ def check_study(study: Study, prefix: str) -> None:
324
+ """Refuse a study that could not run to its end, before any database is touched.
325
+
326
+ Each episode's contract level must be one the scenario lists, its fault plan must fit
327
+ its configuration and timing as a run checks them (`fivefold.harness.run.run_plan`),
328
+ and its database name must be valid. A cell with a fault runs only on the scenario
329
+ `faults/expected.yaml` is written for: its plan targets that scenario's operation and
330
+ facts, and on another scenario it could fail partway through the study. Raises
331
+ `ValueError`.
332
+ """
333
+ scenario = study.scenario
334
+ faulted = [episode for episode in study.episodes if not isinstance(episode.cell, NoFault)]
335
+ if faulted and study.expected.scenario != scenario.id:
336
+ raise ValueError(
337
+ f"the cells of faults/expected.yaml are written for {study.expected.scenario},"
338
+ f" not {scenario.id}; with another scenario, give --cells {NO_FAULT}"
339
+ )
340
+ for episode in study.episodes:
341
+ spec = episode.spec(scenario)
342
+ if spec.contract not in scenario.contracts:
343
+ levels = ", ".join(map(str, scenario.contracts))
344
+ raise ValueError(f"{scenario.id} runs at {levels}, not at {spec.contract}")
345
+ try:
346
+ run_plan(spec)
347
+ check_database_name(database_name(prefix, episode))
348
+ except (ValueError, MigrationError) as error:
349
+ raise ValueError(f"episode {episode.name}: {error}") from None
350
+
351
+
352
+ def check_databases_free(admin_dsn: str, study: Study, prefix: str) -> None:
353
+ """Refuse a study whose episode databases already exist, such as the episodes another
354
+ study kept, rather than drop them. Raises `ValueError`."""
355
+ names = [database_name(prefix, episode) for episode in study.episodes]
356
+ with psycopg.connect(admin_dsn) as conn:
357
+ rows = conn.execute(
358
+ "SELECT datname FROM pg_database WHERE datname = ANY(%s) ORDER BY datname", (names,)
359
+ ).fetchall()
360
+ if rows:
361
+ raise ValueError(taken(str(rows[0][0])))
362
+
363
+
364
+ def taken(name: str) -> str:
365
+ """Why a study refuses the episode database `name`: it already exists."""
366
+ return (
367
+ f"the database {name} already exists, perhaps an episode another study ran or kept;"
368
+ " give --database-prefix another prefix, or drop it"
369
+ )
370
+
371
+
372
+ def drop_own_database(admin_dsn: str, created: CreatedDatabase) -> None:
373
+ """Drop the database an episode created, if the name is still that database (its
374
+ OID), and then remove its row from the run registry.
375
+
376
+ Both happen under the database lock (`fivefold.harness.migrations.database_lock`),
377
+ which every create takes: a study that creates the name again does it after the row
378
+ is gone, so its own row stays.
379
+ """
380
+ with database_lock(admin_dsn) as conn:
381
+ if drop_database_locked(conn, created.name, oid=created.oid):
382
+ forget(admin_dsn, created.name)
383
+
384
+
385
+ def check_out(out: Path) -> None:
386
+ """Refuse an output directory that is a file, or a directory that holds anything."""
387
+ if out.exists() and (not out.is_dir() or any(out.iterdir())):
388
+ raise ValueError(f"{out} is not a new or empty directory; give the study its own")
389
+
390
+
391
+ # Running -------------------------------------------------------------------------------------
392
+
393
+
394
+ @dataclass(frozen=True)
395
+ class Outcome:
396
+ """How an episode ended, and where its files went."""
397
+
398
+ episode: Episode
399
+ evaluation: Evaluation
400
+ ends: dict[str, tuple[WorkerEnd, str]]
401
+ """How each agent worker ended, and why."""
402
+ result: Path
403
+ cassette: Path | None
404
+ """The live episode's cassette; None with the scripted model."""
405
+ calls: int | None
406
+ """The live episode's model calls, refused ones included; None with the scripted model."""
407
+ refused: int
408
+ """The model calls the spend cap refused; each ends its worker as `budget`."""
409
+ database: str | None
410
+ """The episode's database, when it is kept."""
411
+
412
+ @property
413
+ def cut(self) -> bool:
414
+ """Whether the spend cap cut the episode short: it refused one of its model calls."""
415
+ return self.refused > 0
416
+
417
+
418
+ def refused_calls(cassette: Cassette) -> int:
419
+ """How many of the cassette's calls the spend cap refused (`fivefold.agents.live`)."""
420
+ return sum(
421
+ 1 for call in cassette.calls if refusal(response_message(call.response)) == SPEND_CAP
422
+ )
423
+
424
+
425
+ def run_episode(
426
+ admin_dsn: str, study: Study, episode: Episode, *, out: Path, database: str, keep: bool
427
+ ) -> Outcome:
428
+ """Run `episode` in `database`, a fresh copy of the migrated template database, and
429
+ save its result, and its cassette when the model is live, in `out`. The database is
430
+ dropped once the run ends, unless `keep`; a database the episode dropped leaves the
431
+ run registry too.
432
+
433
+ Raises `ValueError` when `database` already exists: it is someone else's, so it is
434
+ neither replaced nor dropped, and its registry row stays.
435
+ """
436
+ spec = episode.spec(study.scenario, study.model)
437
+ try:
438
+ created = create_run_database(admin_dsn, database, DB_DIR / "migrations")
439
+ except DatabaseExists:
440
+ raise ValueError(taken(database)) from None
441
+ try:
442
+ run = InprocRun(spec, created.dsn)
443
+ asyncio.run(run.execute())
444
+ finally:
445
+ if not keep:
446
+ drop_own_database(admin_dsn, created)
447
+ evaluation = run.result.evaluation
448
+ if evaluation is None:
449
+ raise RuntimeError(f"episode {episode.name} was not evaluated")
450
+ ends = dict(run.result.ends)
451
+ cassette: Cassette | None = None
452
+ if isinstance(spec.model, Live):
453
+ cassette = run.models.cassette
454
+ if cassette is None:
455
+ raise RuntimeError(f"episode {episode.name} recorded no cassette")
456
+ refused = 0 if cassette is None else refused_calls(cassette)
457
+ folder = out / SPEND_CAP_DIR if refused else out
458
+ folder.mkdir(parents=True, exist_ok=True)
459
+ result = folder / f"{episode.name}.json"
460
+ result.write_bytes(result_json(evaluation).encode("utf-8"))
461
+ saved: Path | None = None
462
+ if cassette is not None:
463
+ (out / CASSETTES).mkdir(parents=True, exist_ok=True)
464
+ saved = out / CASSETTES / f"{episode.name}.json"
465
+ cassette.save(saved)
466
+ return Outcome(
467
+ episode=episode,
468
+ evaluation=evaluation,
469
+ ends=ends,
470
+ result=result,
471
+ cassette=saved,
472
+ calls=None if cassette is None else len(cassette.calls),
473
+ refused=refused,
474
+ database=database if keep else None,
475
+ )
476
+
477
+
478
+ def run_study(
479
+ admin_dsn: str, study: Study, *, out: Path, prefix: str, keep: bool = False
480
+ ) -> Iterator[Outcome]:
481
+ """Run the study's episodes in order, each in its own database, and yield how each
482
+ ended. A live study stops once its spend reaches the cap: every later episode's first
483
+ model call would be refused."""
484
+ meter = study.model.spend if isinstance(study.model, Live) else None
485
+ for episode in study.episodes:
486
+ if meter is not None and meter.reached():
487
+ return
488
+ database = database_name(prefix, episode)
489
+ try:
490
+ outcome = run_episode(admin_dsn, study, episode, out=out, database=database, keep=keep)
491
+ except Exception as error:
492
+ error.add_note(f"in episode {episode.number}, {episode.name}")
493
+ raise
494
+ yield outcome
495
+
496
+
497
+ # The command ---------------------------------------------------------------------------------
498
+
499
+
500
+ def episode_line(study: Study, outcome: Outcome) -> str:
501
+ """One line on how an episode ended."""
502
+ episode, evaluation = outcome.episode, outcome.evaluation
503
+ failing = [prop.property for prop in evaluation.properties if str(prop.verdict) == "fail"]
504
+ line = f"[{episode.number}/{len(study.episodes)}] {episode.name}: "
505
+ line += outcome_text(str(evaluation.verdict), failing)
506
+ expected = expected_outcome(episode.cell, study.expected, study.scenario)
507
+ if expected is not None:
508
+ line += f", expected {expected}"
509
+ ends = ", ".join(
510
+ f"{worker} {end}" + ("" if end is WorkerEnd.FINISHED else f" ({text})")
511
+ for worker, (end, text) in sorted(outcome.ends.items())
512
+ )
513
+ line += f"; {ends}"
514
+ meter = study.model.spend if isinstance(study.model, Live) else None
515
+ if outcome.calls is not None and meter is not None:
516
+ line += f"; {outcome.calls} model calls"
517
+ line += f", {outcome.refused} refused" if outcome.refused else ""
518
+ line += f", {meter.spent_usd} USD spent so far"
519
+ if outcome.cut:
520
+ line += f"; cut short by the spend cap, so its result is in {SPEND_CAP_DIR}/"
521
+ if outcome.database is not None:
522
+ line += f"; kept in the database {outcome.database}"
523
+ return line
524
+
525
+
526
+ def plan_study(args: argparse.Namespace, model: ModelSpec) -> Study:
527
+ """The study the command's arguments describe, checked before anything runs."""
528
+ expected = load_expected()
529
+ scenario = load_scenario_file(args.scenario)
530
+ configs = [config for config in CONFIGS if config in args.configs]
531
+ contracts = [contract for contract in Contract if str(contract) in args.contracts]
532
+ cells = select_cells(expected.cells, args.cells, configs=configs, contracts=contracts)
533
+ first = expected.seed if args.seed is None else args.seed
534
+ episodes = study_episodes(cells, study_seeds(first, args.episodes))
535
+ study = Study(scenario=scenario, episodes=episodes, model=model, expected=expected)
536
+ check_study(study, args.database_prefix)
537
+ return study
538
+
539
+
540
+ def main(argv: Sequence[str] | None = None) -> int:
541
+ """Run a study. Returns the process exit status."""
542
+ parser = argparse.ArgumentParser(
543
+ prog="python -m fivefold.harness.study",
544
+ description="Run a matrix of episodes and save each run's result for the report.",
545
+ )
546
+ parser.add_argument(
547
+ "--out", type=Path, required=True, metavar="DIR", help="a new or empty directory"
548
+ )
549
+ parser.add_argument(
550
+ "--scenario", type=Path, default=DEV_001, help="the scenario file (default: dev-001)"
551
+ )
552
+ parser.add_argument(
553
+ "--configs",
554
+ nargs="+",
555
+ choices=CONFIGS,
556
+ default=list(CONFIGS),
557
+ metavar="CONFIG",
558
+ help=f"configurations among {', '.join(CONFIGS)} (default: all three)",
559
+ )
560
+ levels = [str(contract) for contract in Contract]
561
+ parser.add_argument(
562
+ "--contracts",
563
+ nargs="+",
564
+ choices=levels,
565
+ default=levels,
566
+ metavar="LEVEL",
567
+ help=f"contract levels among {', '.join(levels)} (default: all four)",
568
+ )
569
+ parser.add_argument(
570
+ "--cells",
571
+ nargs="+",
572
+ default=[ALL],
573
+ metavar="CELL",
574
+ help=f"{ALL}, {NO_FAULT}, a fault (F2), a fault and variant (F11-after_tool_call),"
575
+ f" or a cell (F2-C2-G2) of faults/expected.yaml (default: {ALL})",
576
+ )
577
+ parser.add_argument(
578
+ "--episodes", type=int, default=1, metavar="N", help="episodes per cell (default 1)"
579
+ )
580
+ parser.add_argument(
581
+ "--seed",
582
+ type=int,
583
+ metavar="S",
584
+ help="the first seed; each cell runs with S, S+1, ... (default: faults/expected.yaml's)",
585
+ )
586
+ parser.add_argument(
587
+ "--database-prefix",
588
+ default=DEFAULT_PREFIX,
589
+ metavar="PREFIX",
590
+ help=f"episode N runs in the database PREFIX_NNNN (default {DEFAULT_PREFIX})",
591
+ )
592
+ parser.add_argument(
593
+ "--keep-databases", action="store_true", help="keep each episode's database"
594
+ )
595
+ parser.add_argument(
596
+ "--dsn",
597
+ default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
598
+ help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
599
+ )
600
+ add_live_options(parser)
601
+ args = parser.parse_args(argv)
602
+ check_live_options(parser, args)
603
+ out: Path = args.out
604
+ try:
605
+ live = live_from_args(args, os.environ)
606
+ study = plan_study(args, SCRIPTED if live is None else live)
607
+ check_out(out)
608
+ check_databases_free(args.dsn, study, args.database_prefix)
609
+ bootstrap_server(args.dsn, DB_DIR)
610
+ print(study.describe(), flush=True)
611
+ ran = 0
612
+ try:
613
+ for outcome in run_study(
614
+ args.dsn, study, out=out, prefix=args.database_prefix, keep=args.keep_databases
615
+ ):
616
+ ran += 1
617
+ print(episode_line(study, outcome), flush=True)
618
+ finally:
619
+ if live is not None and live.spend is not None:
620
+ print(f"Model {live.name}; {live.spend.summary()}.")
621
+ print(end_line(study, ran, out))
622
+ except (OSError, ValueError, MigrationError) as error:
623
+ notes = "".join(f" ({note})" for note in getattr(error, "__notes__", ()))
624
+ print(f"study failed: {error}{notes}", file=sys.stderr)
625
+ return 1
626
+ return 0
627
+
628
+
629
+ def end_line(study: Study, ran: int, out: Path) -> str:
630
+ """How many episodes ran, where their results are, and how to report them."""
631
+ total = len(study.episodes)
632
+ if ran < total:
633
+ text = f"The spend cap stopped the study after {ran} of {total} episodes."
634
+ else:
635
+ text = f"Ran all {total} episodes."
636
+ return (
637
+ f"{text} Results are in {out}. Report them with: python -m fivefold.evaluator"
638
+ f" report {out} --csv runs.csv --markdown report.md"
639
+ )
640
+
641
+
642
+ if __name__ == "__main__":
643
+ sys.exit(main())