fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"""Record a run into a cassette, play a cassette back, and branch a recorded run (0009):
|
|
2
|
+
`python -m fivefold.recorder`.
|
|
3
|
+
|
|
4
|
+
python -m fivefold.recorder record --demo 1 --config G2 --cassette demo-1-g2.json
|
|
5
|
+
python -m fivefold.recorder play demo-1-g2.json \\
|
|
6
|
+
--scenario scenarios/dev/dev-001-single-po.yaml
|
|
7
|
+
FIVEFOLD_LIVE=1 python -m fivefold.recorder record --demo 1 --config G2 \\
|
|
8
|
+
--cassette live.json --live ollama:MODEL --spend-cap-usd CAP --prices PRICES
|
|
9
|
+
python -m fivefold.recorder branch demo-1-g2.json \\
|
|
10
|
+
--scenario scenarios/dev/dev-001-single-po.yaml --at 4 \\
|
|
11
|
+
--cassette branch.json --result branch-result.json
|
|
12
|
+
|
|
13
|
+
`record` runs a demo of record 0006, or a scenario file at a contract level with the
|
|
14
|
+
scenario's own fault list, with the scripted model, recording every model call. It runs in
|
|
15
|
+
a fresh run database (`--database`, default `fivefold_recorded`) and writes the cassette.
|
|
16
|
+
|
|
17
|
+
With `--live PROVIDER:MODEL`, `record` runs that live model instead (`fivefold.agents.live`).
|
|
18
|
+
It refuses, before it touches any database, unless the environment sets `FIVEFOLD_LIVE=1`,
|
|
19
|
+
`--spend-cap-usd` gives a cap above 0 USD, and the price file `--prices` prices the model.
|
|
20
|
+
`--param NAME=VALUE` sets one of the model's parameters, VALUE read as JSON, or as text when
|
|
21
|
+
it is not JSON. The study command takes the same options (`fivefold.harness.live_options`).
|
|
22
|
+
The cap holds across the command's runs; the command prints what the run spent and how
|
|
23
|
+
each agent worker ended, so a provider that could not be reached shows as a worker's
|
|
24
|
+
error. The cassette plays back with `play`, with no network.
|
|
25
|
+
|
|
26
|
+
`play` re-executes the run a cassette was recorded from, with the cassette's responses, in
|
|
27
|
+
another fresh run database (`--database`, default `fivefold_replayed`), and compares its
|
|
28
|
+
records with the original run's database (`--original`, default `fivefold_recorded`). It
|
|
29
|
+
exits with 0 only when the replay matches. It refuses to run in the original run's database
|
|
30
|
+
and only ever reads that one.
|
|
31
|
+
|
|
32
|
+
`branch` runs an experimental branch of the run a cassette was recorded from, the parent:
|
|
33
|
+
the same run, whose first K model calls (`--at K`) come from the parent's cassette, with
|
|
34
|
+
the replay's checks, and whose later calls go to the scripted model, or with `--live` and
|
|
35
|
+
the live options, as `record --live` takes them, to a live model. The branch is a new run,
|
|
36
|
+
in a fresh database (`--database`, default `fivefold_branched`), with its own run id, the
|
|
37
|
+
parent's with `.branch-K` added. It writes its own cassette (`--cassette`), holding all its
|
|
38
|
+
calls, the replayed ones included, whose header names the parent's run id and the branch
|
|
39
|
+
point, and its result JSON (`--result`), for `python -m fivefold.evaluator report`. `play`
|
|
40
|
+
replays the branch's cassette against the branch's database as any other. Before it
|
|
41
|
+
touches any database, `branch` refuses a K beyond the parent's cassette, a scenario other
|
|
42
|
+
than the one the parent was recorded on, a live run without the flag, the spend cap or the
|
|
43
|
+
price file, and outputs that name the parent's cassette or each other.
|
|
44
|
+
|
|
45
|
+
Each command drops the database it runs in, if it exists, and makes it again as a copy of
|
|
46
|
+
the migrated template database (`fivefold.harness.migrations.create_run_database`), as the
|
|
47
|
+
demos do; a database named as the template is refused. The admin connection comes from
|
|
48
|
+
`--dsn`, or `FIVEFOLD_ADMIN_DSN`. Only a live model's provider can need a key, and it
|
|
49
|
+
reads the key from the environment, never from this command. A refused input, such as a
|
|
50
|
+
scenario that changed or a fault plan the configuration does not have, ends the command
|
|
51
|
+
with one line on standard error and exit status 1. So does a template that cannot be
|
|
52
|
+
copied.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
from __future__ import annotations
|
|
56
|
+
|
|
57
|
+
import argparse
|
|
58
|
+
import os
|
|
59
|
+
import sys
|
|
60
|
+
from collections.abc import Sequence
|
|
61
|
+
from dataclasses import replace
|
|
62
|
+
from pathlib import Path
|
|
63
|
+
|
|
64
|
+
import psycopg
|
|
65
|
+
from psycopg.conninfo import make_conninfo
|
|
66
|
+
|
|
67
|
+
from fivefold.agents.replay import ReplayError
|
|
68
|
+
from fivefold.agents.worker import WorkerEnd
|
|
69
|
+
from fivefold.domain.contracts import Contract
|
|
70
|
+
from fivefold.domain.scenario import load_scenario_file
|
|
71
|
+
from fivefold.evaluator import result_json
|
|
72
|
+
from fivefold.harness.demo import DEMO_BY_KEY, verdict_line
|
|
73
|
+
from fivefold.harness.live_options import add_live_options, check_live_options, live_from_args
|
|
74
|
+
from fivefold.harness.migrations import (
|
|
75
|
+
DEFAULT_ADMIN_DSN,
|
|
76
|
+
MigrationError,
|
|
77
|
+
bootstrap_server,
|
|
78
|
+
create_run_database,
|
|
79
|
+
)
|
|
80
|
+
from fivefold.harness.models import SCRIPTED, Live
|
|
81
|
+
from fivefold.harness.run import CONFIGS, RunResult, RunSpec
|
|
82
|
+
from fivefold.paths import DB_DIR
|
|
83
|
+
from fivefold.recorder.cassette import Cassette
|
|
84
|
+
from fivefold.recorder.playback import (
|
|
85
|
+
RECORDS,
|
|
86
|
+
VIEWER,
|
|
87
|
+
PlaybackError,
|
|
88
|
+
branch_spec,
|
|
89
|
+
play,
|
|
90
|
+
record,
|
|
91
|
+
replay_spec,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
RECORDED = "fivefold_recorded"
|
|
95
|
+
REPLAYED = "fivefold_replayed"
|
|
96
|
+
BRANCHED = "fivefold_branched"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _fresh_database(admin_dsn: str, name: str) -> str:
|
|
100
|
+
"""Drop the database `name` if it exists, make it again as a copy of the migrated
|
|
101
|
+
template database, and return its DSN."""
|
|
102
|
+
return create_run_database(admin_dsn, name, DB_DIR / "migrations", replace=True).dsn
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _record(args: argparse.Namespace, live: Live | None) -> int:
|
|
106
|
+
if args.demo is not None:
|
|
107
|
+
spec = DEMO_BY_KEY[args.demo].spec(args.config, args.seed)
|
|
108
|
+
else:
|
|
109
|
+
scenario = load_scenario_file(args.scenario)
|
|
110
|
+
spec = RunSpec(scenario, args.config, Contract(args.contract), args.seed)
|
|
111
|
+
if live is not None:
|
|
112
|
+
spec = replace(spec, model=live)
|
|
113
|
+
dsn = _fresh_database(args.dsn, args.database)
|
|
114
|
+
run, cassette = record(spec, dsn)
|
|
115
|
+
cassette.save(args.cassette)
|
|
116
|
+
print(f"Recorded {spec.run_id} in {args.database}: {len(cassette.calls)} model calls.")
|
|
117
|
+
print(f" verdict: {verdict_line(run.result.evaluation)}")
|
|
118
|
+
print(f" cassette: {args.cassette}, sha256 {cassette.digest()}")
|
|
119
|
+
if live is not None and live.spend is not None:
|
|
120
|
+
print(f" model: {live.name}; {live.spend.summary()}")
|
|
121
|
+
_print_ends(run.result)
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _print_ends(result: RunResult) -> None:
|
|
126
|
+
"""How each agent worker ended, one line each."""
|
|
127
|
+
for worker, (end, text) in sorted(result.ends.items()):
|
|
128
|
+
print(f" {worker}: {end}" + ("" if end is WorkerEnd.FINISHED else f", {text}"))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _same_file(path: Path, other: Path) -> bool:
|
|
132
|
+
"""Whether `path` and `other` name one file: the same path once resolved, or one file
|
|
133
|
+
through a hard link when both exist."""
|
|
134
|
+
if path.resolve() == other.resolve():
|
|
135
|
+
return True
|
|
136
|
+
return path.exists() and other.exists() and path.samefile(other)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _branch_spec(args: argparse.Namespace, live: Live | None) -> tuple[Cassette, RunSpec]:
|
|
140
|
+
"""The parent's cassette and the branch to run, checked before any database is
|
|
141
|
+
touched: the parent's cassette, the scenario, the branch point and the outputs."""
|
|
142
|
+
for output, name in ((args.cassette, "--cassette"), (args.result, "--result")):
|
|
143
|
+
if _same_file(output, args.parent):
|
|
144
|
+
raise ValueError(f"{name} {output} would replace the parent's cassette")
|
|
145
|
+
if _same_file(args.cassette, args.result):
|
|
146
|
+
raise ValueError(f"--cassette and --result name the same file, {args.cassette}")
|
|
147
|
+
parent = Cassette.load(args.parent)
|
|
148
|
+
scenario = load_scenario_file(args.scenario)
|
|
149
|
+
return parent, branch_spec(parent, scenario, args.at, SCRIPTED if live is None else live)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _holds_run(admin_dsn: str, name: str, run_id: str) -> bool:
|
|
153
|
+
"""Whether the database `name` exists and holds the run `run_id`, read as `ff_viewer`.
|
|
154
|
+
A database that does not exist, or holds no run records, holds no run."""
|
|
155
|
+
try:
|
|
156
|
+
with psycopg.connect(make_conninfo(admin_dsn, dbname=name, user=VIEWER)) as conn:
|
|
157
|
+
conn.read_only = True
|
|
158
|
+
found = conn.execute(
|
|
159
|
+
"SELECT 1 FROM harness.run WHERE run_id = %s", (run_id,)
|
|
160
|
+
).fetchone()
|
|
161
|
+
except (psycopg.OperationalError, psycopg.errors.UndefinedTable):
|
|
162
|
+
return False
|
|
163
|
+
return found is not None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _branch(args: argparse.Namespace, parent: Cassette, spec: RunSpec, live: Live | None) -> int:
|
|
167
|
+
if _holds_run(args.dsn, args.database, parent.run.run_id):
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"the database {args.database} holds the parent run {parent.run.run_id};"
|
|
170
|
+
" give --database another name, so the branch does not replace its parent"
|
|
171
|
+
)
|
|
172
|
+
dsn = _fresh_database(args.dsn, args.database)
|
|
173
|
+
run, cassette = record(spec, dsn)
|
|
174
|
+
evaluation = run.result.evaluation
|
|
175
|
+
if evaluation is None:
|
|
176
|
+
raise RuntimeError("the branch was never evaluated")
|
|
177
|
+
cassette.save(args.cassette)
|
|
178
|
+
args.result.write_bytes(result_json(evaluation).encode("utf-8"))
|
|
179
|
+
print(
|
|
180
|
+
f"Branched {spec.run_id} from {parent.run.run_id} at branch point {args.at} in"
|
|
181
|
+
f" {args.database}: {len(cassette.calls)} model calls, the first"
|
|
182
|
+
f" {run.models.served} replayed from the parent's cassette."
|
|
183
|
+
)
|
|
184
|
+
print(f" model after the branch point: {run.models.name}")
|
|
185
|
+
print(f" verdict: {verdict_line(evaluation)}")
|
|
186
|
+
print(f" cassette: {args.cassette}, sha256 {cassette.digest()}")
|
|
187
|
+
print(f" result: {args.result}")
|
|
188
|
+
if live is not None and live.spend is not None:
|
|
189
|
+
print(f" spend: {live.spend.summary()}")
|
|
190
|
+
_print_ends(run.result)
|
|
191
|
+
return 0
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _play(args: argparse.Namespace) -> int:
|
|
195
|
+
cassette = Cassette.load(args.cassette)
|
|
196
|
+
scenario = load_scenario_file(args.scenario)
|
|
197
|
+
replay_spec(cassette, scenario) # refuse a scenario that does not fit before any change
|
|
198
|
+
original = make_conninfo(args.dsn, dbname=args.original)
|
|
199
|
+
replay = _fresh_database(args.dsn, args.database)
|
|
200
|
+
playback = play(cassette, scenario, original=original, replay=replay)
|
|
201
|
+
print(
|
|
202
|
+
f"Replayed {playback.run_id} in {args.database} from the cassette"
|
|
203
|
+
f" {playback.cassette[:16]} ({playback.calls} model calls)."
|
|
204
|
+
)
|
|
205
|
+
print(f" verdict: {playback.verdict}")
|
|
206
|
+
differences = {difference.records: difference for difference in playback.differences}
|
|
207
|
+
for name, _ in RECORDS:
|
|
208
|
+
difference = differences.get(name)
|
|
209
|
+
if difference is None:
|
|
210
|
+
print(f" {name}: identical, sha256 {playback.digests[name][:16]}")
|
|
211
|
+
continue
|
|
212
|
+
print(f" {name}: differs from row {difference.row}")
|
|
213
|
+
print(f" original: {difference.original}")
|
|
214
|
+
print(f" replay: {difference.replay}")
|
|
215
|
+
if playback.matches:
|
|
216
|
+
print(f"The replay matches the original run in {args.original}.")
|
|
217
|
+
return 0
|
|
218
|
+
print(f"The replay does not match the original run in {args.original}.")
|
|
219
|
+
return 1
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
223
|
+
"""Record or play back a run. Returns the process exit status."""
|
|
224
|
+
parser = argparse.ArgumentParser(
|
|
225
|
+
prog="python -m fivefold.recorder",
|
|
226
|
+
description="Record a run's model calls into a cassette, or play a cassette back.",
|
|
227
|
+
)
|
|
228
|
+
parser.add_argument(
|
|
229
|
+
"--dsn",
|
|
230
|
+
default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
|
|
231
|
+
help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
|
|
232
|
+
)
|
|
233
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
234
|
+
|
|
235
|
+
recording = commands.add_parser("record", help="run with the scripted model and record it")
|
|
236
|
+
source = recording.add_mutually_exclusive_group(required=True)
|
|
237
|
+
source.add_argument("--demo", choices=sorted(DEMO_BY_KEY), help="a demo of record 0006")
|
|
238
|
+
source.add_argument("--scenario", type=Path, help="a scenario file, with its own faults")
|
|
239
|
+
recording.add_argument(
|
|
240
|
+
"--contract", choices=[str(level) for level in Contract], help="for a scenario file"
|
|
241
|
+
)
|
|
242
|
+
recording.add_argument("--config", choices=CONFIGS, required=True)
|
|
243
|
+
recording.add_argument("--seed", type=int, default=7, help="the run seed (default 7)")
|
|
244
|
+
recording.add_argument("--cassette", type=Path, required=True, help="where to write it")
|
|
245
|
+
recording.add_argument(
|
|
246
|
+
"--database", default=RECORDED, help=f"the run's database (default {RECORDED})"
|
|
247
|
+
)
|
|
248
|
+
add_live_options(recording)
|
|
249
|
+
|
|
250
|
+
playing = commands.add_parser("play", help="replay a cassette and compare the records")
|
|
251
|
+
playing.add_argument("cassette", type=Path)
|
|
252
|
+
playing.add_argument("--scenario", type=Path, required=True, help="the run's scenario file")
|
|
253
|
+
playing.add_argument(
|
|
254
|
+
"--original", default=RECORDED, help=f"the original run's database (default {RECORDED})"
|
|
255
|
+
)
|
|
256
|
+
playing.add_argument(
|
|
257
|
+
"--database", default=REPLAYED, help=f"the replay's database (default {REPLAYED})"
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
branching = commands.add_parser(
|
|
261
|
+
"branch",
|
|
262
|
+
help="replay a cassette's first K model calls, then switch model, as a new run",
|
|
263
|
+
)
|
|
264
|
+
branching.add_argument("parent", type=Path, help="the parent run's cassette")
|
|
265
|
+
branching.add_argument(
|
|
266
|
+
"--scenario", type=Path, required=True, help="the parent run's scenario file"
|
|
267
|
+
)
|
|
268
|
+
branching.add_argument(
|
|
269
|
+
"--at",
|
|
270
|
+
type=int,
|
|
271
|
+
required=True,
|
|
272
|
+
metavar="K",
|
|
273
|
+
help="the branch point: how many of the parent's model calls to replay",
|
|
274
|
+
)
|
|
275
|
+
branching.add_argument(
|
|
276
|
+
"--cassette", type=Path, required=True, help="where to write the branch's cassette"
|
|
277
|
+
)
|
|
278
|
+
branching.add_argument(
|
|
279
|
+
"--result", type=Path, required=True, help="where to write the branch's result JSON"
|
|
280
|
+
)
|
|
281
|
+
branching.add_argument(
|
|
282
|
+
"--database", default=BRANCHED, help=f"the branch's database (default {BRANCHED})"
|
|
283
|
+
)
|
|
284
|
+
add_live_options(branching)
|
|
285
|
+
|
|
286
|
+
args = parser.parse_args(argv)
|
|
287
|
+
if args.command == "record" and args.scenario is not None and args.contract is None:
|
|
288
|
+
parser.error("record --scenario needs --contract")
|
|
289
|
+
if args.command == "record" and args.demo is not None and args.contract is not None:
|
|
290
|
+
parser.error("a demo has its own contract level; leave out --contract")
|
|
291
|
+
if args.command == "play" and args.database == args.original:
|
|
292
|
+
parser.error("the replay needs its own database, not the original run's")
|
|
293
|
+
if args.command in ("record", "branch"):
|
|
294
|
+
check_live_options(parser, args)
|
|
295
|
+
try:
|
|
296
|
+
live = live_from_args(args, os.environ) if args.command in ("record", "branch") else None
|
|
297
|
+
branch = _branch_spec(args, live) if args.command == "branch" else None
|
|
298
|
+
bootstrap_server(args.dsn, DB_DIR)
|
|
299
|
+
if branch is not None:
|
|
300
|
+
return _branch(args, *branch, live)
|
|
301
|
+
return _record(args, live) if args.command == "record" else _play(args)
|
|
302
|
+
except (OSError, MigrationError, PlaybackError, ReplayError, ValueError) as error:
|
|
303
|
+
print(f"{args.command} failed: {error}", file=sys.stderr)
|
|
304
|
+
return 1
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
if __name__ == "__main__":
|
|
308
|
+
sys.exit(main())
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""Model cassettes (0009): the model calls of one run, in order, as one stable file.
|
|
2
|
+
|
|
3
|
+
A cassette holds what a replay needs to serve a run its model's answers, and to check
|
|
4
|
+
that it serves the same run:
|
|
5
|
+
|
|
6
|
+
- `format`: `fivefold-cassette/1`.
|
|
7
|
+
- `run`: the run it was recorded from, as `harness.run` records it: the run id, the
|
|
8
|
+
scenario's id and digest, the configuration, the contract level, the seed, the
|
|
9
|
+
topology, and the run's parameters (the scenario's parameters, the fault plan and the
|
|
10
|
+
workers). A branch's header also names its parent's run id and its branch point
|
|
11
|
+
(`parent_run_id` and `branch_point`); any other run's leaves both out, so its file is
|
|
12
|
+
the same as before branches existed.
|
|
13
|
+
- `model`: the model that answered: its adapter (`scripted` or `live`), its name and its
|
|
14
|
+
parameters. A branch names the model it switched to, and hashes every call with it,
|
|
15
|
+
the calls it replayed from its parent included (`fivefold.agents.branch`).
|
|
16
|
+
- `calls`: every model call, in the order the run's workers made them: the worker that
|
|
17
|
+
asked, the hash of the request, and the response.
|
|
18
|
+
|
|
19
|
+
The request hash is the SHA-256 of the request's canonical JSON,
|
|
20
|
+
`{"messages": [...], "params": {...}, "tools": [...]}`:
|
|
21
|
+
|
|
22
|
+
- messages keep what a model is sent: the type, the content, the name, an AI message's
|
|
23
|
+
tool calls, a tool message's call id and status, a chat message's role, and any
|
|
24
|
+
provider fields in `additional_kwargs`. Never their ids: LangChain and LangGraph draw
|
|
25
|
+
those at random.
|
|
26
|
+
- tools are the schemas bound to the model, in the OpenAI tool format that langchain-core
|
|
27
|
+
writes for every kind of tool. The agent's graph binds the worker's tools to every
|
|
28
|
+
adapter, so a run's requests carry them.
|
|
29
|
+
- params are the model's name and parameters, what was bound with the tools (such as
|
|
30
|
+
`tool_choice`), and the call's own arguments, `stop` among them.
|
|
31
|
+
|
|
32
|
+
Canonical JSON has sorted keys, no spaces, non-ASCII characters as they are, and no NaN or
|
|
33
|
+
infinity. The file is the cassette as JSON with sorted keys, two-space indents, non-ASCII
|
|
34
|
+
characters as they are and a final newline, so one cassette has one text. Its digest, the
|
|
35
|
+
SHA-256 of that text, is what `harness.run.cassette_digest` records. Loading refuses a
|
|
36
|
+
file in any other form, so a file's SHA-256 is always its cassette's digest.
|
|
37
|
+
|
|
38
|
+
Checked in langchain-core 1.8.0: the message fields (`messages/base.py`, `messages/ai.py`,
|
|
39
|
+
`messages/tool.py`, `messages/chat.py`); `AIMessage`'s validator, which gives every tool
|
|
40
|
+
call its `type` (`messages/ai.py`); `convert_to_openai_tool(tool, *, strict=None)`
|
|
41
|
+
(`utils/function_calling.py`). Checked in pydantic 2.14.3: `JsonValue` (`types.py`),
|
|
42
|
+
`BaseModel.model_validate_json` (`main.py`), `Field(ge=...)` (`fields.py`) and
|
|
43
|
+
`model_validator(mode="after")` (`functional_validators.py`).
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import hashlib
|
|
49
|
+
import json
|
|
50
|
+
import math
|
|
51
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
from typing import Any, Final, Literal
|
|
54
|
+
|
|
55
|
+
from langchain_core.messages import AIMessage, BaseMessage, ChatMessage, ToolMessage
|
|
56
|
+
from langchain_core.tools import BaseTool
|
|
57
|
+
from langchain_core.utils.function_calling import convert_to_openai_tool
|
|
58
|
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator
|
|
59
|
+
|
|
60
|
+
FORMAT: Final = "fivefold-cassette/1"
|
|
61
|
+
"""The format a cassette file names, so a later format can refuse an old file clearly."""
|
|
62
|
+
|
|
63
|
+
BRANCH_FIELDS: Final = ("parent_run_id", "branch_point")
|
|
64
|
+
"""The header fields only a branch's cassette writes (0009, Playback and branching)."""
|
|
65
|
+
|
|
66
|
+
Tool = dict[str, Any] | type | Callable[..., Any] | BaseTool
|
|
67
|
+
"""Anything `bind_tools` takes (langchain-core 1.8.0, `BaseChatModel.bind_tools`)."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class CassetteError(ValueError):
|
|
71
|
+
"""A file that is not a cassette in its canonical form."""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# Canonical JSON ---------------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def plain(value: object, where: str = "value") -> JsonValue:
|
|
78
|
+
"""`value` as plain JSON data: dicts with text keys, lists, text, numbers, booleans and None.
|
|
79
|
+
|
|
80
|
+
Mappings become dicts and tuples become lists. Raises `TypeError` or `ValueError`,
|
|
81
|
+
naming the place, for anything JSON cannot hold: other types, keys that are not text,
|
|
82
|
+
NaN and infinity.
|
|
83
|
+
"""
|
|
84
|
+
if value is None or isinstance(value, str | bool | int):
|
|
85
|
+
return value
|
|
86
|
+
if isinstance(value, float):
|
|
87
|
+
if not math.isfinite(value):
|
|
88
|
+
raise ValueError(f"{where} is {value}, which JSON cannot hold")
|
|
89
|
+
return value
|
|
90
|
+
if isinstance(value, Mapping):
|
|
91
|
+
return plain_object(value, where)
|
|
92
|
+
if isinstance(value, list | tuple):
|
|
93
|
+
return [plain(item, f"{where}[{index}]") for index, item in enumerate(value)]
|
|
94
|
+
raise TypeError(f"{where} is a {type(value).__name__}, which JSON cannot hold")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def plain_object(value: Mapping[Any, Any], where: str = "value") -> dict[str, JsonValue]:
|
|
98
|
+
"""A mapping as a plain JSON object; see `plain`."""
|
|
99
|
+
data: dict[str, JsonValue] = {}
|
|
100
|
+
for key, item in value.items():
|
|
101
|
+
if not isinstance(key, str):
|
|
102
|
+
raise TypeError(f"{where} has a key that is not text: {key!r}")
|
|
103
|
+
data[key] = plain(item, f"{where}.{key}")
|
|
104
|
+
return data
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def canonical_json(value: object) -> str:
|
|
108
|
+
"""`value` as canonical JSON: sorted keys, no spaces, non-ASCII characters as they are."""
|
|
109
|
+
return json.dumps(
|
|
110
|
+
plain(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def sha256(text: str) -> str:
|
|
115
|
+
"""The SHA-256 of `text` in UTF-8, as 64 lowercase hex digits."""
|
|
116
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# Requests and responses -------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def message_json(message: BaseMessage) -> dict[str, JsonValue]:
|
|
123
|
+
"""What a model is sent of `message`, as plain JSON. Never its id, which is random."""
|
|
124
|
+
data: dict[str, object] = {"type": message.type, "content": message.content}
|
|
125
|
+
if message.name is not None:
|
|
126
|
+
data["name"] = message.name
|
|
127
|
+
if message.additional_kwargs:
|
|
128
|
+
data["additional_kwargs"] = message.additional_kwargs
|
|
129
|
+
if isinstance(message, AIMessage):
|
|
130
|
+
if message.tool_calls:
|
|
131
|
+
data["tool_calls"] = [
|
|
132
|
+
{"name": call["name"], "args": call["args"], "id": call["id"]}
|
|
133
|
+
for call in message.tool_calls
|
|
134
|
+
]
|
|
135
|
+
if message.invalid_tool_calls:
|
|
136
|
+
data["invalid_tool_calls"] = list(message.invalid_tool_calls)
|
|
137
|
+
if isinstance(message, ToolMessage):
|
|
138
|
+
data["tool_call_id"] = message.tool_call_id
|
|
139
|
+
data["status"] = message.status
|
|
140
|
+
if isinstance(message, ChatMessage):
|
|
141
|
+
data["role"] = message.role
|
|
142
|
+
return plain_object(data, f"a {message.type} message")
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def tool_schemas(tools: Sequence[Tool]) -> list[JsonValue]:
|
|
146
|
+
"""The schemas of the tools bound to a model, in the OpenAI tool format."""
|
|
147
|
+
return [
|
|
148
|
+
plain(convert_to_openai_tool(tool), f"tool {index}") for index, tool in enumerate(tools)
|
|
149
|
+
]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def request_params(
|
|
153
|
+
model: ModelInfo,
|
|
154
|
+
bound: Mapping[str, object],
|
|
155
|
+
stop: Sequence[str] | None,
|
|
156
|
+
call: Mapping[str, object],
|
|
157
|
+
) -> dict[str, JsonValue]:
|
|
158
|
+
"""The parameters of one model call: the model's name and parameters, what was bound
|
|
159
|
+
with its tools, and the call's own arguments, `stop` among them."""
|
|
160
|
+
params: dict[str, object] = {"model": model.name, **model.params, **bound, **call}
|
|
161
|
+
if stop is not None:
|
|
162
|
+
params["stop"] = list(stop)
|
|
163
|
+
return plain_object(params, "the call's parameters")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def request_json(
|
|
167
|
+
messages: Sequence[BaseMessage],
|
|
168
|
+
*,
|
|
169
|
+
tools: Sequence[JsonValue],
|
|
170
|
+
params: Mapping[str, JsonValue],
|
|
171
|
+
) -> str:
|
|
172
|
+
"""A request's canonical JSON: the messages, the tools and the params."""
|
|
173
|
+
request = {
|
|
174
|
+
"messages": [message_json(message) for message in messages],
|
|
175
|
+
"params": dict(params),
|
|
176
|
+
"tools": list(tools),
|
|
177
|
+
}
|
|
178
|
+
return canonical_json(request)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def request_hash(
|
|
182
|
+
messages: Sequence[BaseMessage],
|
|
183
|
+
*,
|
|
184
|
+
tools: Sequence[JsonValue],
|
|
185
|
+
params: Mapping[str, JsonValue],
|
|
186
|
+
) -> str:
|
|
187
|
+
"""The SHA-256 of a request's canonical JSON (`request_json`)."""
|
|
188
|
+
return sha256(request_json(messages, tools=tools, params=params))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def response_json(reply: AIMessage) -> dict[str, JsonValue]:
|
|
192
|
+
"""A model's reply as a cassette keeps it: everything but its id, which is random.
|
|
193
|
+
|
|
194
|
+
Fields that are empty are left out, so the file stays short.
|
|
195
|
+
"""
|
|
196
|
+
data: dict[str, object] = {"content": reply.content}
|
|
197
|
+
if reply.tool_calls:
|
|
198
|
+
data["tool_calls"] = list(reply.tool_calls)
|
|
199
|
+
if reply.invalid_tool_calls:
|
|
200
|
+
data["invalid_tool_calls"] = list(reply.invalid_tool_calls)
|
|
201
|
+
if reply.additional_kwargs:
|
|
202
|
+
data["additional_kwargs"] = reply.additional_kwargs
|
|
203
|
+
if reply.response_metadata:
|
|
204
|
+
data["response_metadata"] = reply.response_metadata
|
|
205
|
+
if reply.usage_metadata is not None:
|
|
206
|
+
data["usage_metadata"] = dict(reply.usage_metadata)
|
|
207
|
+
if reply.name is not None:
|
|
208
|
+
data["name"] = reply.name
|
|
209
|
+
return plain_object(data, "the reply")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def response_message(response: Mapping[str, JsonValue]) -> AIMessage:
|
|
213
|
+
"""The reply a cassette kept, as the model returned it, without its id."""
|
|
214
|
+
return AIMessage.model_validate(dict(response))
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# The cassette -----------------------------------------------------------------------------
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class _Part(BaseModel):
|
|
221
|
+
"""Base of a cassette's parts: unknown fields are refused and values never change."""
|
|
222
|
+
|
|
223
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class RunHeader(_Part):
|
|
227
|
+
"""The run a cassette was recorded from, as `harness.run` records it (0009)."""
|
|
228
|
+
|
|
229
|
+
run_id: str
|
|
230
|
+
scenario_id: str
|
|
231
|
+
scenario_digest: str
|
|
232
|
+
config: Literal["B0", "G1", "G2"]
|
|
233
|
+
contract: Literal["C0", "C1", "C2", "C3"]
|
|
234
|
+
seed: int
|
|
235
|
+
topology: Literal["inproc", "split"]
|
|
236
|
+
params: dict[str, JsonValue]
|
|
237
|
+
"""The values the run used: the scenario's parameters, the fault plan and the workers."""
|
|
238
|
+
parent_run_id: str | None = None
|
|
239
|
+
"""For a branch, the run id of the run it branched from; None for any other run."""
|
|
240
|
+
branch_point: int | None = Field(default=None, ge=0)
|
|
241
|
+
"""For a branch, K: how many of its parent's model calls it replayed before it
|
|
242
|
+
switched model; None for any other run."""
|
|
243
|
+
|
|
244
|
+
@model_validator(mode="after")
|
|
245
|
+
def _parent_and_point_together(self) -> RunHeader:
|
|
246
|
+
"""A branch names both its parent and its branch point; any other run neither, as
|
|
247
|
+
`harness.run` requires."""
|
|
248
|
+
if (self.parent_run_id is None) != (self.branch_point is None):
|
|
249
|
+
raise ValueError("a branch names both its parent_run_id and its branch_point")
|
|
250
|
+
return self
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class ModelInfo(_Part):
|
|
254
|
+
"""The model that answered the calls: its adapter, its name and its parameters."""
|
|
255
|
+
|
|
256
|
+
adapter: Literal["scripted", "live"]
|
|
257
|
+
name: str
|
|
258
|
+
params: dict[str, JsonValue] = Field(default_factory=dict)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
class Call(_Part):
|
|
262
|
+
"""One model call: the worker that asked, the request's hash and the response."""
|
|
263
|
+
|
|
264
|
+
worker: str
|
|
265
|
+
request: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
266
|
+
response: dict[str, JsonValue]
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class Cassette(_Part):
|
|
270
|
+
"""The model calls of one run, in the order the run made them (0009)."""
|
|
271
|
+
|
|
272
|
+
format: Literal["fivefold-cassette/1"] = FORMAT
|
|
273
|
+
run: RunHeader
|
|
274
|
+
model: ModelInfo
|
|
275
|
+
calls: tuple[Call, ...]
|
|
276
|
+
|
|
277
|
+
def text(self) -> str:
|
|
278
|
+
"""The cassette's file: JSON with sorted keys, two-space indents, non-ASCII
|
|
279
|
+
characters as they are, and a final newline. A run that is no branch leaves out the
|
|
280
|
+
header's branch fields, so its file is the same as before branches existed."""
|
|
281
|
+
data = self.model_dump(mode="json")
|
|
282
|
+
if self.run.parent_run_id is None:
|
|
283
|
+
for name in BRANCH_FIELDS:
|
|
284
|
+
del data["run"][name]
|
|
285
|
+
return (
|
|
286
|
+
json.dumps(data, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
def digest(self) -> str:
|
|
290
|
+
"""The SHA-256 of the cassette's file, as `harness.run.cassette_digest` records it."""
|
|
291
|
+
return sha256(self.text())
|
|
292
|
+
|
|
293
|
+
@classmethod
|
|
294
|
+
def from_text(cls, text: str) -> Cassette:
|
|
295
|
+
"""Read a cassette's file; refuse one that is not a cassette in canonical form."""
|
|
296
|
+
try:
|
|
297
|
+
cassette = cls.model_validate_json(text)
|
|
298
|
+
except ValidationError as error:
|
|
299
|
+
raise CassetteError(f"not a cassette: {error}") from error
|
|
300
|
+
try:
|
|
301
|
+
canonical = cassette.text()
|
|
302
|
+
except ValueError as error:
|
|
303
|
+
raise CassetteError(f"not a cassette: {error}") from error
|
|
304
|
+
if canonical != text:
|
|
305
|
+
raise CassetteError(
|
|
306
|
+
"the cassette is not in canonical form: sorted keys, two-space indents, "
|
|
307
|
+
"non-ASCII characters as they are, and a final newline"
|
|
308
|
+
)
|
|
309
|
+
return cassette
|
|
310
|
+
|
|
311
|
+
def save(self, path: Path) -> None:
|
|
312
|
+
"""Write the cassette's file, in UTF-8, with no line-ending translation."""
|
|
313
|
+
path.write_bytes(self.text().encode("utf-8"))
|
|
314
|
+
|
|
315
|
+
@classmethod
|
|
316
|
+
def load(cls, path: Path) -> Cassette:
|
|
317
|
+
"""Read and check a cassette's file."""
|
|
318
|
+
try:
|
|
319
|
+
text = path.read_bytes().decode("utf-8")
|
|
320
|
+
except UnicodeDecodeError as error:
|
|
321
|
+
raise CassetteError(f"{path} is not UTF-8 text") from error
|
|
322
|
+
return cls.from_text(text)
|