fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,308 @@
1
+ """Record a run into a cassette, play a cassette back, and branch a recorded run (0009):
2
+ `python -m fivefold.recorder`.
3
+
4
+ python -m fivefold.recorder record --demo 1 --config G2 --cassette demo-1-g2.json
5
+ python -m fivefold.recorder play demo-1-g2.json \\
6
+ --scenario scenarios/dev/dev-001-single-po.yaml
7
+ FIVEFOLD_LIVE=1 python -m fivefold.recorder record --demo 1 --config G2 \\
8
+ --cassette live.json --live ollama:MODEL --spend-cap-usd CAP --prices PRICES
9
+ python -m fivefold.recorder branch demo-1-g2.json \\
10
+ --scenario scenarios/dev/dev-001-single-po.yaml --at 4 \\
11
+ --cassette branch.json --result branch-result.json
12
+
13
+ `record` runs a demo of record 0006, or a scenario file at a contract level with the
14
+ scenario's own fault list, with the scripted model, recording every model call. It runs in
15
+ a fresh run database (`--database`, default `fivefold_recorded`) and writes the cassette.
16
+
17
+ With `--live PROVIDER:MODEL`, `record` runs that live model instead (`fivefold.agents.live`).
18
+ It refuses, before it touches any database, unless the environment sets `FIVEFOLD_LIVE=1`,
19
+ `--spend-cap-usd` gives a cap above 0 USD, and the price file `--prices` prices the model.
20
+ `--param NAME=VALUE` sets one of the model's parameters, VALUE read as JSON, or as text when
21
+ it is not JSON. The study command takes the same options (`fivefold.harness.live_options`).
22
+ The cap holds across the command's runs; the command prints what the run spent and how
23
+ each agent worker ended, so a provider that could not be reached shows as a worker's
24
+ error. The cassette plays back with `play`, with no network.
25
+
26
+ `play` re-executes the run a cassette was recorded from, with the cassette's responses, in
27
+ another fresh run database (`--database`, default `fivefold_replayed`), and compares its
28
+ records with the original run's database (`--original`, default `fivefold_recorded`). It
29
+ exits with 0 only when the replay matches. It refuses to run in the original run's database
30
+ and only ever reads that one.
31
+
32
+ `branch` runs an experimental branch of the run a cassette was recorded from, the parent:
33
+ the same run, whose first K model calls (`--at K`) come from the parent's cassette, with
34
+ the replay's checks, and whose later calls go to the scripted model, or with `--live` and
35
+ the live options, as `record --live` takes them, to a live model. The branch is a new run,
36
+ in a fresh database (`--database`, default `fivefold_branched`), with its own run id, the
37
+ parent's with `.branch-K` added. It writes its own cassette (`--cassette`), holding all its
38
+ calls, the replayed ones included, whose header names the parent's run id and the branch
39
+ point, and its result JSON (`--result`), for `python -m fivefold.evaluator report`. `play`
40
+ replays the branch's cassette against the branch's database as any other. Before it
41
+ touches any database, `branch` refuses a K beyond the parent's cassette, a scenario other
42
+ than the one the parent was recorded on, a live run without the flag, the spend cap or the
43
+ price file, and outputs that name the parent's cassette or each other.
44
+
45
+ Each command drops the database it runs in, if it exists, and makes it again as a copy of
46
+ the migrated template database (`fivefold.harness.migrations.create_run_database`), as the
47
+ demos do; a database named as the template is refused. The admin connection comes from
48
+ `--dsn`, or `FIVEFOLD_ADMIN_DSN`. Only a live model's provider can need a key, and it
49
+ reads the key from the environment, never from this command. A refused input, such as a
50
+ scenario that changed or a fault plan the configuration does not have, ends the command
51
+ with one line on standard error and exit status 1. So does a template that cannot be
52
+ copied.
53
+ """
54
+
55
+ from __future__ import annotations
56
+
57
+ import argparse
58
+ import os
59
+ import sys
60
+ from collections.abc import Sequence
61
+ from dataclasses import replace
62
+ from pathlib import Path
63
+
64
+ import psycopg
65
+ from psycopg.conninfo import make_conninfo
66
+
67
+ from fivefold.agents.replay import ReplayError
68
+ from fivefold.agents.worker import WorkerEnd
69
+ from fivefold.domain.contracts import Contract
70
+ from fivefold.domain.scenario import load_scenario_file
71
+ from fivefold.evaluator import result_json
72
+ from fivefold.harness.demo import DEMO_BY_KEY, verdict_line
73
+ from fivefold.harness.live_options import add_live_options, check_live_options, live_from_args
74
+ from fivefold.harness.migrations import (
75
+ DEFAULT_ADMIN_DSN,
76
+ MigrationError,
77
+ bootstrap_server,
78
+ create_run_database,
79
+ )
80
+ from fivefold.harness.models import SCRIPTED, Live
81
+ from fivefold.harness.run import CONFIGS, RunResult, RunSpec
82
+ from fivefold.paths import DB_DIR
83
+ from fivefold.recorder.cassette import Cassette
84
+ from fivefold.recorder.playback import (
85
+ RECORDS,
86
+ VIEWER,
87
+ PlaybackError,
88
+ branch_spec,
89
+ play,
90
+ record,
91
+ replay_spec,
92
+ )
93
+
94
+ RECORDED = "fivefold_recorded"
95
+ REPLAYED = "fivefold_replayed"
96
+ BRANCHED = "fivefold_branched"
97
+
98
+
99
+ def _fresh_database(admin_dsn: str, name: str) -> str:
100
+ """Drop the database `name` if it exists, make it again as a copy of the migrated
101
+ template database, and return its DSN."""
102
+ return create_run_database(admin_dsn, name, DB_DIR / "migrations", replace=True).dsn
103
+
104
+
105
+ def _record(args: argparse.Namespace, live: Live | None) -> int:
106
+ if args.demo is not None:
107
+ spec = DEMO_BY_KEY[args.demo].spec(args.config, args.seed)
108
+ else:
109
+ scenario = load_scenario_file(args.scenario)
110
+ spec = RunSpec(scenario, args.config, Contract(args.contract), args.seed)
111
+ if live is not None:
112
+ spec = replace(spec, model=live)
113
+ dsn = _fresh_database(args.dsn, args.database)
114
+ run, cassette = record(spec, dsn)
115
+ cassette.save(args.cassette)
116
+ print(f"Recorded {spec.run_id} in {args.database}: {len(cassette.calls)} model calls.")
117
+ print(f" verdict: {verdict_line(run.result.evaluation)}")
118
+ print(f" cassette: {args.cassette}, sha256 {cassette.digest()}")
119
+ if live is not None and live.spend is not None:
120
+ print(f" model: {live.name}; {live.spend.summary()}")
121
+ _print_ends(run.result)
122
+ return 0
123
+
124
+
125
+ def _print_ends(result: RunResult) -> None:
126
+ """How each agent worker ended, one line each."""
127
+ for worker, (end, text) in sorted(result.ends.items()):
128
+ print(f" {worker}: {end}" + ("" if end is WorkerEnd.FINISHED else f", {text}"))
129
+
130
+
131
+ def _same_file(path: Path, other: Path) -> bool:
132
+ """Whether `path` and `other` name one file: the same path once resolved, or one file
133
+ through a hard link when both exist."""
134
+ if path.resolve() == other.resolve():
135
+ return True
136
+ return path.exists() and other.exists() and path.samefile(other)
137
+
138
+
139
+ def _branch_spec(args: argparse.Namespace, live: Live | None) -> tuple[Cassette, RunSpec]:
140
+ """The parent's cassette and the branch to run, checked before any database is
141
+ touched: the parent's cassette, the scenario, the branch point and the outputs."""
142
+ for output, name in ((args.cassette, "--cassette"), (args.result, "--result")):
143
+ if _same_file(output, args.parent):
144
+ raise ValueError(f"{name} {output} would replace the parent's cassette")
145
+ if _same_file(args.cassette, args.result):
146
+ raise ValueError(f"--cassette and --result name the same file, {args.cassette}")
147
+ parent = Cassette.load(args.parent)
148
+ scenario = load_scenario_file(args.scenario)
149
+ return parent, branch_spec(parent, scenario, args.at, SCRIPTED if live is None else live)
150
+
151
+
152
+ def _holds_run(admin_dsn: str, name: str, run_id: str) -> bool:
153
+ """Whether the database `name` exists and holds the run `run_id`, read as `ff_viewer`.
154
+ A database that does not exist, or holds no run records, holds no run."""
155
+ try:
156
+ with psycopg.connect(make_conninfo(admin_dsn, dbname=name, user=VIEWER)) as conn:
157
+ conn.read_only = True
158
+ found = conn.execute(
159
+ "SELECT 1 FROM harness.run WHERE run_id = %s", (run_id,)
160
+ ).fetchone()
161
+ except (psycopg.OperationalError, psycopg.errors.UndefinedTable):
162
+ return False
163
+ return found is not None
164
+
165
+
166
+ def _branch(args: argparse.Namespace, parent: Cassette, spec: RunSpec, live: Live | None) -> int:
167
+ if _holds_run(args.dsn, args.database, parent.run.run_id):
168
+ raise ValueError(
169
+ f"the database {args.database} holds the parent run {parent.run.run_id};"
170
+ " give --database another name, so the branch does not replace its parent"
171
+ )
172
+ dsn = _fresh_database(args.dsn, args.database)
173
+ run, cassette = record(spec, dsn)
174
+ evaluation = run.result.evaluation
175
+ if evaluation is None:
176
+ raise RuntimeError("the branch was never evaluated")
177
+ cassette.save(args.cassette)
178
+ args.result.write_bytes(result_json(evaluation).encode("utf-8"))
179
+ print(
180
+ f"Branched {spec.run_id} from {parent.run.run_id} at branch point {args.at} in"
181
+ f" {args.database}: {len(cassette.calls)} model calls, the first"
182
+ f" {run.models.served} replayed from the parent's cassette."
183
+ )
184
+ print(f" model after the branch point: {run.models.name}")
185
+ print(f" verdict: {verdict_line(evaluation)}")
186
+ print(f" cassette: {args.cassette}, sha256 {cassette.digest()}")
187
+ print(f" result: {args.result}")
188
+ if live is not None and live.spend is not None:
189
+ print(f" spend: {live.spend.summary()}")
190
+ _print_ends(run.result)
191
+ return 0
192
+
193
+
194
+ def _play(args: argparse.Namespace) -> int:
195
+ cassette = Cassette.load(args.cassette)
196
+ scenario = load_scenario_file(args.scenario)
197
+ replay_spec(cassette, scenario) # refuse a scenario that does not fit before any change
198
+ original = make_conninfo(args.dsn, dbname=args.original)
199
+ replay = _fresh_database(args.dsn, args.database)
200
+ playback = play(cassette, scenario, original=original, replay=replay)
201
+ print(
202
+ f"Replayed {playback.run_id} in {args.database} from the cassette"
203
+ f" {playback.cassette[:16]} ({playback.calls} model calls)."
204
+ )
205
+ print(f" verdict: {playback.verdict}")
206
+ differences = {difference.records: difference for difference in playback.differences}
207
+ for name, _ in RECORDS:
208
+ difference = differences.get(name)
209
+ if difference is None:
210
+ print(f" {name}: identical, sha256 {playback.digests[name][:16]}")
211
+ continue
212
+ print(f" {name}: differs from row {difference.row}")
213
+ print(f" original: {difference.original}")
214
+ print(f" replay: {difference.replay}")
215
+ if playback.matches:
216
+ print(f"The replay matches the original run in {args.original}.")
217
+ return 0
218
+ print(f"The replay does not match the original run in {args.original}.")
219
+ return 1
220
+
221
+
222
+ def main(argv: Sequence[str] | None = None) -> int:
223
+ """Record or play back a run. Returns the process exit status."""
224
+ parser = argparse.ArgumentParser(
225
+ prog="python -m fivefold.recorder",
226
+ description="Record a run's model calls into a cassette, or play a cassette back.",
227
+ )
228
+ parser.add_argument(
229
+ "--dsn",
230
+ default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
231
+ help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
232
+ )
233
+ commands = parser.add_subparsers(dest="command", required=True)
234
+
235
+ recording = commands.add_parser("record", help="run with the scripted model and record it")
236
+ source = recording.add_mutually_exclusive_group(required=True)
237
+ source.add_argument("--demo", choices=sorted(DEMO_BY_KEY), help="a demo of record 0006")
238
+ source.add_argument("--scenario", type=Path, help="a scenario file, with its own faults")
239
+ recording.add_argument(
240
+ "--contract", choices=[str(level) for level in Contract], help="for a scenario file"
241
+ )
242
+ recording.add_argument("--config", choices=CONFIGS, required=True)
243
+ recording.add_argument("--seed", type=int, default=7, help="the run seed (default 7)")
244
+ recording.add_argument("--cassette", type=Path, required=True, help="where to write it")
245
+ recording.add_argument(
246
+ "--database", default=RECORDED, help=f"the run's database (default {RECORDED})"
247
+ )
248
+ add_live_options(recording)
249
+
250
+ playing = commands.add_parser("play", help="replay a cassette and compare the records")
251
+ playing.add_argument("cassette", type=Path)
252
+ playing.add_argument("--scenario", type=Path, required=True, help="the run's scenario file")
253
+ playing.add_argument(
254
+ "--original", default=RECORDED, help=f"the original run's database (default {RECORDED})"
255
+ )
256
+ playing.add_argument(
257
+ "--database", default=REPLAYED, help=f"the replay's database (default {REPLAYED})"
258
+ )
259
+
260
+ branching = commands.add_parser(
261
+ "branch",
262
+ help="replay a cassette's first K model calls, then switch model, as a new run",
263
+ )
264
+ branching.add_argument("parent", type=Path, help="the parent run's cassette")
265
+ branching.add_argument(
266
+ "--scenario", type=Path, required=True, help="the parent run's scenario file"
267
+ )
268
+ branching.add_argument(
269
+ "--at",
270
+ type=int,
271
+ required=True,
272
+ metavar="K",
273
+ help="the branch point: how many of the parent's model calls to replay",
274
+ )
275
+ branching.add_argument(
276
+ "--cassette", type=Path, required=True, help="where to write the branch's cassette"
277
+ )
278
+ branching.add_argument(
279
+ "--result", type=Path, required=True, help="where to write the branch's result JSON"
280
+ )
281
+ branching.add_argument(
282
+ "--database", default=BRANCHED, help=f"the branch's database (default {BRANCHED})"
283
+ )
284
+ add_live_options(branching)
285
+
286
+ args = parser.parse_args(argv)
287
+ if args.command == "record" and args.scenario is not None and args.contract is None:
288
+ parser.error("record --scenario needs --contract")
289
+ if args.command == "record" and args.demo is not None and args.contract is not None:
290
+ parser.error("a demo has its own contract level; leave out --contract")
291
+ if args.command == "play" and args.database == args.original:
292
+ parser.error("the replay needs its own database, not the original run's")
293
+ if args.command in ("record", "branch"):
294
+ check_live_options(parser, args)
295
+ try:
296
+ live = live_from_args(args, os.environ) if args.command in ("record", "branch") else None
297
+ branch = _branch_spec(args, live) if args.command == "branch" else None
298
+ bootstrap_server(args.dsn, DB_DIR)
299
+ if branch is not None:
300
+ return _branch(args, *branch, live)
301
+ return _record(args, live) if args.command == "record" else _play(args)
302
+ except (OSError, MigrationError, PlaybackError, ReplayError, ValueError) as error:
303
+ print(f"{args.command} failed: {error}", file=sys.stderr)
304
+ return 1
305
+
306
+
307
+ if __name__ == "__main__":
308
+ sys.exit(main())
@@ -0,0 +1,322 @@
1
+ """Model cassettes (0009): the model calls of one run, in order, as one stable file.
2
+
3
+ A cassette holds what a replay needs to serve a run its model's answers, and to check
4
+ that it serves the same run:
5
+
6
+ - `format`: `fivefold-cassette/1`.
7
+ - `run`: the run it was recorded from, as `harness.run` records it: the run id, the
8
+ scenario's id and digest, the configuration, the contract level, the seed, the
9
+ topology, and the run's parameters (the scenario's parameters, the fault plan and the
10
+ workers). A branch's header also names its parent's run id and its branch point
11
+ (`parent_run_id` and `branch_point`); any other run's leaves both out, so its file is
12
+ the same as before branches existed.
13
+ - `model`: the model that answered: its adapter (`scripted` or `live`), its name and its
14
+ parameters. A branch names the model it switched to, and hashes every call with it,
15
+ the calls it replayed from its parent included (`fivefold.agents.branch`).
16
+ - `calls`: every model call, in the order the run's workers made them: the worker that
17
+ asked, the hash of the request, and the response.
18
+
19
+ The request hash is the SHA-256 of the request's canonical JSON,
20
+ `{"messages": [...], "params": {...}, "tools": [...]}`:
21
+
22
+ - messages keep what a model is sent: the type, the content, the name, an AI message's
23
+ tool calls, a tool message's call id and status, a chat message's role, and any
24
+ provider fields in `additional_kwargs`. Never their ids: LangChain and LangGraph draw
25
+ those at random.
26
+ - tools are the schemas bound to the model, in the OpenAI tool format that langchain-core
27
+ writes for every kind of tool. The agent's graph binds the worker's tools to every
28
+ adapter, so a run's requests carry them.
29
+ - params are the model's name and parameters, what was bound with the tools (such as
30
+ `tool_choice`), and the call's own arguments, `stop` among them.
31
+
32
+ Canonical JSON has sorted keys, no spaces, non-ASCII characters as they are, and no NaN or
33
+ infinity. The file is the cassette as JSON with sorted keys, two-space indents, non-ASCII
34
+ characters as they are and a final newline, so one cassette has one text. Its digest, the
35
+ SHA-256 of that text, is what `harness.run.cassette_digest` records. Loading refuses a
36
+ file in any other form, so a file's SHA-256 is always its cassette's digest.
37
+
38
+ Checked in langchain-core 1.8.0: the message fields (`messages/base.py`, `messages/ai.py`,
39
+ `messages/tool.py`, `messages/chat.py`); `AIMessage`'s validator, which gives every tool
40
+ call its `type` (`messages/ai.py`); `convert_to_openai_tool(tool, *, strict=None)`
41
+ (`utils/function_calling.py`). Checked in pydantic 2.14.3: `JsonValue` (`types.py`),
42
+ `BaseModel.model_validate_json` (`main.py`), `Field(ge=...)` (`fields.py`) and
43
+ `model_validator(mode="after")` (`functional_validators.py`).
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import hashlib
49
+ import json
50
+ import math
51
+ from collections.abc import Callable, Mapping, Sequence
52
+ from pathlib import Path
53
+ from typing import Any, Final, Literal
54
+
55
+ from langchain_core.messages import AIMessage, BaseMessage, ChatMessage, ToolMessage
56
+ from langchain_core.tools import BaseTool
57
+ from langchain_core.utils.function_calling import convert_to_openai_tool
58
+ from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator
59
+
60
+ FORMAT: Final = "fivefold-cassette/1"
61
+ """The format a cassette file names, so a later format can refuse an old file clearly."""
62
+
63
+ BRANCH_FIELDS: Final = ("parent_run_id", "branch_point")
64
+ """The header fields only a branch's cassette writes (0009, Playback and branching)."""
65
+
66
+ Tool = dict[str, Any] | type | Callable[..., Any] | BaseTool
67
+ """Anything `bind_tools` takes (langchain-core 1.8.0, `BaseChatModel.bind_tools`)."""
68
+
69
+
70
+ class CassetteError(ValueError):
71
+ """A file that is not a cassette in its canonical form."""
72
+
73
+
74
+ # Canonical JSON ---------------------------------------------------------------------------
75
+
76
+
77
+ def plain(value: object, where: str = "value") -> JsonValue:
78
+ """`value` as plain JSON data: dicts with text keys, lists, text, numbers, booleans and None.
79
+
80
+ Mappings become dicts and tuples become lists. Raises `TypeError` or `ValueError`,
81
+ naming the place, for anything JSON cannot hold: other types, keys that are not text,
82
+ NaN and infinity.
83
+ """
84
+ if value is None or isinstance(value, str | bool | int):
85
+ return value
86
+ if isinstance(value, float):
87
+ if not math.isfinite(value):
88
+ raise ValueError(f"{where} is {value}, which JSON cannot hold")
89
+ return value
90
+ if isinstance(value, Mapping):
91
+ return plain_object(value, where)
92
+ if isinstance(value, list | tuple):
93
+ return [plain(item, f"{where}[{index}]") for index, item in enumerate(value)]
94
+ raise TypeError(f"{where} is a {type(value).__name__}, which JSON cannot hold")
95
+
96
+
97
+ def plain_object(value: Mapping[Any, Any], where: str = "value") -> dict[str, JsonValue]:
98
+ """A mapping as a plain JSON object; see `plain`."""
99
+ data: dict[str, JsonValue] = {}
100
+ for key, item in value.items():
101
+ if not isinstance(key, str):
102
+ raise TypeError(f"{where} has a key that is not text: {key!r}")
103
+ data[key] = plain(item, f"{where}.{key}")
104
+ return data
105
+
106
+
107
+ def canonical_json(value: object) -> str:
108
+ """`value` as canonical JSON: sorted keys, no spaces, non-ASCII characters as they are."""
109
+ return json.dumps(
110
+ plain(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
111
+ )
112
+
113
+
114
+ def sha256(text: str) -> str:
115
+ """The SHA-256 of `text` in UTF-8, as 64 lowercase hex digits."""
116
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
117
+
118
+
119
+ # Requests and responses -------------------------------------------------------------------
120
+
121
+
122
+ def message_json(message: BaseMessage) -> dict[str, JsonValue]:
123
+ """What a model is sent of `message`, as plain JSON. Never its id, which is random."""
124
+ data: dict[str, object] = {"type": message.type, "content": message.content}
125
+ if message.name is not None:
126
+ data["name"] = message.name
127
+ if message.additional_kwargs:
128
+ data["additional_kwargs"] = message.additional_kwargs
129
+ if isinstance(message, AIMessage):
130
+ if message.tool_calls:
131
+ data["tool_calls"] = [
132
+ {"name": call["name"], "args": call["args"], "id": call["id"]}
133
+ for call in message.tool_calls
134
+ ]
135
+ if message.invalid_tool_calls:
136
+ data["invalid_tool_calls"] = list(message.invalid_tool_calls)
137
+ if isinstance(message, ToolMessage):
138
+ data["tool_call_id"] = message.tool_call_id
139
+ data["status"] = message.status
140
+ if isinstance(message, ChatMessage):
141
+ data["role"] = message.role
142
+ return plain_object(data, f"a {message.type} message")
143
+
144
+
145
+ def tool_schemas(tools: Sequence[Tool]) -> list[JsonValue]:
146
+ """The schemas of the tools bound to a model, in the OpenAI tool format."""
147
+ return [
148
+ plain(convert_to_openai_tool(tool), f"tool {index}") for index, tool in enumerate(tools)
149
+ ]
150
+
151
+
152
+ def request_params(
153
+ model: ModelInfo,
154
+ bound: Mapping[str, object],
155
+ stop: Sequence[str] | None,
156
+ call: Mapping[str, object],
157
+ ) -> dict[str, JsonValue]:
158
+ """The parameters of one model call: the model's name and parameters, what was bound
159
+ with its tools, and the call's own arguments, `stop` among them."""
160
+ params: dict[str, object] = {"model": model.name, **model.params, **bound, **call}
161
+ if stop is not None:
162
+ params["stop"] = list(stop)
163
+ return plain_object(params, "the call's parameters")
164
+
165
+
166
+ def request_json(
167
+ messages: Sequence[BaseMessage],
168
+ *,
169
+ tools: Sequence[JsonValue],
170
+ params: Mapping[str, JsonValue],
171
+ ) -> str:
172
+ """A request's canonical JSON: the messages, the tools and the params."""
173
+ request = {
174
+ "messages": [message_json(message) for message in messages],
175
+ "params": dict(params),
176
+ "tools": list(tools),
177
+ }
178
+ return canonical_json(request)
179
+
180
+
181
+ def request_hash(
182
+ messages: Sequence[BaseMessage],
183
+ *,
184
+ tools: Sequence[JsonValue],
185
+ params: Mapping[str, JsonValue],
186
+ ) -> str:
187
+ """The SHA-256 of a request's canonical JSON (`request_json`)."""
188
+ return sha256(request_json(messages, tools=tools, params=params))
189
+
190
+
191
+ def response_json(reply: AIMessage) -> dict[str, JsonValue]:
192
+ """A model's reply as a cassette keeps it: everything but its id, which is random.
193
+
194
+ Fields that are empty are left out, so the file stays short.
195
+ """
196
+ data: dict[str, object] = {"content": reply.content}
197
+ if reply.tool_calls:
198
+ data["tool_calls"] = list(reply.tool_calls)
199
+ if reply.invalid_tool_calls:
200
+ data["invalid_tool_calls"] = list(reply.invalid_tool_calls)
201
+ if reply.additional_kwargs:
202
+ data["additional_kwargs"] = reply.additional_kwargs
203
+ if reply.response_metadata:
204
+ data["response_metadata"] = reply.response_metadata
205
+ if reply.usage_metadata is not None:
206
+ data["usage_metadata"] = dict(reply.usage_metadata)
207
+ if reply.name is not None:
208
+ data["name"] = reply.name
209
+ return plain_object(data, "the reply")
210
+
211
+
212
+ def response_message(response: Mapping[str, JsonValue]) -> AIMessage:
213
+ """The reply a cassette kept, as the model returned it, without its id."""
214
+ return AIMessage.model_validate(dict(response))
215
+
216
+
217
+ # The cassette -----------------------------------------------------------------------------
218
+
219
+
220
+ class _Part(BaseModel):
221
+ """Base of a cassette's parts: unknown fields are refused and values never change."""
222
+
223
+ model_config = ConfigDict(extra="forbid", frozen=True)
224
+
225
+
226
+ class RunHeader(_Part):
227
+ """The run a cassette was recorded from, as `harness.run` records it (0009)."""
228
+
229
+ run_id: str
230
+ scenario_id: str
231
+ scenario_digest: str
232
+ config: Literal["B0", "G1", "G2"]
233
+ contract: Literal["C0", "C1", "C2", "C3"]
234
+ seed: int
235
+ topology: Literal["inproc", "split"]
236
+ params: dict[str, JsonValue]
237
+ """The values the run used: the scenario's parameters, the fault plan and the workers."""
238
+ parent_run_id: str | None = None
239
+ """For a branch, the run id of the run it branched from; None for any other run."""
240
+ branch_point: int | None = Field(default=None, ge=0)
241
+ """For a branch, K: how many of its parent's model calls it replayed before it
242
+ switched model; None for any other run."""
243
+
244
+ @model_validator(mode="after")
245
+ def _parent_and_point_together(self) -> RunHeader:
246
+ """A branch names both its parent and its branch point; any other run neither, as
247
+ `harness.run` requires."""
248
+ if (self.parent_run_id is None) != (self.branch_point is None):
249
+ raise ValueError("a branch names both its parent_run_id and its branch_point")
250
+ return self
251
+
252
+
253
+ class ModelInfo(_Part):
254
+ """The model that answered the calls: its adapter, its name and its parameters."""
255
+
256
+ adapter: Literal["scripted", "live"]
257
+ name: str
258
+ params: dict[str, JsonValue] = Field(default_factory=dict)
259
+
260
+
261
+ class Call(_Part):
262
+ """One model call: the worker that asked, the request's hash and the response."""
263
+
264
+ worker: str
265
+ request: str = Field(pattern=r"^[0-9a-f]{64}$")
266
+ response: dict[str, JsonValue]
267
+
268
+
269
+ class Cassette(_Part):
270
+ """The model calls of one run, in the order the run made them (0009)."""
271
+
272
+ format: Literal["fivefold-cassette/1"] = FORMAT
273
+ run: RunHeader
274
+ model: ModelInfo
275
+ calls: tuple[Call, ...]
276
+
277
+ def text(self) -> str:
278
+ """The cassette's file: JSON with sorted keys, two-space indents, non-ASCII
279
+ characters as they are, and a final newline. A run that is no branch leaves out the
280
+ header's branch fields, so its file is the same as before branches existed."""
281
+ data = self.model_dump(mode="json")
282
+ if self.run.parent_run_id is None:
283
+ for name in BRANCH_FIELDS:
284
+ del data["run"][name]
285
+ return (
286
+ json.dumps(data, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
287
+ )
288
+
289
+ def digest(self) -> str:
290
+ """The SHA-256 of the cassette's file, as `harness.run.cassette_digest` records it."""
291
+ return sha256(self.text())
292
+
293
+ @classmethod
294
+ def from_text(cls, text: str) -> Cassette:
295
+ """Read a cassette's file; refuse one that is not a cassette in canonical form."""
296
+ try:
297
+ cassette = cls.model_validate_json(text)
298
+ except ValidationError as error:
299
+ raise CassetteError(f"not a cassette: {error}") from error
300
+ try:
301
+ canonical = cassette.text()
302
+ except ValueError as error:
303
+ raise CassetteError(f"not a cassette: {error}") from error
304
+ if canonical != text:
305
+ raise CassetteError(
306
+ "the cassette is not in canonical form: sorted keys, two-space indents, "
307
+ "non-ASCII characters as they are, and a final newline"
308
+ )
309
+ return cassette
310
+
311
+ def save(self, path: Path) -> None:
312
+ """Write the cassette's file, in UTF-8, with no line-ending translation."""
313
+ path.write_bytes(self.text().encode("utf-8"))
314
+
315
+ @classmethod
316
+ def load(cls, path: Path) -> Cassette:
317
+ """Read and check a cassette's file."""
318
+ try:
319
+ text = path.read_bytes().decode("utf-8")
320
+ except UnicodeDecodeError as error:
321
+ raise CassetteError(f"{path} is not UTF-8 text") from error
322
+ return cls.from_text(text)