fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,368 @@
1
+ """The run's model adapter (0009): the chat model each agent worker gets.
2
+
3
+ `RunSpec.model` chooses one:
4
+
5
+ - `Scripted()`: the scripted adapter, the policy of 0008 behind a chat model.
6
+ - `Scripted(record=True)`: the same, wrapped by the recorder, so every request and response
7
+ goes into a cassette (`fivefold.recorder.recording`).
8
+ - `Replay(cassette)`: the replay adapter, which serves the cassette's responses in order
9
+ (`fivefold.agents.replay`).
10
+ - `Live(provider, model, params, spend)`: the live adapter, a provider's chat model
11
+ (`fivefold.agents.live`), always wrapped by the recorder (0009).
12
+ - `Branch(parent, point, model)`: an experimental branch (0009, Playback and branching).
13
+ The branch adapter replays the first `point` calls of the cassette `parent`, then
14
+ hands every later call to `model`, the scripted model or a live one
15
+ (`fivefold.agents.branch`). A branch always records its own cassette, which holds all
16
+ its calls, the replayed ones included.
17
+
18
+ `harness.run` records the adapter (`scripted`, `live` or `replay`), the model's name, and
19
+ the cassette's digest: for a replay, the cassette it serves, from the start; for a
20
+ recording, the cassette it made, once the run has stopped and the cassette is complete. A
21
+ recording run and its replays therefore name the same digest. A live model's name says the
22
+ provider and the model, such as `ollama:llama3.1:8b`, and its replays name it too.
23
+
24
+ A branch records the adapter and the name of the model it switches to, and the digest of
25
+ its own cassette. Its every model call goes into `harness.model_call` under that name, as
26
+ the run's model, and its cassette hashes every call with that model, the replayed calls
27
+ included, so that `play` replays a branch as any other recorded run. The branch's
28
+ `Parent`, its parent's run id and its branch point, goes into `harness.run` as
29
+ `parent_run_id` and `branch_point`, and into its cassette's header; a replay of the
30
+ branch's cassette records the same.
31
+
32
+ Live runs sit behind an explicit flag and a spend cap (CLAUDE.md). A live run is refused,
33
+ before anything runs, unless the environment sets `FIVEFOLD_LIVE=1`, the run has a spend
34
+ cap in USD with a price file (`Live.spend`), the price file prices the model, so the cap
35
+ can always be enforced, and the environment sets the provider's API key, for a provider
36
+ that needs one (`check_live`). The key's variable is checked for being set and nothing
37
+ more: the provider's package reads the key itself (`fivefold.agents.live.PROVIDERS`). A
38
+ live run is also refused while a variable that would send the provider's requests to
39
+ another backend is set, such as `GOOGLE_GENAI_USE_VERTEXAI`.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import math
45
+ import os
46
+ from collections.abc import Callable, Mapping
47
+ from dataclasses import dataclass, field
48
+ from typing import Final, Literal
49
+
50
+ from langchain_core.language_models.chat_models import BaseChatModel
51
+ from pydantic import JsonValue
52
+
53
+ from fivefold.agents.branch import BranchModel, BranchPlayer, check_branch_point
54
+ from fivefold.agents.live import (
55
+ CALL_LIMIT_S,
56
+ PROVIDERS,
57
+ Hold,
58
+ LiveModel,
59
+ SpendMeter,
60
+ Think,
61
+ live_name,
62
+ provider_model,
63
+ )
64
+ from fivefold.agents.replay import CassettePlayer, ReplayModel
65
+ from fivefold.domain.prices import ModelPrice
66
+ from fivefold.harness.log import RunRecord
67
+ from fivefold.recorder.cassette import Cassette, ModelInfo, RunHeader, plain_object
68
+ from fivefold.recorder.recording import CassetteRecorder, RecordingModel
69
+
70
+ SCRIPTED_MODEL: Final = ModelInfo(adapter="scripted", name="scripted")
71
+ """The scripted adapter's model, as `harness.run` and a cassette name it."""
72
+
73
+ LIVE_FLAG: Final = "FIVEFOLD_LIVE"
74
+ """The environment variable that must be `1` for a live run to start."""
75
+
76
+
77
+ async def _no_think() -> None:
78
+ """The think of a model that answers after the branch adapter has thought for it."""
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class Scripted:
83
+ """The scripted adapter (0009); with `record`, wrapped by the recorder."""
84
+
85
+ record: bool = False
86
+
87
+
88
+ @dataclass(frozen=True)
89
+ class Replay:
90
+ """The replay adapter (0009), serving `cassette`'s responses in order."""
91
+
92
+ cassette: Cassette
93
+
94
+
95
+ @dataclass(frozen=True)
96
+ class Live:
97
+ """The live adapter (0009): `provider`'s `model` with `params`, always recorded.
98
+
99
+ `params` are the model's generation parameters, which the cassette records and every
100
+ request hash covers. `spend` is the spend cap and price file of the command that runs
101
+ it, shared by the command's runs; a live run is refused without one. `limit_s` is how
102
+ long, in wall-clock seconds, each call may wait for the provider before it fails.
103
+ """
104
+
105
+ provider: str
106
+ model: str
107
+ params: Mapping[str, JsonValue] = field(default_factory=dict)
108
+ spend: SpendMeter | None = None
109
+ limit_s: float = CALL_LIMIT_S
110
+
111
+ @property
112
+ def name(self) -> str:
113
+ """The model's name, as `harness.run` and the cassette record it."""
114
+ return live_name(self.provider, self.model)
115
+
116
+
117
+ SCRIPTED: Final = Scripted()
118
+ """The scripted adapter, without recording: a run's default."""
119
+
120
+
121
+ @dataclass(frozen=True)
122
+ class Branch:
123
+ """An experimental branch (0009): the first `point` model calls of the run `parent`
124
+ was recorded from, replayed from it, then `model`, the scripted model or a live one.
125
+
126
+ `point` is the branch point, K: how many of the parent's calls the branch replays,
127
+ from 0 to all of them. The branch always records its own cassette, whatever
128
+ `model`'s `record` says.
129
+ """
130
+
131
+ parent: Cassette
132
+ point: int
133
+ model: Scripted | Live = SCRIPTED
134
+
135
+ def __post_init__(self) -> None:
136
+ check_branch_point(self.parent, self.point)
137
+
138
+
139
+ ModelSpec = Scripted | Replay | Live | Branch
140
+ """Which model adapter a run uses."""
141
+
142
+
143
+ @dataclass(frozen=True)
144
+ class Parent:
145
+ """The run a branch comes from (0009): the parent's run id, and the branch point, K,
146
+ the number of the parent's model calls the branch replays."""
147
+
148
+ run_id: str
149
+ branch_point: int
150
+
151
+
152
+ def branch_run_id(parent: Parent) -> str:
153
+ """A branch's run id: its parent's run id, `.branch-` and its branch point, such as
154
+ `dev-001-single-po.G2.C2.7.branch-4`. A branch of a branch adds a second suffix."""
155
+ return f"{parent.run_id}.branch-{parent.branch_point}"
156
+
157
+
158
+ def parent_of(spec: ModelSpec) -> Parent | None:
159
+ """The parent of a run with the model `spec`: a branch's, or, for a replay of a
160
+ branch's cassette, the one its header names. None for any other run."""
161
+ if isinstance(spec, Branch):
162
+ return Parent(spec.parent.run.run_id, spec.point)
163
+ if isinstance(spec, Replay):
164
+ header = spec.cassette.run
165
+ if header.parent_run_id is not None and header.branch_point is not None:
166
+ return Parent(header.parent_run_id, header.branch_point)
167
+ return None
168
+
169
+
170
+ class LiveRefused(ValueError):
171
+ """A live run that must not start: the flag, the spend cap, the model's price or the
172
+ provider's API key is missing, or a variable would send the provider's requests to
173
+ another backend."""
174
+
175
+
176
+ def check_live_flag(environ: Mapping[str, str]) -> None:
177
+ """Refuse any live run unless `environ` sets `FIVEFOLD_LIVE=1`."""
178
+ if environ.get(LIVE_FLAG) != "1":
179
+ raise LiveRefused(
180
+ f"live runs are off: set {LIVE_FLAG}=1 to allow one, with a spend cap and a price file"
181
+ )
182
+
183
+
184
+ def check_live(spec: Live, environ: Mapping[str, str]) -> tuple[SpendMeter, ModelPrice]:
185
+ """Refuse a live run unless `environ` sets `FIVEFOLD_LIVE=1`, the run has a spend cap
186
+ with a price file that prices its model, and `environ` sets the provider's API key and
187
+ none of the variables the provider refuses; return the meter and the price.
188
+
189
+ The key and the refused variables are checked last, and only for being set; a
190
+ provider this build does not have is refused when its model is built
191
+ (`fivefold.agents.live.check_provider`).
192
+ """
193
+ check_live_flag(environ)
194
+ if spec.spend is None:
195
+ raise LiveRefused("a live run needs a spend cap in USD and a price file")
196
+ if not (math.isfinite(spec.limit_s) and spec.limit_s > 0):
197
+ raise LiveRefused(f"a live call's time limit must be above 0 seconds, not {spec.limit_s}")
198
+ price = spec.spend.prices.price(spec.provider, spec.model)
199
+ entry = PROVIDERS.get(spec.provider)
200
+ if entry is not None and entry.keys and not any(environ.get(key) for key in entry.keys):
201
+ raise LiveRefused(
202
+ f"a live run of {spec.provider} needs its API key in {' or '.join(entry.keys)}"
203
+ )
204
+ for name in entry.forbidden if entry is not None else ():
205
+ if environ.get(name):
206
+ raise LiveRefused(
207
+ f"a live run of {spec.provider} refuses {name}, which would send its requests"
208
+ " to another backend, with other credentials: unset it"
209
+ )
210
+ return spec.spend, price
211
+
212
+
213
+ def run_header(record: RunRecord) -> RunHeader:
214
+ """What a cassette says about the run it was recorded from: the run's record, without
215
+ the model, which the cassette names on its own."""
216
+ return RunHeader(
217
+ run_id=record.run_id,
218
+ scenario_id=record.scenario_id,
219
+ scenario_digest=record.scenario_digest,
220
+ config=record.config,
221
+ contract=record.contract,
222
+ seed=record.seed,
223
+ topology=record.topology,
224
+ params=plain_object(record.params, "the run's params"),
225
+ parent_run_id=record.parent_run_id,
226
+ branch_point=record.branch_point,
227
+ )
228
+
229
+
230
+ @dataclass(frozen=True)
231
+ class _LiveSetup:
232
+ """A live run's model, built once the run passed `check_live`."""
233
+
234
+ spec: Live
235
+ chat: BaseChatModel
236
+ meter: SpendMeter
237
+ price: ModelPrice
238
+
239
+
240
+ class RunModels:
241
+ """The chat models of one run's agent workers, and the run's cassette."""
242
+
243
+ def __init__(self, spec: ModelSpec, *, environ: Mapping[str, str] | None = None) -> None:
244
+ """`environ` is where a live run looks for its flag: the process environment by
245
+ default. A live run is refused here, before the run records anything, and so is a
246
+ live branch."""
247
+ self.spec = spec
248
+ self._recorder: CassetteRecorder | None = None
249
+ self._player: CassettePlayer | None = None
250
+ self._branch: BranchPlayer | None = None
251
+ self._live: _LiveSetup | None = None
252
+ self.cassette: Cassette | None = None
253
+ """The cassette a replay serves, or the one a recording made, once complete."""
254
+ if isinstance(spec, Replay):
255
+ self._player = CassettePlayer(spec.cassette)
256
+ self.cassette = spec.cassette
257
+ elif isinstance(spec, Branch):
258
+ self._branch = BranchPlayer(spec.parent, spec.point)
259
+ self._recorder = CassetteRecorder(self._answering(spec.model, environ))
260
+ elif isinstance(spec, Live):
261
+ self._recorder = CassetteRecorder(self._answering(spec, environ))
262
+ elif spec.record:
263
+ self._recorder = CassetteRecorder(SCRIPTED_MODEL)
264
+
265
+ def _answering(self, spec: Scripted | Live, environ: Mapping[str, str] | None) -> ModelInfo:
266
+ """Set up the model that answers the calls, live after `check_live`; return it as
267
+ the cassette names it."""
268
+ if isinstance(spec, Scripted):
269
+ return SCRIPTED_MODEL
270
+ meter, price = check_live(spec, os.environ if environ is None else environ)
271
+ chat = provider_model(spec.provider, spec.model, spec.params)
272
+ self._live = _LiveSetup(spec, chat, meter, price)
273
+ params = plain_object(spec.params, "the model's parameters")
274
+ return ModelInfo(adapter="live", name=spec.name, params=params)
275
+
276
+ @property
277
+ def adapter(self) -> Literal["scripted", "live", "replay"]:
278
+ """The adapter, as `harness.run.model_adapter` records it: for a branch, the
279
+ adapter of the model it switches to."""
280
+ if self._player is not None:
281
+ return "replay"
282
+ return "live" if self._live is not None else "scripted"
283
+
284
+ @property
285
+ def name(self) -> str:
286
+ """The model's name, as `harness.run.model` records it: a replay names the
287
+ recorded model, and a branch the model it switches to."""
288
+ if self._player is not None:
289
+ return self._player.cassette.model.name
290
+ if self._live is not None:
291
+ return self._live.spec.name
292
+ return SCRIPTED_MODEL.name
293
+
294
+ @property
295
+ def cassette_digest(self) -> str | None:
296
+ """The digest `harness.run` records when the run starts: a replay's cassette."""
297
+ return self._player.cassette.digest() if self._player is not None else None
298
+
299
+ @property
300
+ def served(self) -> int:
301
+ """How many recorded calls a replay or a branch has answered from its cassette."""
302
+ if self._player is not None:
303
+ return self._player.served
304
+ return self._branch.served if self._branch is not None else 0
305
+
306
+ def chat_model(
307
+ self,
308
+ worker: str,
309
+ *,
310
+ scripted: Callable[[Think], BaseChatModel],
311
+ think: Think,
312
+ hold: Hold | None = None,
313
+ ) -> BaseChatModel:
314
+ """The chat model of one incarnation of `worker`.
315
+
316
+ `scripted` builds the worker's scripted model, which takes the think it is given
317
+ before each call; `think` takes `think_ms` of virtual time; `hold` awaits a live
318
+ model's provider with virtual time held still (`Scheduler.hold`), which a live
319
+ model needs. Every call takes `think` once: a branch takes it itself, before it
320
+ decides where the call goes, so the model it switches to takes none.
321
+ """
322
+ if self._player is not None:
323
+ return ReplayModel(self._player, worker=worker, think=think)
324
+ if self._branch is not None:
325
+ if self._recorder is None:
326
+ raise ValueError("a branch records its cassette")
327
+ then = self._answering_model(scripted, _no_think, hold)
328
+ branch = BranchModel(self._branch, then, worker=worker, think=think)
329
+ return RecordingModel(branch, self._recorder, worker=worker)
330
+ model = self._answering_model(scripted, think, hold)
331
+ if self._recorder is not None:
332
+ return RecordingModel(model, self._recorder, worker=worker)
333
+ return model
334
+
335
+ def _answering_model(
336
+ self, scripted: Callable[[Think], BaseChatModel], think: Think, hold: Hold | None
337
+ ) -> BaseChatModel:
338
+ """The model that answers the calls, taking `think` before each one: the live
339
+ adapter, or the scripted model."""
340
+ if self._live is None:
341
+ return scripted(think)
342
+ if hold is None or self._recorder is None:
343
+ raise ValueError("a live model needs the scheduler's hold and the recorder")
344
+ return LiveModel(
345
+ self._live.chat,
346
+ name=self._live.spec.name,
347
+ price=self._live.price,
348
+ meter=self._live.meter,
349
+ think=think,
350
+ hold=hold,
351
+ limit_s=self._live.spec.limit_s,
352
+ )
353
+
354
+ def stopped(self, run: RunRecord) -> str | None:
355
+ """The run has stopped, so no model call is left. A recording's cassette is then
356
+ complete: keep it, and return its digest for `harness.run`."""
357
+ if self._recorder is None:
358
+ return None
359
+ self.cassette = self._recorder.cassette(run_header(run))
360
+ return self.cassette.digest()
361
+
362
+ def check(self) -> None:
363
+ """Fail the run, with `ReplayError`, if a replay did not serve exactly the
364
+ recorded calls, or a branch did not replay its parent's first K calls (0009)."""
365
+ if self._player is not None:
366
+ self._player.finish()
367
+ if self._branch is not None:
368
+ self._branch.finish()
@@ -0,0 +1,105 @@
1
+ """The run registry (0009, 0012): one row per run database, so the viewer can list runs.
2
+
3
+ The registry is a database of its own on the server that holds the run databases:
4
+ `fivefold_registry`, unless FIVEFOLD_REGISTRY_DB names another (`fivefold.harness.
5
+ migrations.registry_database`). `bootstrap_server` in that module creates it and applies
6
+ its migrations, `db/registry/`. A row holds the run database's name, the run's id, its
7
+ scenario, configuration, contract level and seed, and when the row was written, in wall
8
+ time, for display only (0009).
9
+
10
+ The harness writes a run's row when the run records itself in its run database
11
+ (`InprocRun.execute`), so every run gets one, whoever made its database: a demo, a study,
12
+ playback or a test. A database made again under an old name, as each demo's is, gets its
13
+ row replaced. `forget` removes the row of a run database that is dropped.
14
+
15
+ The harness writes as role `ff_registry`, which has no privilege on any run database's
16
+ schemas; the viewer reads as `ff_viewer` (`db/registry/0001_registry.sql`).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from dataclasses import dataclass
22
+ from typing import Final
23
+
24
+ import psycopg
25
+ from psycopg.conninfo import conninfo_to_dict, make_conninfo
26
+ from psycopg.rows import TupleRow
27
+
28
+ from fivefold.harness.migrations import check_database_name, registry_database
29
+
30
+ REGISTRY_ROLE: Final = "ff_registry"
31
+ """The role the harness writes the registry as."""
32
+
33
+
34
+ class RegistryError(Exception):
35
+ """The run registry cannot be reached, or a run's database cannot be named."""
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class RegistryEntry:
40
+ """One run database's row: what the registry lists for the viewer (0012)."""
41
+
42
+ database: str
43
+ run_id: str
44
+ scenario_id: str
45
+ config: str
46
+ contract: str
47
+ seed: int
48
+
49
+
50
+ def database_of(dsn: str) -> str:
51
+ """The name of the database `dsn` connects to. It must name one."""
52
+ name = conninfo_to_dict(dsn).get("dbname")
53
+ if not isinstance(name, str) or not name:
54
+ raise RegistryError("the run's connection string names no database")
55
+ check_database_name(name)
56
+ return name
57
+
58
+
59
+ def register(dsn: str, entry: RegistryEntry, *, registry: str | None = None) -> None:
60
+ """Write `entry` to the registry on the server `dsn` names, replacing the row its
61
+ database had, as `ff_registry`.
62
+
63
+ `registry` defaults to `registry_database()`. Raises `RegistryError` when the
64
+ registry cannot be reached, for example because it was never created.
65
+ """
66
+ name = registry_database() if registry is None else registry
67
+ check_database_name(entry.database)
68
+ with _connect(dsn, name) as conn:
69
+ conn.execute(
70
+ "INSERT INTO registry.run AS r"
71
+ " (database_name, run_id, scenario_id, config, contract, seed)"
72
+ " VALUES (%s, %s, %s, %s, %s, %s)"
73
+ " ON CONFLICT (database_name) DO UPDATE SET run_id = EXCLUDED.run_id,"
74
+ " scenario_id = EXCLUDED.scenario_id, config = EXCLUDED.config,"
75
+ " contract = EXCLUDED.contract, seed = EXCLUDED.seed,"
76
+ " created_wall_at = EXCLUDED.created_wall_at",
77
+ (
78
+ entry.database,
79
+ entry.run_id,
80
+ entry.scenario_id,
81
+ entry.config,
82
+ entry.contract,
83
+ entry.seed,
84
+ ),
85
+ )
86
+
87
+
88
+ def forget(dsn: str, database: str, *, registry: str | None = None) -> None:
89
+ """Remove the row of `database` from the registry on the server `dsn` names, if it
90
+ has one. For a run database that was dropped."""
91
+ name = registry_database() if registry is None else registry
92
+ with _connect(dsn, name) as conn:
93
+ conn.execute("DELETE FROM registry.run WHERE database_name = %s", (database,))
94
+
95
+
96
+ def _connect(dsn: str, registry: str) -> psycopg.Connection[TupleRow]:
97
+ try:
98
+ return psycopg.connect(
99
+ make_conninfo(dsn, dbname=registry, user=REGISTRY_ROLE), autocommit=True
100
+ )
101
+ except psycopg.OperationalError as error:
102
+ raise RegistryError(
103
+ f"the run registry {registry} cannot be reached ({error}); create it with"
104
+ " `python -m fivefold.harness.migrations bootstrap`"
105
+ ) from error
@@ -0,0 +1,100 @@
1
+ """The harness's report sink, where every final report lands (0004, 0008).
2
+
3
+ In G modes the gateway hands each report it accepted (G2) or forwarded (G1) to the sink;
4
+ in baselines the agent submits to the sink directly, unchecked. The sink checks the
5
+ report schema, records the report with whether it was valid, and never refuses one: the
6
+ evaluator counts the last valid report before the run stops (spec, Gateway).
7
+
8
+ PostgreSQL's `jsonb` cannot hold a NUL character or a number that is not finite. A
9
+ report that holds either is recorded as `{"json_text": ...}`, its JSON as text, from
10
+ which `json.loads` gives back the report unchanged. Such a report is always invalid,
11
+ because the report schema allows neither.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import math
18
+ from collections.abc import Collection
19
+ from typing import Literal
20
+
21
+ from fivefold.domain.report import InvalidReport, parse_report
22
+ from fivefold.harness.log import HarnessLog, JsonValue, ReportRecord
23
+ from fivefold.harness.scheduler import Scheduler
24
+ from fivefold.harness.transport import Envelope, Handler, Reply
25
+
26
+ TEXT_KEY = "json_text"
27
+ """The one key of a report recorded as JSON text."""
28
+
29
+
30
+ class ReportSink:
31
+ """Records final reports in the harness log."""
32
+
33
+ def __init__(self, scheduler: Scheduler, log: HarnessLog, intents: Collection[str]) -> None:
34
+ self._scheduler = scheduler
35
+ self._log = log
36
+ self._intents = frozenset(intents)
37
+
38
+ def submit(
39
+ self,
40
+ worker: str,
41
+ report: JsonValue,
42
+ *,
43
+ accepted_by: Literal["gateway", "sink"],
44
+ message_seq: int | None = None,
45
+ ) -> tuple[str, ...]:
46
+ """Record a report; return what breaks its schema, if anything."""
47
+ text = json.dumps(report, sort_keys=True)
48
+ kept: JsonValue = json.loads(text)
49
+ errors: tuple[str, ...] = ()
50
+ try:
51
+ parse_report(kept, self._intents)
52
+ except InvalidReport as invalid:
53
+ errors = tuple(invalid.errors)
54
+ if not storable(kept):
55
+ kept = {TEXT_KEY: text}
56
+ self._log.report_submitted(
57
+ ReportRecord(
58
+ seq=self._log.next_seq(),
59
+ worker=worker,
60
+ report=kept,
61
+ valid=not errors,
62
+ errors=errors,
63
+ accepted_by=accepted_by,
64
+ vt=self._scheduler.now,
65
+ message_seq=message_seq,
66
+ )
67
+ )
68
+ return errors
69
+
70
+
71
+ def storable(value: JsonValue) -> bool:
72
+ """Whether `jsonb` can hold `value`: no NUL character, and only finite numbers."""
73
+ if isinstance(value, str):
74
+ return "\x00" not in value
75
+ if isinstance(value, float):
76
+ return math.isfinite(value)
77
+ if isinstance(value, dict):
78
+ return all(storable(key) and storable(item) for key, item in value.items())
79
+ if isinstance(value, list):
80
+ return all(storable(item) for item in value)
81
+ return True
82
+
83
+
84
+ def sink_endpoint(sink: ReportSink) -> Handler:
85
+ """The report sink on the transport, for baselines' `submit_final_report` (0008).
86
+
87
+ The sink records every report, valid or not. Like the gateway's tool, it refuses a
88
+ report that breaks the schema, so the agent can fix it and resubmit.
89
+ """
90
+
91
+ async def handle(request: Envelope) -> Reply:
92
+ errors = sink.submit(
93
+ request.sender, request.body, accepted_by="sink", message_seq=request.seq
94
+ )
95
+ if errors:
96
+ details = {"message": "; ".join(errors)}
97
+ return Reply({"result": "refused", "reason": "invalid_args", "details": details})
98
+ return Reply({"result": "accepted"})
99
+
100
+ return handle
@@ -0,0 +1,57 @@
1
+ """The run's seeded generators (0009).
2
+
3
+ The run seed feeds independent named generators (latency, fault plans, data generation,
4
+ scripted variation), so adding a new consumer does not shift another's numbers. No other
5
+ module may use the global `random` module, `secrets`, `uuid4` or `os.urandom` (0007).
6
+
7
+ One thing here is not seeded: `credential`, the secret of a split run's per-run credential
8
+ (0008). A secret that the run's seed could reproduce would be no secret, so it comes from
9
+ the system's source. It decides nothing in the run: it only lets a caller prove who it is.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ import random
16
+ import secrets
17
+
18
+ Generator = random.Random
19
+ """A seeded generator. Other modules name the type through this alias, never `random`."""
20
+
21
+
22
+ class Generators:
23
+ """Independent random generators, each derived from the run seed and its name."""
24
+
25
+ def __init__(self, seed: int) -> None:
26
+ self._seed = seed
27
+ self._made: dict[str, Generator] = {}
28
+
29
+ @property
30
+ def seed(self) -> int:
31
+ """The run seed."""
32
+ return self._seed
33
+
34
+ def get(self, name: str) -> Generator:
35
+ """The generator called `name`, the same one on every call in this run.
36
+
37
+ Its seed is the first 8 bytes of SHA-256 over the run seed and the name, so two
38
+ names never share a stream and a new name never shifts an existing one.
39
+ """
40
+ if not name:
41
+ raise ValueError("a generator needs a name")
42
+ made = self._made.get(name)
43
+ if made is None:
44
+ digest = hashlib.sha256(f"{self._seed}\x00{name}".encode()).digest()
45
+ made = random.Random(int.from_bytes(digest[:8], "big"))
46
+ self._made[name] = made
47
+ return made
48
+
49
+
50
+ CREDENTIAL_BYTES = 32
51
+ """The random bytes in each credential's secret: 256 bits."""
52
+
53
+
54
+ def credential() -> str:
55
+ """A new secret for a split run's credential (0008), as URL-safe text, from the system's
56
+ random source. Never seeded: see the module's docstring."""
57
+ return secrets.token_urlsafe(CREDENTIAL_BYTES)