fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""The run's model adapter (0009): the chat model each agent worker gets.
|
|
2
|
+
|
|
3
|
+
`RunSpec.model` chooses one:
|
|
4
|
+
|
|
5
|
+
- `Scripted()`: the scripted adapter, the policy of 0008 behind a chat model.
|
|
6
|
+
- `Scripted(record=True)`: the same, wrapped by the recorder, so every request and response
|
|
7
|
+
goes into a cassette (`fivefold.recorder.recording`).
|
|
8
|
+
- `Replay(cassette)`: the replay adapter, which serves the cassette's responses in order
|
|
9
|
+
(`fivefold.agents.replay`).
|
|
10
|
+
- `Live(provider, model, params, spend)`: the live adapter, a provider's chat model
|
|
11
|
+
(`fivefold.agents.live`), always wrapped by the recorder (0009).
|
|
12
|
+
- `Branch(parent, point, model)`: an experimental branch (0009, Playback and branching).
|
|
13
|
+
The branch adapter replays the first `point` calls of the cassette `parent`, then
|
|
14
|
+
hands every later call to `model`, the scripted model or a live one
|
|
15
|
+
(`fivefold.agents.branch`). A branch always records its own cassette, which holds all
|
|
16
|
+
its calls, the replayed ones included.
|
|
17
|
+
|
|
18
|
+
`harness.run` records the adapter (`scripted`, `live` or `replay`), the model's name, and
|
|
19
|
+
the cassette's digest: for a replay, the cassette it serves, from the start; for a
|
|
20
|
+
recording, the cassette it made, once the run has stopped and the cassette is complete. A
|
|
21
|
+
recording run and its replays therefore name the same digest. A live model's name says the
|
|
22
|
+
provider and the model, such as `ollama:llama3.1:8b`, and its replays name it too.
|
|
23
|
+
|
|
24
|
+
A branch records the adapter and the name of the model it switches to, and the digest of
|
|
25
|
+
its own cassette. Its every model call goes into `harness.model_call` under that name, as
|
|
26
|
+
the run's model, and its cassette hashes every call with that model, the replayed calls
|
|
27
|
+
included, so that `play` replays a branch as any other recorded run. The branch's
|
|
28
|
+
`Parent`, its parent's run id and its branch point, goes into `harness.run` as
|
|
29
|
+
`parent_run_id` and `branch_point`, and into its cassette's header; a replay of the
|
|
30
|
+
branch's cassette records the same.
|
|
31
|
+
|
|
32
|
+
Live runs sit behind an explicit flag and a spend cap (CLAUDE.md). A live run is refused,
|
|
33
|
+
before anything runs, unless the environment sets `FIVEFOLD_LIVE=1`, the run has a spend
|
|
34
|
+
cap in USD with a price file (`Live.spend`), the price file prices the model, so the cap
|
|
35
|
+
can always be enforced, and the environment sets the provider's API key, for a provider
|
|
36
|
+
that needs one (`check_live`). The key's variable is checked for being set and nothing
|
|
37
|
+
more: the provider's package reads the key itself (`fivefold.agents.live.PROVIDERS`). A
|
|
38
|
+
live run is also refused while a variable that would send the provider's requests to
|
|
39
|
+
another backend is set, such as `GOOGLE_GENAI_USE_VERTEXAI`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import math
|
|
45
|
+
import os
|
|
46
|
+
from collections.abc import Callable, Mapping
|
|
47
|
+
from dataclasses import dataclass, field
|
|
48
|
+
from typing import Final, Literal
|
|
49
|
+
|
|
50
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
51
|
+
from pydantic import JsonValue
|
|
52
|
+
|
|
53
|
+
from fivefold.agents.branch import BranchModel, BranchPlayer, check_branch_point
|
|
54
|
+
from fivefold.agents.live import (
|
|
55
|
+
CALL_LIMIT_S,
|
|
56
|
+
PROVIDERS,
|
|
57
|
+
Hold,
|
|
58
|
+
LiveModel,
|
|
59
|
+
SpendMeter,
|
|
60
|
+
Think,
|
|
61
|
+
live_name,
|
|
62
|
+
provider_model,
|
|
63
|
+
)
|
|
64
|
+
from fivefold.agents.replay import CassettePlayer, ReplayModel
|
|
65
|
+
from fivefold.domain.prices import ModelPrice
|
|
66
|
+
from fivefold.harness.log import RunRecord
|
|
67
|
+
from fivefold.recorder.cassette import Cassette, ModelInfo, RunHeader, plain_object
|
|
68
|
+
from fivefold.recorder.recording import CassetteRecorder, RecordingModel
|
|
69
|
+
|
|
70
|
+
SCRIPTED_MODEL: Final = ModelInfo(adapter="scripted", name="scripted")
|
|
71
|
+
"""The scripted adapter's model, as `harness.run` and a cassette name it."""
|
|
72
|
+
|
|
73
|
+
LIVE_FLAG: Final = "FIVEFOLD_LIVE"
|
|
74
|
+
"""The environment variable that must be `1` for a live run to start."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def _no_think() -> None:
|
|
78
|
+
"""The think of a model that answers after the branch adapter has thought for it."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class Scripted:
|
|
83
|
+
"""The scripted adapter (0009); with `record`, wrapped by the recorder."""
|
|
84
|
+
|
|
85
|
+
record: bool = False
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True)
|
|
89
|
+
class Replay:
|
|
90
|
+
"""The replay adapter (0009), serving `cassette`'s responses in order."""
|
|
91
|
+
|
|
92
|
+
cassette: Cassette
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True)
|
|
96
|
+
class Live:
|
|
97
|
+
"""The live adapter (0009): `provider`'s `model` with `params`, always recorded.
|
|
98
|
+
|
|
99
|
+
`params` are the model's generation parameters, which the cassette records and every
|
|
100
|
+
request hash covers. `spend` is the spend cap and price file of the command that runs
|
|
101
|
+
it, shared by the command's runs; a live run is refused without one. `limit_s` is how
|
|
102
|
+
long, in wall-clock seconds, each call may wait for the provider before it fails.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
provider: str
|
|
106
|
+
model: str
|
|
107
|
+
params: Mapping[str, JsonValue] = field(default_factory=dict)
|
|
108
|
+
spend: SpendMeter | None = None
|
|
109
|
+
limit_s: float = CALL_LIMIT_S
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def name(self) -> str:
|
|
113
|
+
"""The model's name, as `harness.run` and the cassette record it."""
|
|
114
|
+
return live_name(self.provider, self.model)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
SCRIPTED: Final = Scripted()
|
|
118
|
+
"""The scripted adapter, without recording: a run's default."""
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass(frozen=True)
|
|
122
|
+
class Branch:
|
|
123
|
+
"""An experimental branch (0009): the first `point` model calls of the run `parent`
|
|
124
|
+
was recorded from, replayed from it, then `model`, the scripted model or a live one.
|
|
125
|
+
|
|
126
|
+
`point` is the branch point, K: how many of the parent's calls the branch replays,
|
|
127
|
+
from 0 to all of them. The branch always records its own cassette, whatever
|
|
128
|
+
`model`'s `record` says.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
parent: Cassette
|
|
132
|
+
point: int
|
|
133
|
+
model: Scripted | Live = SCRIPTED
|
|
134
|
+
|
|
135
|
+
def __post_init__(self) -> None:
|
|
136
|
+
check_branch_point(self.parent, self.point)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
ModelSpec = Scripted | Replay | Live | Branch
|
|
140
|
+
"""Which model adapter a run uses."""
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(frozen=True)
|
|
144
|
+
class Parent:
|
|
145
|
+
"""The run a branch comes from (0009): the parent's run id, and the branch point, K,
|
|
146
|
+
the number of the parent's model calls the branch replays."""
|
|
147
|
+
|
|
148
|
+
run_id: str
|
|
149
|
+
branch_point: int
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def branch_run_id(parent: Parent) -> str:
|
|
153
|
+
"""A branch's run id: its parent's run id, `.branch-` and its branch point, such as
|
|
154
|
+
`dev-001-single-po.G2.C2.7.branch-4`. A branch of a branch adds a second suffix."""
|
|
155
|
+
return f"{parent.run_id}.branch-{parent.branch_point}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def parent_of(spec: ModelSpec) -> Parent | None:
|
|
159
|
+
"""The parent of a run with the model `spec`: a branch's, or, for a replay of a
|
|
160
|
+
branch's cassette, the one its header names. None for any other run."""
|
|
161
|
+
if isinstance(spec, Branch):
|
|
162
|
+
return Parent(spec.parent.run.run_id, spec.point)
|
|
163
|
+
if isinstance(spec, Replay):
|
|
164
|
+
header = spec.cassette.run
|
|
165
|
+
if header.parent_run_id is not None and header.branch_point is not None:
|
|
166
|
+
return Parent(header.parent_run_id, header.branch_point)
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class LiveRefused(ValueError):
|
|
171
|
+
"""A live run that must not start: the flag, the spend cap, the model's price or the
|
|
172
|
+
provider's API key is missing, or a variable would send the provider's requests to
|
|
173
|
+
another backend."""
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def check_live_flag(environ: Mapping[str, str]) -> None:
|
|
177
|
+
"""Refuse any live run unless `environ` sets `FIVEFOLD_LIVE=1`."""
|
|
178
|
+
if environ.get(LIVE_FLAG) != "1":
|
|
179
|
+
raise LiveRefused(
|
|
180
|
+
f"live runs are off: set {LIVE_FLAG}=1 to allow one, with a spend cap and a price file"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def check_live(spec: Live, environ: Mapping[str, str]) -> tuple[SpendMeter, ModelPrice]:
|
|
185
|
+
"""Refuse a live run unless `environ` sets `FIVEFOLD_LIVE=1`, the run has a spend cap
|
|
186
|
+
with a price file that prices its model, and `environ` sets the provider's API key and
|
|
187
|
+
none of the variables the provider refuses; return the meter and the price.
|
|
188
|
+
|
|
189
|
+
The key and the refused variables are checked last, and only for being set; a
|
|
190
|
+
provider this build does not have is refused when its model is built
|
|
191
|
+
(`fivefold.agents.live.check_provider`).
|
|
192
|
+
"""
|
|
193
|
+
check_live_flag(environ)
|
|
194
|
+
if spec.spend is None:
|
|
195
|
+
raise LiveRefused("a live run needs a spend cap in USD and a price file")
|
|
196
|
+
if not (math.isfinite(spec.limit_s) and spec.limit_s > 0):
|
|
197
|
+
raise LiveRefused(f"a live call's time limit must be above 0 seconds, not {spec.limit_s}")
|
|
198
|
+
price = spec.spend.prices.price(spec.provider, spec.model)
|
|
199
|
+
entry = PROVIDERS.get(spec.provider)
|
|
200
|
+
if entry is not None and entry.keys and not any(environ.get(key) for key in entry.keys):
|
|
201
|
+
raise LiveRefused(
|
|
202
|
+
f"a live run of {spec.provider} needs its API key in {' or '.join(entry.keys)}"
|
|
203
|
+
)
|
|
204
|
+
for name in entry.forbidden if entry is not None else ():
|
|
205
|
+
if environ.get(name):
|
|
206
|
+
raise LiveRefused(
|
|
207
|
+
f"a live run of {spec.provider} refuses {name}, which would send its requests"
|
|
208
|
+
" to another backend, with other credentials: unset it"
|
|
209
|
+
)
|
|
210
|
+
return spec.spend, price
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def run_header(record: RunRecord) -> RunHeader:
|
|
214
|
+
"""What a cassette says about the run it was recorded from: the run's record, without
|
|
215
|
+
the model, which the cassette names on its own."""
|
|
216
|
+
return RunHeader(
|
|
217
|
+
run_id=record.run_id,
|
|
218
|
+
scenario_id=record.scenario_id,
|
|
219
|
+
scenario_digest=record.scenario_digest,
|
|
220
|
+
config=record.config,
|
|
221
|
+
contract=record.contract,
|
|
222
|
+
seed=record.seed,
|
|
223
|
+
topology=record.topology,
|
|
224
|
+
params=plain_object(record.params, "the run's params"),
|
|
225
|
+
parent_run_id=record.parent_run_id,
|
|
226
|
+
branch_point=record.branch_point,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@dataclass(frozen=True)
|
|
231
|
+
class _LiveSetup:
|
|
232
|
+
"""A live run's model, built once the run passed `check_live`."""
|
|
233
|
+
|
|
234
|
+
spec: Live
|
|
235
|
+
chat: BaseChatModel
|
|
236
|
+
meter: SpendMeter
|
|
237
|
+
price: ModelPrice
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class RunModels:
|
|
241
|
+
"""The chat models of one run's agent workers, and the run's cassette."""
|
|
242
|
+
|
|
243
|
+
def __init__(self, spec: ModelSpec, *, environ: Mapping[str, str] | None = None) -> None:
|
|
244
|
+
"""`environ` is where a live run looks for its flag: the process environment by
|
|
245
|
+
default. A live run is refused here, before the run records anything, and so is a
|
|
246
|
+
live branch."""
|
|
247
|
+
self.spec = spec
|
|
248
|
+
self._recorder: CassetteRecorder | None = None
|
|
249
|
+
self._player: CassettePlayer | None = None
|
|
250
|
+
self._branch: BranchPlayer | None = None
|
|
251
|
+
self._live: _LiveSetup | None = None
|
|
252
|
+
self.cassette: Cassette | None = None
|
|
253
|
+
"""The cassette a replay serves, or the one a recording made, once complete."""
|
|
254
|
+
if isinstance(spec, Replay):
|
|
255
|
+
self._player = CassettePlayer(spec.cassette)
|
|
256
|
+
self.cassette = spec.cassette
|
|
257
|
+
elif isinstance(spec, Branch):
|
|
258
|
+
self._branch = BranchPlayer(spec.parent, spec.point)
|
|
259
|
+
self._recorder = CassetteRecorder(self._answering(spec.model, environ))
|
|
260
|
+
elif isinstance(spec, Live):
|
|
261
|
+
self._recorder = CassetteRecorder(self._answering(spec, environ))
|
|
262
|
+
elif spec.record:
|
|
263
|
+
self._recorder = CassetteRecorder(SCRIPTED_MODEL)
|
|
264
|
+
|
|
265
|
+
def _answering(self, spec: Scripted | Live, environ: Mapping[str, str] | None) -> ModelInfo:
|
|
266
|
+
"""Set up the model that answers the calls, live after `check_live`; return it as
|
|
267
|
+
the cassette names it."""
|
|
268
|
+
if isinstance(spec, Scripted):
|
|
269
|
+
return SCRIPTED_MODEL
|
|
270
|
+
meter, price = check_live(spec, os.environ if environ is None else environ)
|
|
271
|
+
chat = provider_model(spec.provider, spec.model, spec.params)
|
|
272
|
+
self._live = _LiveSetup(spec, chat, meter, price)
|
|
273
|
+
params = plain_object(spec.params, "the model's parameters")
|
|
274
|
+
return ModelInfo(adapter="live", name=spec.name, params=params)
|
|
275
|
+
|
|
276
|
+
@property
|
|
277
|
+
def adapter(self) -> Literal["scripted", "live", "replay"]:
|
|
278
|
+
"""The adapter, as `harness.run.model_adapter` records it: for a branch, the
|
|
279
|
+
adapter of the model it switches to."""
|
|
280
|
+
if self._player is not None:
|
|
281
|
+
return "replay"
|
|
282
|
+
return "live" if self._live is not None else "scripted"
|
|
283
|
+
|
|
284
|
+
@property
|
|
285
|
+
def name(self) -> str:
|
|
286
|
+
"""The model's name, as `harness.run.model` records it: a replay names the
|
|
287
|
+
recorded model, and a branch the model it switches to."""
|
|
288
|
+
if self._player is not None:
|
|
289
|
+
return self._player.cassette.model.name
|
|
290
|
+
if self._live is not None:
|
|
291
|
+
return self._live.spec.name
|
|
292
|
+
return SCRIPTED_MODEL.name
|
|
293
|
+
|
|
294
|
+
@property
|
|
295
|
+
def cassette_digest(self) -> str | None:
|
|
296
|
+
"""The digest `harness.run` records when the run starts: a replay's cassette."""
|
|
297
|
+
return self._player.cassette.digest() if self._player is not None else None
|
|
298
|
+
|
|
299
|
+
@property
|
|
300
|
+
def served(self) -> int:
|
|
301
|
+
"""How many recorded calls a replay or a branch has answered from its cassette."""
|
|
302
|
+
if self._player is not None:
|
|
303
|
+
return self._player.served
|
|
304
|
+
return self._branch.served if self._branch is not None else 0
|
|
305
|
+
|
|
306
|
+
def chat_model(
|
|
307
|
+
self,
|
|
308
|
+
worker: str,
|
|
309
|
+
*,
|
|
310
|
+
scripted: Callable[[Think], BaseChatModel],
|
|
311
|
+
think: Think,
|
|
312
|
+
hold: Hold | None = None,
|
|
313
|
+
) -> BaseChatModel:
|
|
314
|
+
"""The chat model of one incarnation of `worker`.
|
|
315
|
+
|
|
316
|
+
`scripted` builds the worker's scripted model, which takes the think it is given
|
|
317
|
+
before each call; `think` takes `think_ms` of virtual time; `hold` awaits a live
|
|
318
|
+
model's provider with virtual time held still (`Scheduler.hold`), which a live
|
|
319
|
+
model needs. Every call takes `think` once: a branch takes it itself, before it
|
|
320
|
+
decides where the call goes, so the model it switches to takes none.
|
|
321
|
+
"""
|
|
322
|
+
if self._player is not None:
|
|
323
|
+
return ReplayModel(self._player, worker=worker, think=think)
|
|
324
|
+
if self._branch is not None:
|
|
325
|
+
if self._recorder is None:
|
|
326
|
+
raise ValueError("a branch records its cassette")
|
|
327
|
+
then = self._answering_model(scripted, _no_think, hold)
|
|
328
|
+
branch = BranchModel(self._branch, then, worker=worker, think=think)
|
|
329
|
+
return RecordingModel(branch, self._recorder, worker=worker)
|
|
330
|
+
model = self._answering_model(scripted, think, hold)
|
|
331
|
+
if self._recorder is not None:
|
|
332
|
+
return RecordingModel(model, self._recorder, worker=worker)
|
|
333
|
+
return model
|
|
334
|
+
|
|
335
|
+
def _answering_model(
|
|
336
|
+
self, scripted: Callable[[Think], BaseChatModel], think: Think, hold: Hold | None
|
|
337
|
+
) -> BaseChatModel:
|
|
338
|
+
"""The model that answers the calls, taking `think` before each one: the live
|
|
339
|
+
adapter, or the scripted model."""
|
|
340
|
+
if self._live is None:
|
|
341
|
+
return scripted(think)
|
|
342
|
+
if hold is None or self._recorder is None:
|
|
343
|
+
raise ValueError("a live model needs the scheduler's hold and the recorder")
|
|
344
|
+
return LiveModel(
|
|
345
|
+
self._live.chat,
|
|
346
|
+
name=self._live.spec.name,
|
|
347
|
+
price=self._live.price,
|
|
348
|
+
meter=self._live.meter,
|
|
349
|
+
think=think,
|
|
350
|
+
hold=hold,
|
|
351
|
+
limit_s=self._live.spec.limit_s,
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
def stopped(self, run: RunRecord) -> str | None:
|
|
355
|
+
"""The run has stopped, so no model call is left. A recording's cassette is then
|
|
356
|
+
complete: keep it, and return its digest for `harness.run`."""
|
|
357
|
+
if self._recorder is None:
|
|
358
|
+
return None
|
|
359
|
+
self.cassette = self._recorder.cassette(run_header(run))
|
|
360
|
+
return self.cassette.digest()
|
|
361
|
+
|
|
362
|
+
def check(self) -> None:
|
|
363
|
+
"""Fail the run, with `ReplayError`, if a replay did not serve exactly the
|
|
364
|
+
recorded calls, or a branch did not replay its parent's first K calls (0009)."""
|
|
365
|
+
if self._player is not None:
|
|
366
|
+
self._player.finish()
|
|
367
|
+
if self._branch is not None:
|
|
368
|
+
self._branch.finish()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""The run registry (0009, 0012): one row per run database, so the viewer can list runs.
|
|
2
|
+
|
|
3
|
+
The registry is a database of its own on the server that holds the run databases:
|
|
4
|
+
`fivefold_registry`, unless FIVEFOLD_REGISTRY_DB names another (`fivefold.harness.
|
|
5
|
+
migrations.registry_database`). `bootstrap_server` in that module creates it and applies
|
|
6
|
+
its migrations, `db/registry/`. A row holds the run database's name, the run's id, its
|
|
7
|
+
scenario, configuration, contract level and seed, and when the row was written, in wall
|
|
8
|
+
time, for display only (0009).
|
|
9
|
+
|
|
10
|
+
The harness writes a run's row when the run records itself in its run database
|
|
11
|
+
(`InprocRun.execute`), so every run gets one, whoever made its database: a demo, a study,
|
|
12
|
+
playback or a test. A database made again under an old name, as each demo's is, gets its
|
|
13
|
+
row replaced. `forget` removes the row of a run database that is dropped.
|
|
14
|
+
|
|
15
|
+
The harness writes as role `ff_registry`, which has no privilege on any run database's
|
|
16
|
+
schemas; the viewer reads as `ff_viewer` (`db/registry/0001_registry.sql`).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Final
|
|
23
|
+
|
|
24
|
+
import psycopg
|
|
25
|
+
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
|
26
|
+
from psycopg.rows import TupleRow
|
|
27
|
+
|
|
28
|
+
from fivefold.harness.migrations import check_database_name, registry_database
|
|
29
|
+
|
|
30
|
+
REGISTRY_ROLE: Final = "ff_registry"
|
|
31
|
+
"""The role the harness writes the registry as."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RegistryError(Exception):
|
|
35
|
+
"""The run registry cannot be reached, or a run's database cannot be named."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class RegistryEntry:
|
|
40
|
+
"""One run database's row: what the registry lists for the viewer (0012)."""
|
|
41
|
+
|
|
42
|
+
database: str
|
|
43
|
+
run_id: str
|
|
44
|
+
scenario_id: str
|
|
45
|
+
config: str
|
|
46
|
+
contract: str
|
|
47
|
+
seed: int
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def database_of(dsn: str) -> str:
|
|
51
|
+
"""The name of the database `dsn` connects to. It must name one."""
|
|
52
|
+
name = conninfo_to_dict(dsn).get("dbname")
|
|
53
|
+
if not isinstance(name, str) or not name:
|
|
54
|
+
raise RegistryError("the run's connection string names no database")
|
|
55
|
+
check_database_name(name)
|
|
56
|
+
return name
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def register(dsn: str, entry: RegistryEntry, *, registry: str | None = None) -> None:
|
|
60
|
+
"""Write `entry` to the registry on the server `dsn` names, replacing the row its
|
|
61
|
+
database had, as `ff_registry`.
|
|
62
|
+
|
|
63
|
+
`registry` defaults to `registry_database()`. Raises `RegistryError` when the
|
|
64
|
+
registry cannot be reached, for example because it was never created.
|
|
65
|
+
"""
|
|
66
|
+
name = registry_database() if registry is None else registry
|
|
67
|
+
check_database_name(entry.database)
|
|
68
|
+
with _connect(dsn, name) as conn:
|
|
69
|
+
conn.execute(
|
|
70
|
+
"INSERT INTO registry.run AS r"
|
|
71
|
+
" (database_name, run_id, scenario_id, config, contract, seed)"
|
|
72
|
+
" VALUES (%s, %s, %s, %s, %s, %s)"
|
|
73
|
+
" ON CONFLICT (database_name) DO UPDATE SET run_id = EXCLUDED.run_id,"
|
|
74
|
+
" scenario_id = EXCLUDED.scenario_id, config = EXCLUDED.config,"
|
|
75
|
+
" contract = EXCLUDED.contract, seed = EXCLUDED.seed,"
|
|
76
|
+
" created_wall_at = EXCLUDED.created_wall_at",
|
|
77
|
+
(
|
|
78
|
+
entry.database,
|
|
79
|
+
entry.run_id,
|
|
80
|
+
entry.scenario_id,
|
|
81
|
+
entry.config,
|
|
82
|
+
entry.contract,
|
|
83
|
+
entry.seed,
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def forget(dsn: str, database: str, *, registry: str | None = None) -> None:
|
|
89
|
+
"""Remove the row of `database` from the registry on the server `dsn` names, if it
|
|
90
|
+
has one. For a run database that was dropped."""
|
|
91
|
+
name = registry_database() if registry is None else registry
|
|
92
|
+
with _connect(dsn, name) as conn:
|
|
93
|
+
conn.execute("DELETE FROM registry.run WHERE database_name = %s", (database,))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _connect(dsn: str, registry: str) -> psycopg.Connection[TupleRow]:
|
|
97
|
+
try:
|
|
98
|
+
return psycopg.connect(
|
|
99
|
+
make_conninfo(dsn, dbname=registry, user=REGISTRY_ROLE), autocommit=True
|
|
100
|
+
)
|
|
101
|
+
except psycopg.OperationalError as error:
|
|
102
|
+
raise RegistryError(
|
|
103
|
+
f"the run registry {registry} cannot be reached ({error}); create it with"
|
|
104
|
+
" `python -m fivefold.harness.migrations bootstrap`"
|
|
105
|
+
) from error
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""The harness's report sink, where every final report lands (0004, 0008).
|
|
2
|
+
|
|
3
|
+
In G modes the gateway hands each report it accepted (G2) or forwarded (G1) to the sink;
|
|
4
|
+
in baselines the agent submits to the sink directly, unchecked. The sink checks the
|
|
5
|
+
report schema, records the report with whether it was valid, and never refuses one: the
|
|
6
|
+
evaluator counts the last valid report before the run stops (spec, Gateway).
|
|
7
|
+
|
|
8
|
+
PostgreSQL's `jsonb` cannot hold a NUL character or a number that is not finite. A
|
|
9
|
+
report that holds either is recorded as `{"json_text": ...}`, its JSON as text, from
|
|
10
|
+
which `json.loads` gives back the report unchanged. Such a report is always invalid,
|
|
11
|
+
because the report schema allows neither.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
from collections.abc import Collection
|
|
19
|
+
from typing import Literal
|
|
20
|
+
|
|
21
|
+
from fivefold.domain.report import InvalidReport, parse_report
|
|
22
|
+
from fivefold.harness.log import HarnessLog, JsonValue, ReportRecord
|
|
23
|
+
from fivefold.harness.scheduler import Scheduler
|
|
24
|
+
from fivefold.harness.transport import Envelope, Handler, Reply
|
|
25
|
+
|
|
26
|
+
TEXT_KEY = "json_text"
|
|
27
|
+
"""The one key of a report recorded as JSON text."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ReportSink:
|
|
31
|
+
"""Records final reports in the harness log."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, scheduler: Scheduler, log: HarnessLog, intents: Collection[str]) -> None:
|
|
34
|
+
self._scheduler = scheduler
|
|
35
|
+
self._log = log
|
|
36
|
+
self._intents = frozenset(intents)
|
|
37
|
+
|
|
38
|
+
def submit(
|
|
39
|
+
self,
|
|
40
|
+
worker: str,
|
|
41
|
+
report: JsonValue,
|
|
42
|
+
*,
|
|
43
|
+
accepted_by: Literal["gateway", "sink"],
|
|
44
|
+
message_seq: int | None = None,
|
|
45
|
+
) -> tuple[str, ...]:
|
|
46
|
+
"""Record a report; return what breaks its schema, if anything."""
|
|
47
|
+
text = json.dumps(report, sort_keys=True)
|
|
48
|
+
kept: JsonValue = json.loads(text)
|
|
49
|
+
errors: tuple[str, ...] = ()
|
|
50
|
+
try:
|
|
51
|
+
parse_report(kept, self._intents)
|
|
52
|
+
except InvalidReport as invalid:
|
|
53
|
+
errors = tuple(invalid.errors)
|
|
54
|
+
if not storable(kept):
|
|
55
|
+
kept = {TEXT_KEY: text}
|
|
56
|
+
self._log.report_submitted(
|
|
57
|
+
ReportRecord(
|
|
58
|
+
seq=self._log.next_seq(),
|
|
59
|
+
worker=worker,
|
|
60
|
+
report=kept,
|
|
61
|
+
valid=not errors,
|
|
62
|
+
errors=errors,
|
|
63
|
+
accepted_by=accepted_by,
|
|
64
|
+
vt=self._scheduler.now,
|
|
65
|
+
message_seq=message_seq,
|
|
66
|
+
)
|
|
67
|
+
)
|
|
68
|
+
return errors
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def storable(value: JsonValue) -> bool:
|
|
72
|
+
"""Whether `jsonb` can hold `value`: no NUL character, and only finite numbers."""
|
|
73
|
+
if isinstance(value, str):
|
|
74
|
+
return "\x00" not in value
|
|
75
|
+
if isinstance(value, float):
|
|
76
|
+
return math.isfinite(value)
|
|
77
|
+
if isinstance(value, dict):
|
|
78
|
+
return all(storable(key) and storable(item) for key, item in value.items())
|
|
79
|
+
if isinstance(value, list):
|
|
80
|
+
return all(storable(item) for item in value)
|
|
81
|
+
return True
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def sink_endpoint(sink: ReportSink) -> Handler:
|
|
85
|
+
"""The report sink on the transport, for baselines' `submit_final_report` (0008).
|
|
86
|
+
|
|
87
|
+
The sink records every report, valid or not. Like the gateway's tool, it refuses a
|
|
88
|
+
report that breaks the schema, so the agent can fix it and resubmit.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
async def handle(request: Envelope) -> Reply:
|
|
92
|
+
errors = sink.submit(
|
|
93
|
+
request.sender, request.body, accepted_by="sink", message_seq=request.seq
|
|
94
|
+
)
|
|
95
|
+
if errors:
|
|
96
|
+
details = {"message": "; ".join(errors)}
|
|
97
|
+
return Reply({"result": "refused", "reason": "invalid_args", "details": details})
|
|
98
|
+
return Reply({"result": "accepted"})
|
|
99
|
+
|
|
100
|
+
return handle
|
fivefold/harness/rng.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The run's seeded generators (0009).
|
|
2
|
+
|
|
3
|
+
The run seed feeds independent named generators (latency, fault plans, data generation,
|
|
4
|
+
scripted variation), so adding a new consumer does not shift another's numbers. No other
|
|
5
|
+
module may use the global `random` module, `secrets`, `uuid4` or `os.urandom` (0007).
|
|
6
|
+
|
|
7
|
+
One thing here is not seeded: `credential`, the secret of a split run's per-run credential
|
|
8
|
+
(0008). A secret that the run's seed could reproduce would be no secret, so it comes from
|
|
9
|
+
the system's source. It decides nothing in the run: it only lets a caller prove who it is.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import random
|
|
16
|
+
import secrets
|
|
17
|
+
|
|
18
|
+
Generator = random.Random
|
|
19
|
+
"""A seeded generator. Other modules name the type through this alias, never `random`."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Generators:
|
|
23
|
+
"""Independent random generators, each derived from the run seed and its name."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, seed: int) -> None:
|
|
26
|
+
self._seed = seed
|
|
27
|
+
self._made: dict[str, Generator] = {}
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def seed(self) -> int:
|
|
31
|
+
"""The run seed."""
|
|
32
|
+
return self._seed
|
|
33
|
+
|
|
34
|
+
def get(self, name: str) -> Generator:
|
|
35
|
+
"""The generator called `name`, the same one on every call in this run.
|
|
36
|
+
|
|
37
|
+
Its seed is the first 8 bytes of SHA-256 over the run seed and the name, so two
|
|
38
|
+
names never share a stream and a new name never shifts an existing one.
|
|
39
|
+
"""
|
|
40
|
+
if not name:
|
|
41
|
+
raise ValueError("a generator needs a name")
|
|
42
|
+
made = self._made.get(name)
|
|
43
|
+
if made is None:
|
|
44
|
+
digest = hashlib.sha256(f"{self._seed}\x00{name}".encode()).digest()
|
|
45
|
+
made = random.Random(int.from_bytes(digest[:8], "big"))
|
|
46
|
+
self._made[name] = made
|
|
47
|
+
return made
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
CREDENTIAL_BYTES = 32
|
|
51
|
+
"""The random bytes in each credential's secret: 256 bits."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def credential() -> str:
|
|
55
|
+
"""A new secret for a split run's credential (0008), as URL-safe text, from the system's
|
|
56
|
+
random source. Never seeded: see the module's docstring."""
|
|
57
|
+
return secrets.token_urlsafe(CREDENTIAL_BYTES)
|