fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
fivefold/harness/log.py
ADDED
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
"""The harness's records of a run: messages, faults, scripted humans, reports and model
|
|
2
|
+
calls (0004).
|
|
3
|
+
|
|
4
|
+
The evaluator scores from these records and the simulator's, never from the gateway's
|
|
5
|
+
(0004). One counter numbers every harness event in order: a message's send and its
|
|
6
|
+
delivery, a fault firing, an action of a scripted human and a submitted report. So "this
|
|
7
|
+
message was delivered before that report" is a comparison of two numbers. Each fault
|
|
8
|
+
names the task intent it touched, which recovery latency starts from (0014).
|
|
9
|
+
|
|
10
|
+
The database stamps each record with the wall-clock time it was written (`wall_at`), and
|
|
11
|
+
a delivery with the time of its fate (`delivered_wall_at`), for metrics only (0009).
|
|
12
|
+
|
|
13
|
+
Model calls have their own numbers, in the order the model answered them, which is the
|
|
14
|
+
order of the run's cassette (0009). They take none of the event counter's, so logging them
|
|
15
|
+
leaves every other record's numbers as they were. The evaluator computes record 0004's
|
|
16
|
+
token metrics from them, and its report prices those tokens.
|
|
17
|
+
|
|
18
|
+
`MemoryHarnessLog` keeps the records in memory, for unit tests. `PgHarnessLog` writes them
|
|
19
|
+
to the `harness` schema of the run's database (`db/migrations/0004_harness.sql`) as role
|
|
20
|
+
`ff_harness`. Both number events the same way. `SharedPgHarnessLog` is the split
|
|
21
|
+
topology's: the simulator's and the gateway's services each write the records of the
|
|
22
|
+
traffic they see, and draw event numbers from one database sequence (0009,
|
|
23
|
+
`db/migrations/0013_split_topology.sql`).
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from abc import ABC, abstractmethod
|
|
29
|
+
from collections.abc import Mapping
|
|
30
|
+
from dataclasses import dataclass, field, replace
|
|
31
|
+
from typing import Literal
|
|
32
|
+
|
|
33
|
+
import psycopg
|
|
34
|
+
from psycopg.rows import TupleRow
|
|
35
|
+
from psycopg.types.json import Jsonb
|
|
36
|
+
|
|
37
|
+
JsonValue = object
|
|
38
|
+
"""A value `json.dumps` accepts: dicts, lists, text, numbers, booleans and None."""
|
|
39
|
+
|
|
40
|
+
Kind = Literal["request", "answer", "notification"]
|
|
41
|
+
Fate = Literal["delivered", "dropped"]
|
|
42
|
+
DropReason = Literal[
|
|
43
|
+
"fault", "receiver_down", "caller_timed_out", "caller_crashed", "answer_to_copy"
|
|
44
|
+
]
|
|
45
|
+
"""Why a message was dropped. `answer_to_copy` is the answer to a request the transport
|
|
46
|
+
delivered a second time (F4): the transport never delivers it to the sender (0006)."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
NUL_REFUSED = "it holds a NUL character (\\u0000), which the harness's log cannot store"
|
|
50
|
+
"""Why a message holding a NUL is refused, before anything logs it (`holds_nul`)."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def holds_nul(value: object) -> bool:
|
|
54
|
+
"""Whether a JSON value holds a NUL character, in a string or a key. PostgreSQL's JSONB
|
|
55
|
+
cannot store one, so neither can the harness's log. Both topologies refuse an agent's
|
|
56
|
+
message holding one before anything logs it, as a malformed one is refused: the split
|
|
57
|
+
services (`fivefold.harness.split`) and the inproc agent's connection to the gateway
|
|
58
|
+
(`fivefold.harness.mcp_link`). B0's tools need no such check: every request model, the
|
|
59
|
+
final report's included, already refuses a NUL in any field."""
|
|
60
|
+
pending = [value]
|
|
61
|
+
while pending:
|
|
62
|
+
item = pending.pop()
|
|
63
|
+
if isinstance(item, str):
|
|
64
|
+
if "\x00" in item:
|
|
65
|
+
return True
|
|
66
|
+
elif isinstance(item, Mapping):
|
|
67
|
+
for key, inner in item.items():
|
|
68
|
+
if isinstance(key, str) and "\x00" in key:
|
|
69
|
+
return True
|
|
70
|
+
pending.append(inner)
|
|
71
|
+
elif isinstance(item, list | tuple):
|
|
72
|
+
pending.extend(item)
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class MessageRecord:
|
|
78
|
+
"""One message on one hop, as sent. Its fate comes later (0004, 0006)."""
|
|
79
|
+
|
|
80
|
+
seq: int
|
|
81
|
+
hop: str
|
|
82
|
+
kind: Kind
|
|
83
|
+
sender: str
|
|
84
|
+
receiver: str
|
|
85
|
+
api: str | None
|
|
86
|
+
correlation: int | None
|
|
87
|
+
"""For an answer, the seq of the request it answers."""
|
|
88
|
+
intent_id: str | None
|
|
89
|
+
idempotency_key: str | None
|
|
90
|
+
answer_class: str | None
|
|
91
|
+
body: JsonValue
|
|
92
|
+
sent_vt: int
|
|
93
|
+
fate: Fate | None = None
|
|
94
|
+
drop_reason: DropReason | None = None
|
|
95
|
+
delivered_seq: int | None = None
|
|
96
|
+
delivered_vt: int | None = None
|
|
97
|
+
copy_of: int | None = None
|
|
98
|
+
"""For a copy that the transport made of a request (F4), the seq of that request. Its
|
|
99
|
+
sender sent the request once; the copy is the second delivery (0006)."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True)
|
|
103
|
+
class FaultRecord:
|
|
104
|
+
"""A fault that fired, and what it did (0006)."""
|
|
105
|
+
|
|
106
|
+
seq: int
|
|
107
|
+
fault: str
|
|
108
|
+
variant: str | None
|
|
109
|
+
point: str
|
|
110
|
+
occurrence: int
|
|
111
|
+
action: str
|
|
112
|
+
vt: int
|
|
113
|
+
target_seq: int | None = None
|
|
114
|
+
"""The message the fault acted on, if any."""
|
|
115
|
+
component: str | None = None
|
|
116
|
+
"""The component the fault acted on, if any: the one a crash stopped, the worker F9
|
|
117
|
+
started, or the gateway where F6 or F7 fired at a dispatch point."""
|
|
118
|
+
voided: tuple[int, ...] = ()
|
|
119
|
+
"""The seqs of messages whose delivery the fault voided (0006, crashes)."""
|
|
120
|
+
params: Mapping[str, object] = field(default_factory=dict)
|
|
121
|
+
intent_id: str | None = None
|
|
122
|
+
"""The task intent the fault touched: the intent of the message, action or tool call
|
|
123
|
+
it acted on. None for F9, which touches every task intent, and for a message without
|
|
124
|
+
an intent, such as a read (0014). F10 gets one more record for each status answer it
|
|
125
|
+
changes, naming that answer and its intent."""
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True)
|
|
129
|
+
class RunRecord:
|
|
130
|
+
"""What a run is: its scenario, configuration, contract, seed and model (0009)."""
|
|
131
|
+
|
|
132
|
+
run_id: str
|
|
133
|
+
scenario_id: str
|
|
134
|
+
scenario_digest: str
|
|
135
|
+
config: Literal["B0", "G1", "G2"]
|
|
136
|
+
contract: Literal["C0", "C1", "C2", "C3"]
|
|
137
|
+
seed: int
|
|
138
|
+
model_adapter: Literal["scripted", "live", "replay", "external"]
|
|
139
|
+
"""`external` for a split run, whose agent is the user's own: the harness runs no model
|
|
140
|
+
for it, and `model` is the name the user gave."""
|
|
141
|
+
model: str
|
|
142
|
+
params: Mapping[str, object]
|
|
143
|
+
topology: Literal["inproc", "split"] = "inproc"
|
|
144
|
+
cassette_digest: str | None = None
|
|
145
|
+
"""The cassette a replay serves, or the one a recording made (`cassette_recorded`)."""
|
|
146
|
+
parent_run_id: str | None = None
|
|
147
|
+
"""For an experimental branch, the run id of the run it branched from (0009)."""
|
|
148
|
+
branch_point: int | None = None
|
|
149
|
+
"""For an experimental branch, K: how many of its parent's model calls it replayed
|
|
150
|
+
before it switched model (0009). Set exactly when `parent_run_id` is."""
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class ActorRecord:
|
|
155
|
+
"""An action of a scripted human: the approver or the resolver (0002, 0004)."""
|
|
156
|
+
|
|
157
|
+
seq: int
|
|
158
|
+
actor: Literal["approver", "resolver"]
|
|
159
|
+
action: str
|
|
160
|
+
subject: str
|
|
161
|
+
details: Mapping[str, object]
|
|
162
|
+
vt: int
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class ReportRecord:
|
|
167
|
+
"""A final report an agent worker submitted, as the report sink received it (0004).
|
|
168
|
+
|
|
169
|
+
`valid` says whether it met the report schema; `errors` says how it did not.
|
|
170
|
+
`accepted_by` is `gateway` when G2 accepted it, or G1 forwarded it, and `sink` when
|
|
171
|
+
a baseline agent submitted it to the sink directly (0008).
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
seq: int
|
|
175
|
+
worker: str
|
|
176
|
+
report: JsonValue
|
|
177
|
+
valid: bool
|
|
178
|
+
errors: tuple[str, ...]
|
|
179
|
+
accepted_by: Literal["gateway", "sink"]
|
|
180
|
+
vt: int
|
|
181
|
+
message_seq: int | None = None
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass(frozen=True)
|
|
185
|
+
class ModelCallRecord:
|
|
186
|
+
"""One model call of an agent worker, as its model answered it (0004, 0009)."""
|
|
187
|
+
|
|
188
|
+
call_no: int
|
|
189
|
+
"""The call's number in the run, from 1, in the order the model answered."""
|
|
190
|
+
worker: str
|
|
191
|
+
asked_vt: int
|
|
192
|
+
answered_vt: int
|
|
193
|
+
model: str
|
|
194
|
+
"""The run's model, as `harness.run.model` names it."""
|
|
195
|
+
input_tokens: int | None
|
|
196
|
+
"""None when the reply reports no usage, as scripted replies do."""
|
|
197
|
+
output_tokens: int | None
|
|
198
|
+
refused: str | None
|
|
199
|
+
"""Why the harness answered instead of the model, such as `spend_cap`; None when the
|
|
200
|
+
model answered."""
|
|
201
|
+
wall_ms: float
|
|
202
|
+
"""Wall-clock milliseconds from the call to its answer, for metrics only (0009)."""
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class HarnessLog(ABC):
|
|
206
|
+
"""Numbers harness events and records messages, faults, reports and model calls."""
|
|
207
|
+
|
|
208
|
+
@abstractmethod
|
|
209
|
+
def run_recorded(self, record: RunRecord) -> None:
|
|
210
|
+
"""Record the run itself, once, before anything happens in it."""
|
|
211
|
+
|
|
212
|
+
@abstractmethod
|
|
213
|
+
def run_stopped(self, vt: int) -> None:
|
|
214
|
+
"""Record when the run stopped, once, at the end of the drain window (0009)."""
|
|
215
|
+
|
|
216
|
+
@abstractmethod
|
|
217
|
+
def cassette_recorded(self, digest: str) -> None:
|
|
218
|
+
"""Record, once, the digest of the cassette the run recorded (0009). It is known
|
|
219
|
+
only when the run stops, after the run itself was recorded."""
|
|
220
|
+
|
|
221
|
+
@abstractmethod
|
|
222
|
+
def next_seq(self) -> int:
|
|
223
|
+
"""Reserve the next event number."""
|
|
224
|
+
|
|
225
|
+
@abstractmethod
|
|
226
|
+
def message_sent(self, record: MessageRecord) -> None:
|
|
227
|
+
"""Record a message as it is sent, with no fate yet."""
|
|
228
|
+
|
|
229
|
+
@abstractmethod
|
|
230
|
+
def message_fate(
|
|
231
|
+
self,
|
|
232
|
+
seq: int,
|
|
233
|
+
fate: Fate,
|
|
234
|
+
*,
|
|
235
|
+
vt: int,
|
|
236
|
+
reason: DropReason | None = None,
|
|
237
|
+
delivered_seq: int | None = None,
|
|
238
|
+
) -> None:
|
|
239
|
+
"""Record what became of message `seq`: delivered, with the number of its
|
|
240
|
+
delivery, or dropped, with why."""
|
|
241
|
+
|
|
242
|
+
@abstractmethod
|
|
243
|
+
def fault_fired(self, record: FaultRecord) -> None:
|
|
244
|
+
"""Record a fault that fired."""
|
|
245
|
+
|
|
246
|
+
@abstractmethod
|
|
247
|
+
def report_submitted(self, record: ReportRecord) -> None:
|
|
248
|
+
"""Record a final report as the sink received it."""
|
|
249
|
+
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def actor_acted(self, record: ActorRecord) -> None:
|
|
252
|
+
"""Record what a scripted human did."""
|
|
253
|
+
|
|
254
|
+
@abstractmethod
|
|
255
|
+
def model_called(self, record: ModelCallRecord) -> None:
|
|
256
|
+
"""Record a model call once its model answered."""
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _check_fate(fate: Fate, reason: DropReason | None, delivered_seq: int | None) -> None:
|
|
260
|
+
if fate == "delivered" and (delivered_seq is None or reason is not None):
|
|
261
|
+
raise ValueError("a delivered message has a delivery number and no drop reason")
|
|
262
|
+
if fate == "dropped" and (delivered_seq is not None or reason is None):
|
|
263
|
+
raise ValueError("a dropped message has a drop reason and no delivery number")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class MemoryHarnessLog(HarnessLog):
|
|
267
|
+
"""Harness records kept in memory."""
|
|
268
|
+
|
|
269
|
+
def __init__(self) -> None:
|
|
270
|
+
self._last_seq = 0
|
|
271
|
+
self._messages: dict[int, MessageRecord] = {}
|
|
272
|
+
self.faults: list[FaultRecord] = []
|
|
273
|
+
self.reports: list[ReportRecord] = []
|
|
274
|
+
self.actors: list[ActorRecord] = []
|
|
275
|
+
self.model_calls: list[ModelCallRecord] = []
|
|
276
|
+
self.run: RunRecord | None = None
|
|
277
|
+
self.stopped_vt: int | None = None
|
|
278
|
+
|
|
279
|
+
def run_recorded(self, record: RunRecord) -> None:
|
|
280
|
+
if self.run is not None:
|
|
281
|
+
raise ValueError("the run is already recorded")
|
|
282
|
+
self.run = record
|
|
283
|
+
|
|
284
|
+
def run_stopped(self, vt: int) -> None:
|
|
285
|
+
if self.stopped_vt is not None:
|
|
286
|
+
raise ValueError("the run has already stopped")
|
|
287
|
+
self.stopped_vt = vt
|
|
288
|
+
|
|
289
|
+
def cassette_recorded(self, digest: str) -> None:
|
|
290
|
+
if self.run is None or self.run.cassette_digest is not None:
|
|
291
|
+
raise ValueError("the run was never recorded, or already names its cassette")
|
|
292
|
+
self.run = replace(self.run, cassette_digest=digest)
|
|
293
|
+
|
|
294
|
+
@property
|
|
295
|
+
def messages(self) -> list[MessageRecord]:
|
|
296
|
+
"""Every message in send order, with its fate so far."""
|
|
297
|
+
return [self._messages[seq] for seq in sorted(self._messages)]
|
|
298
|
+
|
|
299
|
+
def next_seq(self) -> int:
|
|
300
|
+
self._last_seq += 1
|
|
301
|
+
return self._last_seq
|
|
302
|
+
|
|
303
|
+
def message_sent(self, record: MessageRecord) -> None:
|
|
304
|
+
if record.seq in self._messages:
|
|
305
|
+
raise ValueError(f"message {record.seq} was already sent")
|
|
306
|
+
if record.fate is not None:
|
|
307
|
+
raise ValueError("a message has no fate when it is sent")
|
|
308
|
+
self._messages[record.seq] = record
|
|
309
|
+
|
|
310
|
+
def message_fate(
|
|
311
|
+
self,
|
|
312
|
+
seq: int,
|
|
313
|
+
fate: Fate,
|
|
314
|
+
*,
|
|
315
|
+
vt: int,
|
|
316
|
+
reason: DropReason | None = None,
|
|
317
|
+
delivered_seq: int | None = None,
|
|
318
|
+
) -> None:
|
|
319
|
+
_check_fate(fate, reason, delivered_seq)
|
|
320
|
+
record = self._messages[seq]
|
|
321
|
+
if record.fate is not None:
|
|
322
|
+
raise ValueError(f"message {seq} already has a fate")
|
|
323
|
+
self._messages[seq] = replace(
|
|
324
|
+
record,
|
|
325
|
+
fate=fate,
|
|
326
|
+
drop_reason=reason,
|
|
327
|
+
delivered_seq=delivered_seq,
|
|
328
|
+
delivered_vt=vt if fate == "delivered" else None,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
def fault_fired(self, record: FaultRecord) -> None:
|
|
332
|
+
self.faults.append(record)
|
|
333
|
+
|
|
334
|
+
def report_submitted(self, record: ReportRecord) -> None:
|
|
335
|
+
self.reports.append(record)
|
|
336
|
+
|
|
337
|
+
def actor_acted(self, record: ActorRecord) -> None:
|
|
338
|
+
self.actors.append(record)
|
|
339
|
+
|
|
340
|
+
def model_called(self, record: ModelCallRecord) -> None:
|
|
341
|
+
if record.call_no != len(self.model_calls) + 1:
|
|
342
|
+
raise ValueError(f"model call {record.call_no} is not the next call")
|
|
343
|
+
self.model_calls.append(record)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
Connection = psycopg.Connection[TupleRow]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class PgHarnessLog(HarnessLog):
|
|
350
|
+
"""Harness records in the run database's `harness` schema, written as `ff_harness`.
|
|
351
|
+
|
|
352
|
+
Each record is written at once, outside any other transaction. The harness never
|
|
353
|
+
crashes, so the event counter lives in memory, starting after the last stored event.
|
|
354
|
+
"""
|
|
355
|
+
|
|
356
|
+
def __init__(self, conn: Connection) -> None:
|
|
357
|
+
if not conn.autocommit:
|
|
358
|
+
raise ValueError("the harness log writes each record at once: use autocommit")
|
|
359
|
+
self._conn = conn
|
|
360
|
+
row = conn.execute(
|
|
361
|
+
"SELECT greatest((SELECT max(seq) FROM harness.message_log),"
|
|
362
|
+
" (SELECT max(delivered_seq) FROM harness.message_log),"
|
|
363
|
+
" (SELECT max(seq) FROM harness.fault_log),"
|
|
364
|
+
" (SELECT max(seq) FROM harness.final_report),"
|
|
365
|
+
" (SELECT max(seq) FROM harness.actor_log))"
|
|
366
|
+
).fetchone()
|
|
367
|
+
self._last_seq = 0 if row is None or row[0] is None else int(row[0])
|
|
368
|
+
self.run: RunRecord | None = None
|
|
369
|
+
"""The run as this log recorded it."""
|
|
370
|
+
|
|
371
|
+
def run_recorded(self, record: RunRecord) -> None:
|
|
372
|
+
self._conn.execute(
|
|
373
|
+
"INSERT INTO harness.run (run_id, scenario_id, scenario_digest, config, contract,"
|
|
374
|
+
" seed, topology, model_adapter, model, cassette_digest, parent_run_id,"
|
|
375
|
+
" branch_point, params) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
|
376
|
+
(
|
|
377
|
+
record.run_id,
|
|
378
|
+
record.scenario_id,
|
|
379
|
+
record.scenario_digest,
|
|
380
|
+
record.config,
|
|
381
|
+
record.contract,
|
|
382
|
+
record.seed,
|
|
383
|
+
record.topology,
|
|
384
|
+
record.model_adapter,
|
|
385
|
+
record.model,
|
|
386
|
+
record.cassette_digest,
|
|
387
|
+
record.parent_run_id,
|
|
388
|
+
record.branch_point,
|
|
389
|
+
Jsonb(dict(record.params)),
|
|
390
|
+
),
|
|
391
|
+
)
|
|
392
|
+
self.run = record
|
|
393
|
+
|
|
394
|
+
def run_stopped(self, vt: int) -> None:
|
|
395
|
+
self._conn.execute(
|
|
396
|
+
"INSERT INTO harness.run_stop (run_id, stopped_vt) SELECT run_id, %s FROM harness.run",
|
|
397
|
+
(vt,),
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
def cassette_recorded(self, digest: str) -> None:
|
|
401
|
+
updated = self._conn.execute(
|
|
402
|
+
"UPDATE harness.run SET cassette_digest = %s WHERE cassette_digest IS NULL",
|
|
403
|
+
(digest,),
|
|
404
|
+
)
|
|
405
|
+
if updated.rowcount != 1 or self.run is None:
|
|
406
|
+
raise ValueError("the run was never recorded, or already names its cassette")
|
|
407
|
+
self.run = replace(self.run, cassette_digest=digest)
|
|
408
|
+
|
|
409
|
+
def next_seq(self) -> int:
|
|
410
|
+
self._last_seq += 1
|
|
411
|
+
return self._last_seq
|
|
412
|
+
|
|
413
|
+
def message_sent(self, record: MessageRecord) -> None:
|
|
414
|
+
if record.fate is not None:
|
|
415
|
+
raise ValueError("a message has no fate when it is sent")
|
|
416
|
+
self._conn.execute(
|
|
417
|
+
"INSERT INTO harness.message_log (seq, hop, kind, sender, receiver, api,"
|
|
418
|
+
" correlation, intent_id, idempotency_key, answer_class, body, sent_vt, copy_of)"
|
|
419
|
+
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
|
420
|
+
(
|
|
421
|
+
record.seq,
|
|
422
|
+
record.hop,
|
|
423
|
+
record.kind,
|
|
424
|
+
record.sender,
|
|
425
|
+
record.receiver,
|
|
426
|
+
record.api,
|
|
427
|
+
record.correlation,
|
|
428
|
+
record.intent_id,
|
|
429
|
+
record.idempotency_key,
|
|
430
|
+
record.answer_class,
|
|
431
|
+
Jsonb(record.body),
|
|
432
|
+
record.sent_vt,
|
|
433
|
+
record.copy_of,
|
|
434
|
+
),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
def message_fate(
|
|
438
|
+
self,
|
|
439
|
+
seq: int,
|
|
440
|
+
fate: Fate,
|
|
441
|
+
*,
|
|
442
|
+
vt: int,
|
|
443
|
+
reason: DropReason | None = None,
|
|
444
|
+
delivered_seq: int | None = None,
|
|
445
|
+
) -> None:
|
|
446
|
+
_check_fate(fate, reason, delivered_seq)
|
|
447
|
+
delivered = fate == "delivered"
|
|
448
|
+
updated = self._conn.execute(
|
|
449
|
+
"UPDATE harness.message_log SET fate = %s, drop_reason = %s, delivered_seq = %s,"
|
|
450
|
+
" delivered_vt = %s, delivered_wall_at = CASE WHEN %s THEN now() END"
|
|
451
|
+
" WHERE seq = %s AND fate IS NULL",
|
|
452
|
+
(fate, reason, delivered_seq, vt if delivered else None, delivered, seq),
|
|
453
|
+
)
|
|
454
|
+
if updated.rowcount != 1:
|
|
455
|
+
raise ValueError(f"message {seq} was never sent or already has a fate")
|
|
456
|
+
|
|
457
|
+
def fault_fired(self, record: FaultRecord) -> None:
|
|
458
|
+
self._conn.execute(
|
|
459
|
+
"INSERT INTO harness.fault_log (seq, fault, variant, point, occurrence, action,"
|
|
460
|
+
" target_seq, component, voided, params, vt, intent_id)"
|
|
461
|
+
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
|
462
|
+
(
|
|
463
|
+
record.seq,
|
|
464
|
+
record.fault,
|
|
465
|
+
record.variant,
|
|
466
|
+
record.point,
|
|
467
|
+
record.occurrence,
|
|
468
|
+
record.action,
|
|
469
|
+
record.target_seq,
|
|
470
|
+
record.component,
|
|
471
|
+
list(record.voided),
|
|
472
|
+
Jsonb(dict(record.params)),
|
|
473
|
+
record.vt,
|
|
474
|
+
record.intent_id,
|
|
475
|
+
),
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
def report_submitted(self, record: ReportRecord) -> None:
|
|
479
|
+
self._conn.execute(
|
|
480
|
+
"INSERT INTO harness.final_report (seq, worker, report, valid, errors,"
|
|
481
|
+
" accepted_by, message_seq, vt) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)",
|
|
482
|
+
(
|
|
483
|
+
record.seq,
|
|
484
|
+
record.worker,
|
|
485
|
+
Jsonb(record.report),
|
|
486
|
+
record.valid,
|
|
487
|
+
Jsonb(list(record.errors)),
|
|
488
|
+
record.accepted_by,
|
|
489
|
+
record.message_seq,
|
|
490
|
+
record.vt,
|
|
491
|
+
),
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
def actor_acted(self, record: ActorRecord) -> None:
|
|
495
|
+
self._conn.execute(
|
|
496
|
+
"INSERT INTO harness.actor_log (seq, actor, action, subject, details, vt)"
|
|
497
|
+
" VALUES (%s, %s, %s, %s, %s, %s)",
|
|
498
|
+
(
|
|
499
|
+
record.seq,
|
|
500
|
+
record.actor,
|
|
501
|
+
record.action,
|
|
502
|
+
record.subject,
|
|
503
|
+
Jsonb(dict(record.details)),
|
|
504
|
+
record.vt,
|
|
505
|
+
),
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
def model_called(self, record: ModelCallRecord) -> None:
|
|
509
|
+
self._conn.execute(
|
|
510
|
+
"INSERT INTO harness.model_call (call_no, worker, asked_vt, answered_vt, model,"
|
|
511
|
+
" input_tokens, output_tokens, refused, wall_ms)"
|
|
512
|
+
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
|
513
|
+
(
|
|
514
|
+
record.call_no,
|
|
515
|
+
record.worker,
|
|
516
|
+
record.asked_vt,
|
|
517
|
+
record.answered_vt,
|
|
518
|
+
record.model,
|
|
519
|
+
record.input_tokens,
|
|
520
|
+
record.output_tokens,
|
|
521
|
+
record.refused,
|
|
522
|
+
record.wall_ms,
|
|
523
|
+
),
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
class SharedPgHarnessLog(PgHarnessLog):
|
|
528
|
+
"""Harness records that several processes of one run write: the split topology's.
|
|
529
|
+
|
|
530
|
+
Event numbers come from the sequence `harness.event_seq`, so the simulator's and the
|
|
531
|
+
gateway's services number one series of events between them, in the order they reach
|
|
532
|
+
it. A sequence never hands out a number twice. A number that a process took and never
|
|
533
|
+
used leaves a gap, which no reader relies on.
|
|
534
|
+
"""
|
|
535
|
+
|
|
536
|
+
def next_seq(self) -> int:
|
|
537
|
+
row = self._conn.execute("SELECT nextval('harness.event_seq')").fetchone()
|
|
538
|
+
if row is None:
|
|
539
|
+
raise RuntimeError("harness.event_seq gave no number")
|
|
540
|
+
return int(row[0])
|