fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""The harness's records and faults of the split services' traffic (0004, 0006, 0009).
|
|
2
|
+
|
|
3
|
+
In the inproc topology the harness's transport carries every message, logs it with its
|
|
4
|
+
fate and fires the plan's faults on it. In the split topology messages travel over real
|
|
5
|
+
HTTP connections, so each service logs the messages it sees and fires the faults at the
|
|
6
|
+
points it owns (`fivefold.harness.split.faults`). The log is the same harness log, with
|
|
7
|
+
these readings of when a message is sent and delivered:
|
|
8
|
+
|
|
9
|
+
- A message from the user's agent, which the harness cannot watch, is logged when it
|
|
10
|
+
arrives: sent and delivered at once, at the same virtual time.
|
|
11
|
+
- An answer a service sends to the agent is logged as sent, then as delivered when the
|
|
12
|
+
service hands it to its HTTP server, before the server writes it: the harness cannot
|
|
13
|
+
see whether the agent read it. The simulator first asks whether the agent's connection
|
|
14
|
+
is still open, and logs the answer as dropped (`caller_timed_out`) if it is not. The
|
|
15
|
+
gateway's answers go through the MCP SDK, which writes them itself, so the gateway logs
|
|
16
|
+
each one as delivered when its handler returns, even if the agent has gone. In G1 and
|
|
17
|
+
G2 the evaluator reads no answer on the agent's hop as evidence (0004), so this changes
|
|
18
|
+
no verdict there.
|
|
19
|
+
- On the `gw_sim` hop the gateway logs each request as it sends it, and each answer as it
|
|
20
|
+
arrives; the simulator logs each request as it arrives, and each answer as it sends it.
|
|
21
|
+
A request carries the seq the gateway logged it under, and an answer the seq the
|
|
22
|
+
simulator logged it under, in the header `fivefold-seq`.
|
|
23
|
+
- A write request is sent at the time its attempt records, `gw.attempt.sent_vt` (0003),
|
|
24
|
+
the time the gateway's settle bounds start from (0005). The gateway records the
|
|
25
|
+
attempt in a transaction, then logs the request, and on the host's clock milliseconds
|
|
26
|
+
pass in between; the log takes the attempt's time, so the evaluator's bounds start from
|
|
27
|
+
the same number (0004). A message the simulator logs meanwhile can then have a lower
|
|
28
|
+
seq and a later `sent_vt`. Nothing reads the log as if virtual time grew with seq: the
|
|
29
|
+
evaluator compares times within one request's evidence, recovery latency compares a
|
|
30
|
+
fault's time with its evidence's deliveries and attestations, the viewer orders events
|
|
31
|
+
by time, and the log's own checks ask only that a message is delivered no earlier than
|
|
32
|
+
it is sent, and that a copy repeats its request's time.
|
|
33
|
+
- A message whose fate nobody logged, because its receiver never took it, is dropped when
|
|
34
|
+
the run ends (`fivefold.harness.split.lifecycle`).
|
|
35
|
+
- A message from the agent that holds a NUL character is refused before it is logged
|
|
36
|
+
(`fivefold.harness.log.holds_nul`): PostgreSQL's JSONB cannot store one. The simulator
|
|
37
|
+
answers 422, as for a malformed body, and the gateway answers invalid params.
|
|
38
|
+
|
|
39
|
+
What a fault does where it fires (0006):
|
|
40
|
+
|
|
41
|
+
- DROP: the message never reaches its receiver. A service that drops a request it
|
|
42
|
+
received holds the connection open until the caller gives up, so the caller sees no
|
|
43
|
+
answer, as when a request is lost. The gateway, dropping an answer, waits until its
|
|
44
|
+
call's timeout ends, as when an answer is lost.
|
|
45
|
+
- DUPLICATE (F4): the service handles the request a second time, `copy_delay_ms` after
|
|
46
|
+
the first delivery, as a copy that names the request in `copy_of`. The copy's answer is
|
|
47
|
+
dropped with reason `answer_to_copy`.
|
|
48
|
+
- REVOKE and CHANGE_QUOTE (F6, F7): applied through the simulator's control interface
|
|
49
|
+
before the message goes on, by the simulator's service, which owns that interface.
|
|
50
|
+
|
|
51
|
+
Each fault's record names the message it acted on and that message's intent, which
|
|
52
|
+
recovery latency starts from (0014). The log stamps each delivery with the database's
|
|
53
|
+
wall-clock time (`delivered_wall_at`), as it stamps each message it writes (`wall_at`).
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
from __future__ import annotations
|
|
57
|
+
|
|
58
|
+
import json
|
|
59
|
+
from collections.abc import Callable, Mapping
|
|
60
|
+
from dataclasses import dataclass, replace
|
|
61
|
+
|
|
62
|
+
import psycopg
|
|
63
|
+
from psycopg.rows import TupleRow
|
|
64
|
+
|
|
65
|
+
from fivefold.domain.api import ApiName
|
|
66
|
+
from fivefold.faults.catalog import Action
|
|
67
|
+
from fivefold.faults.plan import FaultInjector, Fired
|
|
68
|
+
from fivefold.harness.log import (
|
|
69
|
+
DropReason,
|
|
70
|
+
FaultRecord,
|
|
71
|
+
HarnessLog,
|
|
72
|
+
JsonValue,
|
|
73
|
+
Kind,
|
|
74
|
+
MessageRecord,
|
|
75
|
+
)
|
|
76
|
+
from fivefold.harness.scheduler import Scheduler
|
|
77
|
+
from fivefold.harness.transport import Envelope, Hop
|
|
78
|
+
|
|
79
|
+
SEQ_HEADER = "fivefold-seq"
|
|
80
|
+
"""The header that carries a message's seq in the harness log between the services."""
|
|
81
|
+
|
|
82
|
+
WRITES = frozenset({str(ApiName.SUBMIT_PURCHASE_ORDER), str(ApiName.RESERVE_INVENTORY)})
|
|
83
|
+
|
|
84
|
+
Effect = Callable[[Fired, Envelope], None]
|
|
85
|
+
"""Applies a fault that changes the simulator's state (F6, F7) for the message it hit."""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def answer_class(api: str | None, body: object) -> str | None:
|
|
89
|
+
"""The answer class the log records for a simulator answer: a write's `answer`
|
|
90
|
+
(`committed`, `rejected` or `in_progress`), and None for every other call, as the
|
|
91
|
+
inproc endpoint records it (`fivefold.sim.endpoint`)."""
|
|
92
|
+
if api in WRITES and isinstance(body, Mapping):
|
|
93
|
+
found = body.get("answer")
|
|
94
|
+
return None if found is None else str(found)
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class Outcome:
|
|
100
|
+
"""What the faults that fired at a message do to it."""
|
|
101
|
+
|
|
102
|
+
dropped: bool
|
|
103
|
+
copies: tuple[Fired, ...] = ()
|
|
104
|
+
"""F4 entries that fired: each delivers the message once more."""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class Hops:
|
|
108
|
+
"""One service's records of its traffic, and the faults at the points it owns."""
|
|
109
|
+
|
|
110
|
+
def __init__(
|
|
111
|
+
self,
|
|
112
|
+
scheduler: Scheduler,
|
|
113
|
+
log: HarnessLog,
|
|
114
|
+
faults: FaultInjector,
|
|
115
|
+
*,
|
|
116
|
+
effect: Effect | None = None,
|
|
117
|
+
) -> None:
|
|
118
|
+
self._scheduler = scheduler
|
|
119
|
+
self._log = log
|
|
120
|
+
self._faults = faults
|
|
121
|
+
self._effect = effect
|
|
122
|
+
|
|
123
|
+
def sent(
|
|
124
|
+
self,
|
|
125
|
+
*,
|
|
126
|
+
hop: Hop,
|
|
127
|
+
kind: Kind,
|
|
128
|
+
sender: str,
|
|
129
|
+
receiver: str,
|
|
130
|
+
api: str | None,
|
|
131
|
+
body: JsonValue,
|
|
132
|
+
intent_id: str | None = None,
|
|
133
|
+
idempotency_key: str | None = None,
|
|
134
|
+
correlation: int | None = None,
|
|
135
|
+
answer_class: str | None = None,
|
|
136
|
+
sent_vt: int | None = None,
|
|
137
|
+
) -> Envelope:
|
|
138
|
+
"""Log a message as sent, with no fate yet; return it as its receiver gets it.
|
|
139
|
+
|
|
140
|
+
It is sent now, unless `sent_vt` gives the earlier time at which its sender
|
|
141
|
+
recorded the send: the gateway's write attempt (`SimulatorLink`).
|
|
142
|
+
"""
|
|
143
|
+
kept = json.loads(json.dumps(body, sort_keys=True))
|
|
144
|
+
seq = self._log.next_seq()
|
|
145
|
+
now = self._scheduler.now
|
|
146
|
+
if sent_vt is None:
|
|
147
|
+
sent_vt = now
|
|
148
|
+
elif sent_vt > now:
|
|
149
|
+
raise ValueError(f"a message logged at {now} cannot be sent later, at {sent_vt}")
|
|
150
|
+
envelope = Envelope(
|
|
151
|
+
seq=seq,
|
|
152
|
+
hop=hop,
|
|
153
|
+
kind=kind,
|
|
154
|
+
sender=sender,
|
|
155
|
+
receiver=receiver,
|
|
156
|
+
api=api,
|
|
157
|
+
correlation=correlation,
|
|
158
|
+
intent_id=intent_id,
|
|
159
|
+
idempotency_key=idempotency_key,
|
|
160
|
+
answer_class=answer_class,
|
|
161
|
+
body=kept,
|
|
162
|
+
sent_vt=sent_vt,
|
|
163
|
+
)
|
|
164
|
+
self._log.message_sent(_record(envelope))
|
|
165
|
+
return envelope
|
|
166
|
+
|
|
167
|
+
def delivered(self, envelope: Envelope) -> None:
|
|
168
|
+
"""Log that `envelope` reached its receiver now."""
|
|
169
|
+
self._log.message_fate(
|
|
170
|
+
envelope.seq, "delivered", vt=self._scheduler.now, delivered_seq=self._log.next_seq()
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
def dropped(self, envelope: Envelope, reason: DropReason) -> None:
|
|
174
|
+
"""Log that `envelope` never reached its receiver, and why."""
|
|
175
|
+
self._log.message_fate(envelope.seq, "dropped", vt=self._scheduler.now, reason=reason)
|
|
176
|
+
|
|
177
|
+
def fire(self, point: str, envelope: Envelope) -> Outcome:
|
|
178
|
+
"""Reach `point` with `envelope`; record and apply the faults that fire there.
|
|
179
|
+
|
|
180
|
+
A plan entry matches on the call (as `api` or `tool`), the answer class and the
|
|
181
|
+
intent, as the inproc transport's points do. Faults that change the simulator
|
|
182
|
+
apply here, before the message goes on.
|
|
183
|
+
"""
|
|
184
|
+
fields = {
|
|
185
|
+
"api": envelope.api,
|
|
186
|
+
"tool": envelope.api,
|
|
187
|
+
"answer": envelope.answer_class,
|
|
188
|
+
"intent_id": envelope.intent_id,
|
|
189
|
+
}
|
|
190
|
+
dropped = False
|
|
191
|
+
copies: list[Fired] = []
|
|
192
|
+
for fired in self._faults.at(point, fields):
|
|
193
|
+
action = fired.planned.kind.action
|
|
194
|
+
if action is Action.DUPLICATE:
|
|
195
|
+
copies.append(fired)
|
|
196
|
+
continue
|
|
197
|
+
self._record(fired, point, envelope)
|
|
198
|
+
if action is Action.DROP:
|
|
199
|
+
dropped = True
|
|
200
|
+
elif self._effect is not None:
|
|
201
|
+
self._effect(fired, envelope)
|
|
202
|
+
else:
|
|
203
|
+
raise RuntimeError(f"{fired.fault} fired at {point} with nothing to apply it")
|
|
204
|
+
return Outcome(dropped, tuple(copies))
|
|
205
|
+
|
|
206
|
+
def copy(self, fired: Fired, point: str, original: Envelope) -> Envelope:
|
|
207
|
+
"""Log F4's copy of the request `original`, and the fault that made it (0006).
|
|
208
|
+
|
|
209
|
+
The copy repeats the request field for field, with its own seq and `copy_of`
|
|
210
|
+
naming the request. It is due `copy_delay_ms` after the request's delivery.
|
|
211
|
+
"""
|
|
212
|
+
fault_seq = self._log.next_seq()
|
|
213
|
+
copy = replace(original, seq=self._log.next_seq(), copy_of=original.seq)
|
|
214
|
+
self._log.fault_fired(
|
|
215
|
+
FaultRecord(
|
|
216
|
+
seq=fault_seq,
|
|
217
|
+
fault=str(fired.fault),
|
|
218
|
+
variant=fired.planned.variant,
|
|
219
|
+
point=point,
|
|
220
|
+
occurrence=fired.occurrence,
|
|
221
|
+
action=str(Action.DUPLICATE),
|
|
222
|
+
vt=self._scheduler.now,
|
|
223
|
+
target_seq=original.seq,
|
|
224
|
+
params={**fired.planned.params, "copy_seq": copy.seq},
|
|
225
|
+
intent_id=original.intent_id,
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
self._log.message_sent(_record(copy))
|
|
229
|
+
return copy
|
|
230
|
+
|
|
231
|
+
def _record(self, fired: Fired, point: str, envelope: Envelope) -> None:
|
|
232
|
+
self._log.fault_fired(
|
|
233
|
+
FaultRecord(
|
|
234
|
+
seq=self._log.next_seq(),
|
|
235
|
+
fault=str(fired.fault),
|
|
236
|
+
variant=fired.planned.variant,
|
|
237
|
+
point=point,
|
|
238
|
+
occurrence=fired.occurrence,
|
|
239
|
+
action=str(fired.planned.kind.action),
|
|
240
|
+
vt=self._scheduler.now,
|
|
241
|
+
target_seq=envelope.seq,
|
|
242
|
+
params=dict(fired.planned.params),
|
|
243
|
+
intent_id=envelope.intent_id,
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _record(envelope: Envelope) -> MessageRecord:
|
|
249
|
+
return MessageRecord(
|
|
250
|
+
seq=envelope.seq,
|
|
251
|
+
hop=envelope.hop,
|
|
252
|
+
kind=envelope.kind,
|
|
253
|
+
sender=envelope.sender,
|
|
254
|
+
receiver=envelope.receiver,
|
|
255
|
+
api=envelope.api,
|
|
256
|
+
correlation=envelope.correlation,
|
|
257
|
+
intent_id=envelope.intent_id,
|
|
258
|
+
idempotency_key=envelope.idempotency_key,
|
|
259
|
+
answer_class=envelope.answer_class,
|
|
260
|
+
body=envelope.body,
|
|
261
|
+
sent_vt=envelope.sent_vt,
|
|
262
|
+
copy_of=envelope.copy_of,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def logged_message(conn: psycopg.Connection[TupleRow], seq: int) -> tuple[Envelope, bool] | None:
|
|
267
|
+
"""The message logged under `seq`, as an envelope, and whether it has a fate yet; None
|
|
268
|
+
if there is none. Reads the harness log as `ff_harness`."""
|
|
269
|
+
row = conn.execute(
|
|
270
|
+
"SELECT hop, kind, sender, receiver, api, correlation, intent_id, idempotency_key,"
|
|
271
|
+
" answer_class, body, sent_vt, copy_of, fate FROM harness.message_log WHERE seq = %s",
|
|
272
|
+
(seq,),
|
|
273
|
+
).fetchone()
|
|
274
|
+
if row is None:
|
|
275
|
+
return None
|
|
276
|
+
envelope = Envelope(
|
|
277
|
+
seq=seq,
|
|
278
|
+
hop=row[0],
|
|
279
|
+
kind=row[1],
|
|
280
|
+
sender=str(row[2]),
|
|
281
|
+
receiver=str(row[3]),
|
|
282
|
+
api=None if row[4] is None else str(row[4]),
|
|
283
|
+
correlation=None if row[5] is None else int(row[5]),
|
|
284
|
+
intent_id=None if row[6] is None else str(row[6]),
|
|
285
|
+
idempotency_key=None if row[7] is None else str(row[7]),
|
|
286
|
+
answer_class=None if row[8] is None else str(row[8]),
|
|
287
|
+
body=row[9],
|
|
288
|
+
sent_vt=int(row[10]),
|
|
289
|
+
copy_of=None if row[11] is None else int(row[11]),
|
|
290
|
+
)
|
|
291
|
+
return envelope, row[12] is not None
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""A split run's lifecycle: start it, end it, score it (0004, 0008, 0009).
|
|
2
|
+
|
|
3
|
+
`start_run` makes the run, before its services start:
|
|
4
|
+
|
|
5
|
+
1. It loads the scenario and checks the fault plan: the scenario's own, or one given. The
|
|
6
|
+
plan must be a fixed list, not a random plan, must fit the configuration and the run's
|
|
7
|
+
timing (`fivefold.faults.plan`), and the split topology must be able to run it
|
|
8
|
+
(`fivefold.harness.split.faults`). Seeded parameters are drawn from the run's seed, as
|
|
9
|
+
in the inproc topology.
|
|
10
|
+
2. It makes the run's database from the migrated template database (0009), first
|
|
11
|
+
migrating the template, or making it if it does not exist yet, as every run database
|
|
12
|
+
is made (`fivefold.harness.migrations.create_run_database`). The template is
|
|
13
|
+
FIVEFOLD_TEMPLATE_DB, or `fivefold_template`, unless the start names another. A
|
|
14
|
+
database that already has the run's name is refused; with replace it is dropped
|
|
15
|
+
first, but only if it is this run's: its `harness.run` holds this run, or no run yet,
|
|
16
|
+
as a copy of the template whose start failed before recording its run. Run ids that
|
|
17
|
+
differ only in case or punctuation, such as those of scenarios `sample-a` and
|
|
18
|
+
`sample_a`, get one default name, so replace never drops a database that holds
|
|
19
|
+
another run, or one that is no run database. The check and the drop happen under the
|
|
20
|
+
database lock.
|
|
21
|
+
3. It records the run in `harness.run`, with topology `split`, model adapter `external`
|
|
22
|
+
and the model the user names, and in the run registry, for the viewer (0012). It sets
|
|
23
|
+
up the simulator's run with its credentials: the gateway in G1 and G2, the agent
|
|
24
|
+
worker in B0 (0008).
|
|
25
|
+
4. It makes the run's secrets, and writes the run's directory (`rundir`).
|
|
26
|
+
|
|
27
|
+
`end_run` ends the run once the user's agent is done. It tells both services that the
|
|
28
|
+
agents' phase is over (G2 then closes the run, 0003), waits the drain window in wall
|
|
29
|
+
time, `2 x (D + max_hop_latency) + L` (0009), and stops the gateway, then the simulator,
|
|
30
|
+
which records the stop time. Last, it drops every message whose fate no service logged:
|
|
31
|
+
nothing delivered it. A request is dropped as `receiver_down`, an answer as
|
|
32
|
+
`caller_timed_out`: its caller stopped waiting before it arrived.
|
|
33
|
+
|
|
34
|
+
`score_run` scores the stopped run with the evaluator, which works on any run's database.
|
|
35
|
+
|
|
36
|
+
Checked in httpx 0.28.1: `AsyncClient(timeout=)` and `post(url, headers=)`, and
|
|
37
|
+
`Response.raise_for_status()`, which raises `HTTPStatusError` for a 4xx or 5xx status.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import hashlib
|
|
43
|
+
import re
|
|
44
|
+
from collections.abc import Mapping, Sequence
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any
|
|
47
|
+
|
|
48
|
+
import httpx
|
|
49
|
+
import psycopg
|
|
50
|
+
from psycopg.conninfo import make_conninfo
|
|
51
|
+
|
|
52
|
+
from fivefold.domain.contracts import Contract
|
|
53
|
+
from fivefold.domain.scenario import Scenario, load_scenario, scenario_digest
|
|
54
|
+
from fivefold.evaluator import RunResult as Evaluation
|
|
55
|
+
from fivefold.evaluator import evaluate_run
|
|
56
|
+
from fivefold.faults.plan import (
|
|
57
|
+
check_configuration,
|
|
58
|
+
check_run,
|
|
59
|
+
parse_plan,
|
|
60
|
+
seed_parameters,
|
|
61
|
+
)
|
|
62
|
+
from fivefold.harness.clock import HostClock
|
|
63
|
+
from fivefold.harness.log import RunRecord, SharedPgHarnessLog
|
|
64
|
+
from fivefold.harness.migrations import (
|
|
65
|
+
Connection,
|
|
66
|
+
DatabaseExists,
|
|
67
|
+
bootstrap_server,
|
|
68
|
+
check_database_name,
|
|
69
|
+
create_run_database,
|
|
70
|
+
drop_database_locked,
|
|
71
|
+
template_database,
|
|
72
|
+
)
|
|
73
|
+
from fivefold.harness.registry import RegistryEntry, register
|
|
74
|
+
from fivefold.harness.rng import Generators
|
|
75
|
+
from fivefold.harness.run import drain_window_ms
|
|
76
|
+
from fivefold.harness.scheduler import HostScheduler
|
|
77
|
+
from fivefold.harness.split.credentials import AGENT, GATEWAY, HARNESS, Callers, issue
|
|
78
|
+
from fivefold.harness.split.faults import SplitRefused, check_split
|
|
79
|
+
from fivefold.harness.split.rundir import (
|
|
80
|
+
RUN_FILE,
|
|
81
|
+
Config,
|
|
82
|
+
RunDirError,
|
|
83
|
+
SplitRun,
|
|
84
|
+
clear_run,
|
|
85
|
+
read_run,
|
|
86
|
+
write_run,
|
|
87
|
+
)
|
|
88
|
+
from fivefold.paths import DB_DIR
|
|
89
|
+
from fivefold.sim import SimConfig, setup_run
|
|
90
|
+
|
|
91
|
+
DEFAULT_MODEL = "unknown"
|
|
92
|
+
"""The model a split run records when the user names none."""
|
|
93
|
+
|
|
94
|
+
CONTROL_TIMEOUT_S = 30.0
|
|
95
|
+
"""How long `end` waits for a service to answer a control request, in wall-clock seconds."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
MAX_NAME_BYTES = 63
|
|
99
|
+
"""PostgreSQL keeps the first 63 bytes of a name (NAMEDATALEN - 1)."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def default_database(run_id: str) -> str:
|
|
103
|
+
"""A run database's name made from its run id: `fivefold_split_` and the id, in
|
|
104
|
+
lowercase letters, digits and underscores.
|
|
105
|
+
|
|
106
|
+
A name longer than 63 bytes keeps its start, then an underscore and the first 8 hex
|
|
107
|
+
digits of the run id's SHA-256, as the tests' database names do (`tests/conftest.py`).
|
|
108
|
+
Cut short without the hash, the names of runs that differ only at the end, such as two
|
|
109
|
+
seeds of a scenario with a long id, were one, and `start` with replace dropped the
|
|
110
|
+
other run's database.
|
|
111
|
+
"""
|
|
112
|
+
slug = re.sub(r"[^a-z0-9]+", "_", run_id.lower()).strip("_")
|
|
113
|
+
name = f"fivefold_split_{slug}"
|
|
114
|
+
if len(name.encode()) <= MAX_NAME_BYTES:
|
|
115
|
+
return name
|
|
116
|
+
digest = hashlib.sha256(run_id.encode()).hexdigest()[:8]
|
|
117
|
+
head = name[: MAX_NAME_BYTES - len(digest) - 1].rstrip("_")
|
|
118
|
+
return f"{head}_{digest}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def run_id_of(scenario: Scenario, config: Config, contract: Contract, seed: int) -> str:
|
|
122
|
+
"""A run's id, as the inproc topology makes it: the run's choices."""
|
|
123
|
+
return f"{scenario.id}.{config}.{contract}.{seed}"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def start_run(
|
|
127
|
+
*,
|
|
128
|
+
admin_dsn: str,
|
|
129
|
+
scenario_path: Path,
|
|
130
|
+
config: Config,
|
|
131
|
+
contract: Contract,
|
|
132
|
+
seed: int,
|
|
133
|
+
run_dir: Path,
|
|
134
|
+
plan: Sequence[Mapping[str, Any]] | None = None,
|
|
135
|
+
model: str = DEFAULT_MODEL,
|
|
136
|
+
database: str | None = None,
|
|
137
|
+
template: str | None = None,
|
|
138
|
+
replace: bool = False,
|
|
139
|
+
) -> SplitRun:
|
|
140
|
+
"""Make a split run and its directory; see the module. `plan` defaults to the
|
|
141
|
+
scenario's fault list, and `template` to `template_database()`. With `replace`, a run
|
|
142
|
+
already in `run_dir`, and a database of the same name that is this run's, are
|
|
143
|
+
replaced; without it they are refused."""
|
|
144
|
+
scenario_text = scenario_path.read_text(encoding="utf-8")
|
|
145
|
+
scenario = load_scenario(scenario_text)
|
|
146
|
+
entries = list(plan) if plan is not None else _plan_of(scenario)
|
|
147
|
+
planned = parse_plan(entries)
|
|
148
|
+
check_configuration(planned, gateway=config != "B0")
|
|
149
|
+
check_run(planned, params=scenario.params, contract=contract)
|
|
150
|
+
check_split(planned, scenario)
|
|
151
|
+
planned = seed_parameters(planned, Generators(seed).get("faults"))
|
|
152
|
+
run_id = run_id_of(scenario, config, contract, seed)
|
|
153
|
+
name = database if database is not None else default_database(run_id)
|
|
154
|
+
template = template_database() if template is None else template
|
|
155
|
+
check_database_name(name)
|
|
156
|
+
check_database_name(template)
|
|
157
|
+
if name == template:
|
|
158
|
+
raise RunDirError(f"database {name} is the template run databases are made from")
|
|
159
|
+
held_here = (run_dir / RUN_FILE).exists()
|
|
160
|
+
if held_here and not replace:
|
|
161
|
+
held = read_run(run_dir)
|
|
162
|
+
raise RunDirError(
|
|
163
|
+
f"{run_dir} holds run {held.run_id}: end and score it, then start again with"
|
|
164
|
+
" replace, or choose another run directory"
|
|
165
|
+
)
|
|
166
|
+
dsn = _run_database(admin_dsn, name, template, run_id=run_id, replace=replace)
|
|
167
|
+
if held_here:
|
|
168
|
+
clear_run(run_dir)
|
|
169
|
+
run = SplitRun(run_dir, run_id, name, config, contract, seed)
|
|
170
|
+
with psycopg.connect(make_conninfo(dsn, user="ff_harness"), autocommit=True) as conn:
|
|
171
|
+
SharedPgHarnessLog(conn).run_recorded(
|
|
172
|
+
RunRecord(
|
|
173
|
+
run_id=run_id,
|
|
174
|
+
scenario_id=scenario.id,
|
|
175
|
+
scenario_digest=scenario_digest(scenario),
|
|
176
|
+
config=config,
|
|
177
|
+
contract=str(contract), # type: ignore[arg-type]
|
|
178
|
+
seed=seed,
|
|
179
|
+
model_adapter="external",
|
|
180
|
+
model=model,
|
|
181
|
+
params={
|
|
182
|
+
"params": scenario.params.model_dump(mode="json"),
|
|
183
|
+
"plan": [entry.model_dump(mode="json") for entry in planned],
|
|
184
|
+
"workers": 1,
|
|
185
|
+
"agent_workers": 1,
|
|
186
|
+
},
|
|
187
|
+
topology="split",
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
register(
|
|
191
|
+
dsn,
|
|
192
|
+
RegistryEntry(
|
|
193
|
+
database=name,
|
|
194
|
+
run_id=run_id,
|
|
195
|
+
scenario_id=scenario.id,
|
|
196
|
+
config=config,
|
|
197
|
+
contract=str(contract),
|
|
198
|
+
seed=seed,
|
|
199
|
+
),
|
|
200
|
+
)
|
|
201
|
+
credentials = (GATEWAY,) if config != "B0" else (AGENT,)
|
|
202
|
+
simulator = SimConfig.preset(
|
|
203
|
+
contract, credentials=credentials, params=scenario.params.contract_params()
|
|
204
|
+
)
|
|
205
|
+
setup_run(make_conninfo(dsn, user="ff_sim"), scenario, simulator, vt=0)
|
|
206
|
+
secrets = issue()
|
|
207
|
+
write_run(run, scenario_text=scenario_text, secrets=secrets, callers=Callers.of(secrets))
|
|
208
|
+
return run
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _plan_of(scenario: Scenario) -> list[Mapping[str, Any]]:
|
|
212
|
+
"""The scenario's fault list. A random plan is refused (`fivefold.harness.split.faults`)."""
|
|
213
|
+
if isinstance(scenario.faults, dict):
|
|
214
|
+
raise SplitRefused(
|
|
215
|
+
"the split topology runs a fixed fault plan only: give the scenario a list of"
|
|
216
|
+
" faults, or a plan file; a random plan runs in the inproc topology (0006)"
|
|
217
|
+
)
|
|
218
|
+
return list(scenario.faults)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _run_database(admin_dsn: str, name: str, template: str, *, run_id: str, replace: bool) -> str:
|
|
222
|
+
"""Make the run's database from the migrated template; return its admin DSN. The
|
|
223
|
+
component roles and the run registry are made first, if they are missing.
|
|
224
|
+
|
|
225
|
+
The template is made or migrated forward, the run database is replaced, when asked
|
|
226
|
+
and when it is run `run_id`'s (`_drop_this_runs`), and the template is copied, in one
|
|
227
|
+
hold of the database lock that every create and drop takes
|
|
228
|
+
(`fivefold.harness.migrations.create_run_database`), so a study's cleanup never sees
|
|
229
|
+
the name half changed, and no other start makes it in between.
|
|
230
|
+
"""
|
|
231
|
+
bootstrap_server(admin_dsn, DB_DIR)
|
|
232
|
+
|
|
233
|
+
def drop_this_runs(conn: Connection) -> None:
|
|
234
|
+
if replace:
|
|
235
|
+
_drop_this_runs(conn, admin_dsn, name, run_id)
|
|
236
|
+
|
|
237
|
+
try:
|
|
238
|
+
created = create_run_database(
|
|
239
|
+
admin_dsn, name, DB_DIR / "migrations", template=template, before_copy=drop_this_runs
|
|
240
|
+
)
|
|
241
|
+
except DatabaseExists:
|
|
242
|
+
raise RunDirError(f"database {name} already exists: replace it, or name another") from None
|
|
243
|
+
return created.dsn
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _drop_this_runs(conn: Connection, admin_dsn: str, name: str, run_id: str) -> None:
|
|
247
|
+
"""Drop the database `name` for a start with replace, if it is run `run_id`'s: its
|
|
248
|
+
`harness.run` holds that run, or no run yet. `conn` holds the database lock.
|
|
249
|
+
|
|
250
|
+
Raises `RunDirError`, and keeps the database, when it holds another run or is no run
|
|
251
|
+
database: it has no `harness.run`.
|
|
252
|
+
"""
|
|
253
|
+
row = conn.execute("SELECT oid FROM pg_database WHERE datname = %s", (name,)).fetchone()
|
|
254
|
+
if row is None:
|
|
255
|
+
return
|
|
256
|
+
with psycopg.connect(make_conninfo(admin_dsn, dbname=name), autocommit=True) as found:
|
|
257
|
+
table = found.execute("SELECT to_regclass('harness.run')").fetchone()
|
|
258
|
+
if table is None or table[0] is None:
|
|
259
|
+
raise RunDirError(
|
|
260
|
+
f"database {name} is not a run database, and replace drops only this run's:"
|
|
261
|
+
" name another"
|
|
262
|
+
)
|
|
263
|
+
held = sorted({str(run[0]) for run in found.execute("SELECT run_id FROM harness.run")})
|
|
264
|
+
others = [held_id for held_id in held if held_id != run_id]
|
|
265
|
+
if others:
|
|
266
|
+
raise RunDirError(
|
|
267
|
+
f"database {name} holds run {others[0]}, and replace drops only this run's:"
|
|
268
|
+
" name another"
|
|
269
|
+
)
|
|
270
|
+
drop_database_locked(conn, name, oid=int(row[0]))
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
async def end_run(
|
|
274
|
+
run: SplitRun, *, admin_dsn: str, simulator_url: str, gateway_url: str | None = None
|
|
275
|
+
) -> int:
|
|
276
|
+
"""End the run; see the module. Returns the virtual time it stopped at."""
|
|
277
|
+
if run.ended:
|
|
278
|
+
raise RunDirError(f"run {run.run_id} has already ended")
|
|
279
|
+
if run.has_gateway and gateway_url is None:
|
|
280
|
+
raise ValueError(f"run {run.run_id} is {run.config}: give the gateway's address")
|
|
281
|
+
secret = run.secret(HARNESS)
|
|
282
|
+
async with httpx.AsyncClient(timeout=CONTROL_TIMEOUT_S) as client:
|
|
283
|
+
if gateway_url is not None and run.has_gateway:
|
|
284
|
+
await _control(client, gateway_url, "drain", secret)
|
|
285
|
+
await _control(client, simulator_url, "drain", secret)
|
|
286
|
+
# The drain window passes in wall time: in the split topology the run's virtual
|
|
287
|
+
# time follows the host's clock (0009).
|
|
288
|
+
await HostScheduler(HostClock(HostClock.host_ms())).sleep(drain_window_ms(run.scenario()))
|
|
289
|
+
if gateway_url is not None and run.has_gateway:
|
|
290
|
+
await _control(client, gateway_url, "stop", secret)
|
|
291
|
+
stopped = await _control(client, simulator_url, "stop", secret)
|
|
292
|
+
stopped_vt = int(stopped["stopped_vt"])
|
|
293
|
+
drop_unlogged(run, admin_dsn)
|
|
294
|
+
run.mark_ended(stopped_vt)
|
|
295
|
+
return stopped_vt
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
async def _control(client: httpx.AsyncClient, url: str, action: str, secret: str) -> dict[str, Any]:
|
|
299
|
+
response = await client.post(
|
|
300
|
+
f"{url.rstrip('/')}/harness/{action}", headers={"authorization": f"Bearer {secret}"}
|
|
301
|
+
)
|
|
302
|
+
response.raise_for_status()
|
|
303
|
+
answer: dict[str, Any] = response.json()
|
|
304
|
+
return answer
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def drop_unlogged(run: SplitRun, admin_dsn: str) -> int:
|
|
308
|
+
"""Drop every message whose fate no service logged; return how many there were."""
|
|
309
|
+
with psycopg.connect(run.conninfo(admin_dsn, "ff_harness"), autocommit=True) as conn:
|
|
310
|
+
updated = conn.execute(
|
|
311
|
+
"UPDATE harness.message_log SET fate = 'dropped', drop_reason = CASE kind"
|
|
312
|
+
" WHEN 'answer' THEN 'caller_timed_out' ELSE 'receiver_down' END"
|
|
313
|
+
" WHERE fate IS NULL"
|
|
314
|
+
)
|
|
315
|
+
return updated.rowcount
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def score_run(run: SplitRun, *, admin_dsn: str) -> Evaluation:
|
|
319
|
+
"""Score the stopped run with the evaluator, as `ff_eval`, and store its result."""
|
|
320
|
+
if not run.ended:
|
|
321
|
+
raise RunDirError(f"run {run.run_id} has not ended: end it first")
|
|
322
|
+
return evaluate_run(run.conninfo(admin_dsn, "ff_eval"), run.scenario())
|