fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,291 @@
1
+ """The harness's records and faults of the split services' traffic (0004, 0006, 0009).
2
+
3
+ In the inproc topology the harness's transport carries every message, logs it with its
4
+ fate and fires the plan's faults on it. In the split topology messages travel over real
5
+ HTTP connections, so each service logs the messages it sees and fires the faults at the
6
+ points it owns (`fivefold.harness.split.faults`). The log is the same harness log, with
7
+ these readings of when a message is sent and delivered:
8
+
9
+ - A message from the user's agent, which the harness cannot watch, is logged when it
10
+ arrives: sent and delivered at once, at the same virtual time.
11
+ - An answer a service sends to the agent is logged as sent, then as delivered when the
12
+ service hands it to its HTTP server, before the server writes it: the harness cannot
13
+ see whether the agent read it. The simulator first asks whether the agent's connection
14
+ is still open, and logs the answer as dropped (`caller_timed_out`) if it is not. The
15
+ gateway's answers go through the MCP SDK, which writes them itself, so the gateway logs
16
+ each one as delivered when its handler returns, even if the agent has gone. In G1 and
17
+ G2 the evaluator reads no answer on the agent's hop as evidence (0004), so this changes
18
+ no verdict there.
19
+ - On the `gw_sim` hop the gateway logs each request as it sends it, and each answer as it
20
+ arrives; the simulator logs each request as it arrives, and each answer as it sends it.
21
+ A request carries the seq the gateway logged it under, and an answer the seq the
22
+ simulator logged it under, in the header `fivefold-seq`.
23
+ - A write request is sent at the time its attempt records, `gw.attempt.sent_vt` (0003),
24
+ the time the gateway's settle bounds start from (0005). The gateway records the
25
+ attempt in a transaction, then logs the request, and on the host's clock milliseconds
26
+ pass in between; the log takes the attempt's time, so the evaluator's bounds start from
27
+ the same number (0004). A message the simulator logs meanwhile can then have a lower
28
+ seq and a later `sent_vt`. Nothing reads the log as if virtual time grew with seq: the
29
+ evaluator compares times within one request's evidence, recovery latency compares a
30
+ fault's time with its evidence's deliveries and attestations, the viewer orders events
31
+ by time, and the log's own checks ask only that a message is delivered no earlier than
32
+ it is sent, and that a copy repeats its request's time.
33
+ - A message whose fate nobody logged, because its receiver never took it, is dropped when
34
+ the run ends (`fivefold.harness.split.lifecycle`).
35
+ - A message from the agent that holds a NUL character is refused before it is logged
36
+ (`fivefold.harness.log.holds_nul`): PostgreSQL's JSONB cannot store one. The simulator
37
+ answers 422, as for a malformed body, and the gateway answers invalid params.
38
+
39
+ What a fault does where it fires (0006):
40
+
41
+ - DROP: the message never reaches its receiver. A service that drops a request it
42
+ received holds the connection open until the caller gives up, so the caller sees no
43
+ answer, as when a request is lost. The gateway, dropping an answer, waits until its
44
+ call's timeout ends, as when an answer is lost.
45
+ - DUPLICATE (F4): the service handles the request a second time, `copy_delay_ms` after
46
+ the first delivery, as a copy that names the request in `copy_of`. The copy's answer is
47
+ dropped with reason `answer_to_copy`.
48
+ - REVOKE and CHANGE_QUOTE (F6, F7): applied through the simulator's control interface
49
+ before the message goes on, by the simulator's service, which owns that interface.
50
+
51
+ Each fault's record names the message it acted on and that message's intent, which
52
+ recovery latency starts from (0014). The log stamps each delivery with the database's
53
+ wall-clock time (`delivered_wall_at`), as it stamps each message it writes (`wall_at`).
54
+ """
55
+
56
+ from __future__ import annotations
57
+
58
+ import json
59
+ from collections.abc import Callable, Mapping
60
+ from dataclasses import dataclass, replace
61
+
62
+ import psycopg
63
+ from psycopg.rows import TupleRow
64
+
65
+ from fivefold.domain.api import ApiName
66
+ from fivefold.faults.catalog import Action
67
+ from fivefold.faults.plan import FaultInjector, Fired
68
+ from fivefold.harness.log import (
69
+ DropReason,
70
+ FaultRecord,
71
+ HarnessLog,
72
+ JsonValue,
73
+ Kind,
74
+ MessageRecord,
75
+ )
76
+ from fivefold.harness.scheduler import Scheduler
77
+ from fivefold.harness.transport import Envelope, Hop
78
+
79
+ SEQ_HEADER = "fivefold-seq"
80
+ """The header that carries a message's seq in the harness log between the services."""
81
+
82
+ WRITES = frozenset({str(ApiName.SUBMIT_PURCHASE_ORDER), str(ApiName.RESERVE_INVENTORY)})
83
+
84
+ Effect = Callable[[Fired, Envelope], None]
85
+ """Applies a fault that changes the simulator's state (F6, F7) for the message it hit."""
86
+
87
+
88
+ def answer_class(api: str | None, body: object) -> str | None:
89
+ """The answer class the log records for a simulator answer: a write's `answer`
90
+ (`committed`, `rejected` or `in_progress`), and None for every other call, as the
91
+ inproc endpoint records it (`fivefold.sim.endpoint`)."""
92
+ if api in WRITES and isinstance(body, Mapping):
93
+ found = body.get("answer")
94
+ return None if found is None else str(found)
95
+ return None
96
+
97
+
98
+ @dataclass(frozen=True)
99
+ class Outcome:
100
+ """What the faults that fired at a message do to it."""
101
+
102
+ dropped: bool
103
+ copies: tuple[Fired, ...] = ()
104
+ """F4 entries that fired: each delivers the message once more."""
105
+
106
+
107
+ class Hops:
108
+ """One service's records of its traffic, and the faults at the points it owns."""
109
+
110
+ def __init__(
111
+ self,
112
+ scheduler: Scheduler,
113
+ log: HarnessLog,
114
+ faults: FaultInjector,
115
+ *,
116
+ effect: Effect | None = None,
117
+ ) -> None:
118
+ self._scheduler = scheduler
119
+ self._log = log
120
+ self._faults = faults
121
+ self._effect = effect
122
+
123
+ def sent(
124
+ self,
125
+ *,
126
+ hop: Hop,
127
+ kind: Kind,
128
+ sender: str,
129
+ receiver: str,
130
+ api: str | None,
131
+ body: JsonValue,
132
+ intent_id: str | None = None,
133
+ idempotency_key: str | None = None,
134
+ correlation: int | None = None,
135
+ answer_class: str | None = None,
136
+ sent_vt: int | None = None,
137
+ ) -> Envelope:
138
+ """Log a message as sent, with no fate yet; return it as its receiver gets it.
139
+
140
+ It is sent now, unless `sent_vt` gives the earlier time at which its sender
141
+ recorded the send: the gateway's write attempt (`SimulatorLink`).
142
+ """
143
+ kept = json.loads(json.dumps(body, sort_keys=True))
144
+ seq = self._log.next_seq()
145
+ now = self._scheduler.now
146
+ if sent_vt is None:
147
+ sent_vt = now
148
+ elif sent_vt > now:
149
+ raise ValueError(f"a message logged at {now} cannot be sent later, at {sent_vt}")
150
+ envelope = Envelope(
151
+ seq=seq,
152
+ hop=hop,
153
+ kind=kind,
154
+ sender=sender,
155
+ receiver=receiver,
156
+ api=api,
157
+ correlation=correlation,
158
+ intent_id=intent_id,
159
+ idempotency_key=idempotency_key,
160
+ answer_class=answer_class,
161
+ body=kept,
162
+ sent_vt=sent_vt,
163
+ )
164
+ self._log.message_sent(_record(envelope))
165
+ return envelope
166
+
167
+ def delivered(self, envelope: Envelope) -> None:
168
+ """Log that `envelope` reached its receiver now."""
169
+ self._log.message_fate(
170
+ envelope.seq, "delivered", vt=self._scheduler.now, delivered_seq=self._log.next_seq()
171
+ )
172
+
173
+ def dropped(self, envelope: Envelope, reason: DropReason) -> None:
174
+ """Log that `envelope` never reached its receiver, and why."""
175
+ self._log.message_fate(envelope.seq, "dropped", vt=self._scheduler.now, reason=reason)
176
+
177
+ def fire(self, point: str, envelope: Envelope) -> Outcome:
178
+ """Reach `point` with `envelope`; record and apply the faults that fire there.
179
+
180
+ A plan entry matches on the call (as `api` or `tool`), the answer class and the
181
+ intent, as the inproc transport's points do. Faults that change the simulator
182
+ apply here, before the message goes on.
183
+ """
184
+ fields = {
185
+ "api": envelope.api,
186
+ "tool": envelope.api,
187
+ "answer": envelope.answer_class,
188
+ "intent_id": envelope.intent_id,
189
+ }
190
+ dropped = False
191
+ copies: list[Fired] = []
192
+ for fired in self._faults.at(point, fields):
193
+ action = fired.planned.kind.action
194
+ if action is Action.DUPLICATE:
195
+ copies.append(fired)
196
+ continue
197
+ self._record(fired, point, envelope)
198
+ if action is Action.DROP:
199
+ dropped = True
200
+ elif self._effect is not None:
201
+ self._effect(fired, envelope)
202
+ else:
203
+ raise RuntimeError(f"{fired.fault} fired at {point} with nothing to apply it")
204
+ return Outcome(dropped, tuple(copies))
205
+
206
+ def copy(self, fired: Fired, point: str, original: Envelope) -> Envelope:
207
+ """Log F4's copy of the request `original`, and the fault that made it (0006).
208
+
209
+ The copy repeats the request field for field, with its own seq and `copy_of`
210
+ naming the request. It is due `copy_delay_ms` after the request's delivery.
211
+ """
212
+ fault_seq = self._log.next_seq()
213
+ copy = replace(original, seq=self._log.next_seq(), copy_of=original.seq)
214
+ self._log.fault_fired(
215
+ FaultRecord(
216
+ seq=fault_seq,
217
+ fault=str(fired.fault),
218
+ variant=fired.planned.variant,
219
+ point=point,
220
+ occurrence=fired.occurrence,
221
+ action=str(Action.DUPLICATE),
222
+ vt=self._scheduler.now,
223
+ target_seq=original.seq,
224
+ params={**fired.planned.params, "copy_seq": copy.seq},
225
+ intent_id=original.intent_id,
226
+ )
227
+ )
228
+ self._log.message_sent(_record(copy))
229
+ return copy
230
+
231
+ def _record(self, fired: Fired, point: str, envelope: Envelope) -> None:
232
+ self._log.fault_fired(
233
+ FaultRecord(
234
+ seq=self._log.next_seq(),
235
+ fault=str(fired.fault),
236
+ variant=fired.planned.variant,
237
+ point=point,
238
+ occurrence=fired.occurrence,
239
+ action=str(fired.planned.kind.action),
240
+ vt=self._scheduler.now,
241
+ target_seq=envelope.seq,
242
+ params=dict(fired.planned.params),
243
+ intent_id=envelope.intent_id,
244
+ )
245
+ )
246
+
247
+
248
+ def _record(envelope: Envelope) -> MessageRecord:
249
+ return MessageRecord(
250
+ seq=envelope.seq,
251
+ hop=envelope.hop,
252
+ kind=envelope.kind,
253
+ sender=envelope.sender,
254
+ receiver=envelope.receiver,
255
+ api=envelope.api,
256
+ correlation=envelope.correlation,
257
+ intent_id=envelope.intent_id,
258
+ idempotency_key=envelope.idempotency_key,
259
+ answer_class=envelope.answer_class,
260
+ body=envelope.body,
261
+ sent_vt=envelope.sent_vt,
262
+ copy_of=envelope.copy_of,
263
+ )
264
+
265
+
266
+ def logged_message(conn: psycopg.Connection[TupleRow], seq: int) -> tuple[Envelope, bool] | None:
267
+ """The message logged under `seq`, as an envelope, and whether it has a fate yet; None
268
+ if there is none. Reads the harness log as `ff_harness`."""
269
+ row = conn.execute(
270
+ "SELECT hop, kind, sender, receiver, api, correlation, intent_id, idempotency_key,"
271
+ " answer_class, body, sent_vt, copy_of, fate FROM harness.message_log WHERE seq = %s",
272
+ (seq,),
273
+ ).fetchone()
274
+ if row is None:
275
+ return None
276
+ envelope = Envelope(
277
+ seq=seq,
278
+ hop=row[0],
279
+ kind=row[1],
280
+ sender=str(row[2]),
281
+ receiver=str(row[3]),
282
+ api=None if row[4] is None else str(row[4]),
283
+ correlation=None if row[5] is None else int(row[5]),
284
+ intent_id=None if row[6] is None else str(row[6]),
285
+ idempotency_key=None if row[7] is None else str(row[7]),
286
+ answer_class=None if row[8] is None else str(row[8]),
287
+ body=row[9],
288
+ sent_vt=int(row[10]),
289
+ copy_of=None if row[11] is None else int(row[11]),
290
+ )
291
+ return envelope, row[12] is not None
@@ -0,0 +1,322 @@
1
+ """A split run's lifecycle: start it, end it, score it (0004, 0008, 0009).
2
+
3
+ `start_run` makes the run, before its services start:
4
+
5
+ 1. It loads the scenario and checks the fault plan: the scenario's own, or one given. The
6
+ plan must be a fixed list, not a random plan, must fit the configuration and the run's
7
+ timing (`fivefold.faults.plan`), and the split topology must be able to run it
8
+ (`fivefold.harness.split.faults`). Seeded parameters are drawn from the run's seed, as
9
+ in the inproc topology.
10
+ 2. It makes the run's database from the migrated template database (0009), first
11
+ migrating the template, or making it if it does not exist yet, as every run database
12
+ is made (`fivefold.harness.migrations.create_run_database`). The template is
13
+ FIVEFOLD_TEMPLATE_DB, or `fivefold_template`, unless the start names another. A
14
+ database that already has the run's name is refused; with replace it is dropped
15
+ first, but only if it is this run's: its `harness.run` holds this run, or no run yet,
16
+ as a copy of the template whose start failed before recording its run. Run ids that
17
+ differ only in case or punctuation, such as those of scenarios `sample-a` and
18
+ `sample_a`, get one default name, so replace never drops a database that holds
19
+ another run, or one that is no run database. The check and the drop happen under the
20
+ database lock.
21
+ 3. It records the run in `harness.run`, with topology `split`, model adapter `external`
22
+ and the model the user names, and in the run registry, for the viewer (0012). It sets
23
+ up the simulator's run with its credentials: the gateway in G1 and G2, the agent
24
+ worker in B0 (0008).
25
+ 4. It makes the run's secrets, and writes the run's directory (`rundir`).
26
+
27
+ `end_run` ends the run once the user's agent is done. It tells both services that the
28
+ agents' phase is over (G2 then closes the run, 0003), waits the drain window in wall
29
+ time, `2 x (D + max_hop_latency) + L` (0009), and stops the gateway, then the simulator,
30
+ which records the stop time. Last, it drops every message whose fate no service logged:
31
+ nothing delivered it. A request is dropped as `receiver_down`, an answer as
32
+ `caller_timed_out`: its caller stopped waiting before it arrived.
33
+
34
+ `score_run` scores the stopped run with the evaluator, which works on any run's database.
35
+
36
+ Checked in httpx 0.28.1: `AsyncClient(timeout=)` and `post(url, headers=)`, and
37
+ `Response.raise_for_status()`, which raises `HTTPStatusError` for a 4xx or 5xx status.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import hashlib
43
+ import re
44
+ from collections.abc import Mapping, Sequence
45
+ from pathlib import Path
46
+ from typing import Any
47
+
48
+ import httpx
49
+ import psycopg
50
+ from psycopg.conninfo import make_conninfo
51
+
52
+ from fivefold.domain.contracts import Contract
53
+ from fivefold.domain.scenario import Scenario, load_scenario, scenario_digest
54
+ from fivefold.evaluator import RunResult as Evaluation
55
+ from fivefold.evaluator import evaluate_run
56
+ from fivefold.faults.plan import (
57
+ check_configuration,
58
+ check_run,
59
+ parse_plan,
60
+ seed_parameters,
61
+ )
62
+ from fivefold.harness.clock import HostClock
63
+ from fivefold.harness.log import RunRecord, SharedPgHarnessLog
64
+ from fivefold.harness.migrations import (
65
+ Connection,
66
+ DatabaseExists,
67
+ bootstrap_server,
68
+ check_database_name,
69
+ create_run_database,
70
+ drop_database_locked,
71
+ template_database,
72
+ )
73
+ from fivefold.harness.registry import RegistryEntry, register
74
+ from fivefold.harness.rng import Generators
75
+ from fivefold.harness.run import drain_window_ms
76
+ from fivefold.harness.scheduler import HostScheduler
77
+ from fivefold.harness.split.credentials import AGENT, GATEWAY, HARNESS, Callers, issue
78
+ from fivefold.harness.split.faults import SplitRefused, check_split
79
+ from fivefold.harness.split.rundir import (
80
+ RUN_FILE,
81
+ Config,
82
+ RunDirError,
83
+ SplitRun,
84
+ clear_run,
85
+ read_run,
86
+ write_run,
87
+ )
88
+ from fivefold.paths import DB_DIR
89
+ from fivefold.sim import SimConfig, setup_run
90
+
91
+ DEFAULT_MODEL = "unknown"
92
+ """The model a split run records when the user names none."""
93
+
94
+ CONTROL_TIMEOUT_S = 30.0
95
+ """How long `end` waits for a service to answer a control request, in wall-clock seconds."""
96
+
97
+
98
+ MAX_NAME_BYTES = 63
99
+ """PostgreSQL keeps the first 63 bytes of a name (NAMEDATALEN - 1)."""
100
+
101
+
102
+ def default_database(run_id: str) -> str:
103
+ """A run database's name made from its run id: `fivefold_split_` and the id, in
104
+ lowercase letters, digits and underscores.
105
+
106
+ A name longer than 63 bytes keeps its start, then an underscore and the first 8 hex
107
+ digits of the run id's SHA-256, as the tests' database names do (`tests/conftest.py`).
108
+ Cut short without the hash, the names of runs that differ only at the end, such as two
109
+ seeds of a scenario with a long id, were one, and `start` with replace dropped the
110
+ other run's database.
111
+ """
112
+ slug = re.sub(r"[^a-z0-9]+", "_", run_id.lower()).strip("_")
113
+ name = f"fivefold_split_{slug}"
114
+ if len(name.encode()) <= MAX_NAME_BYTES:
115
+ return name
116
+ digest = hashlib.sha256(run_id.encode()).hexdigest()[:8]
117
+ head = name[: MAX_NAME_BYTES - len(digest) - 1].rstrip("_")
118
+ return f"{head}_{digest}"
119
+
120
+
121
+ def run_id_of(scenario: Scenario, config: Config, contract: Contract, seed: int) -> str:
122
+ """A run's id, as the inproc topology makes it: the run's choices."""
123
+ return f"{scenario.id}.{config}.{contract}.{seed}"
124
+
125
+
126
+ def start_run(
127
+ *,
128
+ admin_dsn: str,
129
+ scenario_path: Path,
130
+ config: Config,
131
+ contract: Contract,
132
+ seed: int,
133
+ run_dir: Path,
134
+ plan: Sequence[Mapping[str, Any]] | None = None,
135
+ model: str = DEFAULT_MODEL,
136
+ database: str | None = None,
137
+ template: str | None = None,
138
+ replace: bool = False,
139
+ ) -> SplitRun:
140
+ """Make a split run and its directory; see the module. `plan` defaults to the
141
+ scenario's fault list, and `template` to `template_database()`. With `replace`, a run
142
+ already in `run_dir`, and a database of the same name that is this run's, are
143
+ replaced; without it they are refused."""
144
+ scenario_text = scenario_path.read_text(encoding="utf-8")
145
+ scenario = load_scenario(scenario_text)
146
+ entries = list(plan) if plan is not None else _plan_of(scenario)
147
+ planned = parse_plan(entries)
148
+ check_configuration(planned, gateway=config != "B0")
149
+ check_run(planned, params=scenario.params, contract=contract)
150
+ check_split(planned, scenario)
151
+ planned = seed_parameters(planned, Generators(seed).get("faults"))
152
+ run_id = run_id_of(scenario, config, contract, seed)
153
+ name = database if database is not None else default_database(run_id)
154
+ template = template_database() if template is None else template
155
+ check_database_name(name)
156
+ check_database_name(template)
157
+ if name == template:
158
+ raise RunDirError(f"database {name} is the template run databases are made from")
159
+ held_here = (run_dir / RUN_FILE).exists()
160
+ if held_here and not replace:
161
+ held = read_run(run_dir)
162
+ raise RunDirError(
163
+ f"{run_dir} holds run {held.run_id}: end and score it, then start again with"
164
+ " replace, or choose another run directory"
165
+ )
166
+ dsn = _run_database(admin_dsn, name, template, run_id=run_id, replace=replace)
167
+ if held_here:
168
+ clear_run(run_dir)
169
+ run = SplitRun(run_dir, run_id, name, config, contract, seed)
170
+ with psycopg.connect(make_conninfo(dsn, user="ff_harness"), autocommit=True) as conn:
171
+ SharedPgHarnessLog(conn).run_recorded(
172
+ RunRecord(
173
+ run_id=run_id,
174
+ scenario_id=scenario.id,
175
+ scenario_digest=scenario_digest(scenario),
176
+ config=config,
177
+ contract=str(contract), # type: ignore[arg-type]
178
+ seed=seed,
179
+ model_adapter="external",
180
+ model=model,
181
+ params={
182
+ "params": scenario.params.model_dump(mode="json"),
183
+ "plan": [entry.model_dump(mode="json") for entry in planned],
184
+ "workers": 1,
185
+ "agent_workers": 1,
186
+ },
187
+ topology="split",
188
+ )
189
+ )
190
+ register(
191
+ dsn,
192
+ RegistryEntry(
193
+ database=name,
194
+ run_id=run_id,
195
+ scenario_id=scenario.id,
196
+ config=config,
197
+ contract=str(contract),
198
+ seed=seed,
199
+ ),
200
+ )
201
+ credentials = (GATEWAY,) if config != "B0" else (AGENT,)
202
+ simulator = SimConfig.preset(
203
+ contract, credentials=credentials, params=scenario.params.contract_params()
204
+ )
205
+ setup_run(make_conninfo(dsn, user="ff_sim"), scenario, simulator, vt=0)
206
+ secrets = issue()
207
+ write_run(run, scenario_text=scenario_text, secrets=secrets, callers=Callers.of(secrets))
208
+ return run
209
+
210
+
211
+ def _plan_of(scenario: Scenario) -> list[Mapping[str, Any]]:
212
+ """The scenario's fault list. A random plan is refused (`fivefold.harness.split.faults`)."""
213
+ if isinstance(scenario.faults, dict):
214
+ raise SplitRefused(
215
+ "the split topology runs a fixed fault plan only: give the scenario a list of"
216
+ " faults, or a plan file; a random plan runs in the inproc topology (0006)"
217
+ )
218
+ return list(scenario.faults)
219
+
220
+
221
+ def _run_database(admin_dsn: str, name: str, template: str, *, run_id: str, replace: bool) -> str:
222
+ """Make the run's database from the migrated template; return its admin DSN. The
223
+ component roles and the run registry are made first, if they are missing.
224
+
225
+ The template is made or migrated forward, the run database is replaced, when asked
226
+ and when it is run `run_id`'s (`_drop_this_runs`), and the template is copied, in one
227
+ hold of the database lock that every create and drop takes
228
+ (`fivefold.harness.migrations.create_run_database`), so a study's cleanup never sees
229
+ the name half changed, and no other start makes it in between.
230
+ """
231
+ bootstrap_server(admin_dsn, DB_DIR)
232
+
233
+ def drop_this_runs(conn: Connection) -> None:
234
+ if replace:
235
+ _drop_this_runs(conn, admin_dsn, name, run_id)
236
+
237
+ try:
238
+ created = create_run_database(
239
+ admin_dsn, name, DB_DIR / "migrations", template=template, before_copy=drop_this_runs
240
+ )
241
+ except DatabaseExists:
242
+ raise RunDirError(f"database {name} already exists: replace it, or name another") from None
243
+ return created.dsn
244
+
245
+
246
+ def _drop_this_runs(conn: Connection, admin_dsn: str, name: str, run_id: str) -> None:
247
+ """Drop the database `name` for a start with replace, if it is run `run_id`'s: its
248
+ `harness.run` holds that run, or no run yet. `conn` holds the database lock.
249
+
250
+ Raises `RunDirError`, and keeps the database, when it holds another run or is no run
251
+ database: it has no `harness.run`.
252
+ """
253
+ row = conn.execute("SELECT oid FROM pg_database WHERE datname = %s", (name,)).fetchone()
254
+ if row is None:
255
+ return
256
+ with psycopg.connect(make_conninfo(admin_dsn, dbname=name), autocommit=True) as found:
257
+ table = found.execute("SELECT to_regclass('harness.run')").fetchone()
258
+ if table is None or table[0] is None:
259
+ raise RunDirError(
260
+ f"database {name} is not a run database, and replace drops only this run's:"
261
+ " name another"
262
+ )
263
+ held = sorted({str(run[0]) for run in found.execute("SELECT run_id FROM harness.run")})
264
+ others = [held_id for held_id in held if held_id != run_id]
265
+ if others:
266
+ raise RunDirError(
267
+ f"database {name} holds run {others[0]}, and replace drops only this run's:"
268
+ " name another"
269
+ )
270
+ drop_database_locked(conn, name, oid=int(row[0]))
271
+
272
+
273
+ async def end_run(
274
+ run: SplitRun, *, admin_dsn: str, simulator_url: str, gateway_url: str | None = None
275
+ ) -> int:
276
+ """End the run; see the module. Returns the virtual time it stopped at."""
277
+ if run.ended:
278
+ raise RunDirError(f"run {run.run_id} has already ended")
279
+ if run.has_gateway and gateway_url is None:
280
+ raise ValueError(f"run {run.run_id} is {run.config}: give the gateway's address")
281
+ secret = run.secret(HARNESS)
282
+ async with httpx.AsyncClient(timeout=CONTROL_TIMEOUT_S) as client:
283
+ if gateway_url is not None and run.has_gateway:
284
+ await _control(client, gateway_url, "drain", secret)
285
+ await _control(client, simulator_url, "drain", secret)
286
+ # The drain window passes in wall time: in the split topology the run's virtual
287
+ # time follows the host's clock (0009).
288
+ await HostScheduler(HostClock(HostClock.host_ms())).sleep(drain_window_ms(run.scenario()))
289
+ if gateway_url is not None and run.has_gateway:
290
+ await _control(client, gateway_url, "stop", secret)
291
+ stopped = await _control(client, simulator_url, "stop", secret)
292
+ stopped_vt = int(stopped["stopped_vt"])
293
+ drop_unlogged(run, admin_dsn)
294
+ run.mark_ended(stopped_vt)
295
+ return stopped_vt
296
+
297
+
298
+ async def _control(client: httpx.AsyncClient, url: str, action: str, secret: str) -> dict[str, Any]:
299
+ response = await client.post(
300
+ f"{url.rstrip('/')}/harness/{action}", headers={"authorization": f"Bearer {secret}"}
301
+ )
302
+ response.raise_for_status()
303
+ answer: dict[str, Any] = response.json()
304
+ return answer
305
+
306
+
307
+ def drop_unlogged(run: SplitRun, admin_dsn: str) -> int:
308
+ """Drop every message whose fate no service logged; return how many there were."""
309
+ with psycopg.connect(run.conninfo(admin_dsn, "ff_harness"), autocommit=True) as conn:
310
+ updated = conn.execute(
311
+ "UPDATE harness.message_log SET fate = 'dropped', drop_reason = CASE kind"
312
+ " WHEN 'answer' THEN 'caller_timed_out' ELSE 'receiver_down' END"
313
+ " WHERE fate IS NULL"
314
+ )
315
+ return updated.rowcount
316
+
317
+
318
+ def score_run(run: SplitRun, *, admin_dsn: str) -> Evaluation:
319
+ """Score the stopped run with the evaluator, as `ff_eval`, and store its result."""
320
+ if not run.ended:
321
+ raise RunDirError(f"run {run.run_id} has not ended: end it first")
322
+ return evaluate_run(run.conninfo(admin_dsn, "ff_eval"), run.scenario())