fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,881 @@
1
+ """One run in the inproc topology: every component on one scheduler (0009).
2
+
3
+ A run is a scenario, a configuration, a contract level, a seed and a model (0009). This
4
+ build runs B0, G1 and G2 with the scripted model, recorded into a cassette or not, with a
5
+ cassette's replay, with a live model, always recorded, or as an experimental branch of a
6
+ recorded run, which replays the parent's first K model calls and then switches to the
7
+ scripted model or a live one, also always recorded (`fivefold.harness.models`). A branch
8
+ runs its parent's run: the same scenario, configuration, contract level, seed, fault plan
9
+ and workers. It is a new run all the same, with its own run id, the parent's with
10
+ `.branch-K` added, and `harness.run` names its parent and its branch point.
11
+
12
+ The run's fault plan is fixed before anything runs (`run_plan`): the run's own plan, or
13
+ else the scenario's, is a list of faults, checked against the run, or a random plan,
14
+ from which the run's `faults` generator draws such a list (0006, Plans;
15
+ `fivefold.faults.plan`). Either way, the list, with the parameters drawn from the seed,
16
+ is what fires and what `harness.run` records, so playback, branches and the report see
17
+ the faults the run had.
18
+
19
+ `InprocRun.execute`:
20
+
21
+ 1. records the run, in its run database and in the run registry (0012,
22
+ `fivefold.harness.registry`), sets up the simulator for the contract with the
23
+ configuration's credentials (only the gateway in G1 and G2, the agent workers in B0;
24
+ 0008), and serves the simulator's API and, in B0, the report sink on the transport;
25
+ 2. in G1 and G2, starts the gateway: its store, its dispatcher after startup recovery, and
26
+ its tools, one MCP server per agent connection. G1's store has action identity off,
27
+ which makes the gateway authorization only (0008);
28
+ 3. starts each agent worker with its own checkpoint thread, and the scripted approver,
29
+ which decides queued approval requests by the scenario's rules (0002, rule 7). F9
30
+ starts one more worker on the same task when it fires; the gateway expects a report
31
+ from every worker the plan can start (0010);
32
+ 4. fires the plan's faults: the transport fires those on its hops, the dispatcher those
33
+ at its points (F1, F6 and F7 early), and this module those at the agent's points (F1's
34
+ B0 equivalent, F11). A crashed gateway restarts through startup recovery (0003), and
35
+ each agent worker then opens a new MCP connection to it; a crashed worker restarts
36
+ from its last checkpoint. A component that a crash stopped reaches no further fault
37
+ point, at the gateway's points and the agent's alike: when two crash faults meet at
38
+ one event, only the first fires, and the component restarts after its delay. The
39
+ simulator's endpoint fires F5 and F10 at the simulator's points
40
+ (`fivefold.harness.sim_points`). Each fault's record names the task intent it
41
+ touched, which recovery latency starts from (0014);
42
+ 5. runs until the agents' phase ends: every worker finished, used up its step budget or
43
+ failed;
44
+ 6. in G2, closes the run if the last report did not (0003); then keeps everything but
45
+ the agents running for the drain window, `2 x (D + max_hop_latency) + L` (0009),
46
+ records the stop time, and a recording's cassette digest, and stops every component.
47
+ A replay that did not serve exactly its cassette's calls then fails the run with
48
+ `ReplayError` (0009);
49
+ 7. runs the evaluator, which scores the run from the simulator's and the harness's
50
+ records and writes its result to the `eval` schema (0004, 0009).
51
+
52
+ When the drain ends, before the stop is recorded, the harness has the simulator abort
53
+ every write still in flight past its deadline, unless a fault waived the deadline (F5
54
+ beyond the deadline): such a write never commits, and the ledger then says so (0005).
55
+
56
+ In G modes, when the scenario has a scripted resolver (0009), the gateway's dispatcher
57
+ tells it when an action is left to a human, and it attests what the simulator's ledger
58
+ shows through the gateway's admin path (`fivefold.harness.actors`, 0003).
59
+
60
+ Each component connects to the run database as its own role: `ff_sim`, `ff_gw`,
61
+ `ff_harness`, `ff_agent` and `ff_eval` (0008). The registry's row is written as
62
+ `ff_registry`, in the registry's own database on the same server.
63
+
64
+ Every model call of every worker goes into the harness's model-call log once the model
65
+ answers, with its virtual times, the run's model and the reply's token usage (0004). Each
66
+ worker has the scenario's step and token budgets (0009, `fivefold.agents.worker`).
67
+ """
68
+
69
+ from __future__ import annotations
70
+
71
+ import asyncio
72
+ from collections.abc import Awaitable, Callable, Mapping, Sequence
73
+ from dataclasses import dataclass, field
74
+ from typing import Any, Literal
75
+
76
+ import psycopg
77
+ from langchain_core.language_models.chat_models import BaseChatModel
78
+ from langchain_core.messages import AIMessage
79
+ from psycopg.conninfo import make_conninfo
80
+
81
+ from fivefold.agents.checkpoint import InlinePostgresSaver, connect
82
+ from fivefold.agents.connection import GatewayConnection
83
+ from fivefold.agents.live import Think, refusal
84
+ from fivefold.agents.policy import Mode, scripted_policy
85
+ from fivefold.agents.scripted import ScriptedModel
86
+ from fivefold.agents.tools import Timeouts, baseline_tools, gateway_tools
87
+ from fivefold.agents.worker import CallLog, WorkerEnd, WorkerSpec, build_graph, run_worker
88
+ from fivefold.domain.contracts import Capability, Contract
89
+ from fivefold.domain.scenario import Scenario, scenario_digest
90
+ from fivefold.evaluator import RunResult as Evaluation
91
+ from fivefold.evaluator import evaluate_run
92
+ from fivefold.faults.catalog import Action
93
+ from fivefold.faults.plan import (
94
+ FaultInjector,
95
+ Fired,
96
+ PlannedFault,
97
+ check_configuration,
98
+ check_run,
99
+ draw_plan,
100
+ parse_plan,
101
+ parse_random_plan,
102
+ seed_parameters,
103
+ )
104
+ from fivefold.gateway.admin import GatewayAdmin
105
+ from fivefold.gateway.dispatcher import Dispatcher, Timing
106
+ from fivefold.gateway.freshness import RuleFreshness
107
+ from fivefold.gateway.pg_store import PgStore
108
+ from fivefold.gateway.server import serve
109
+ from fivefold.gateway.sim_client import SimClient
110
+ from fivefold.gateway.state_machine import GatewaySettings
111
+ from fivefold.gateway.tools import GatewayTools, ToolTiming
112
+ from fivefold.harness.actors import PgEffectsLedger, ScriptedApprover, ScriptedResolver
113
+ from fivefold.harness.clock import settle, wall_ms
114
+ from fivefold.harness.fault_effects import SimulatorFaults, message_context
115
+ from fivefold.harness.log import FaultRecord, ModelCallRecord, PgHarnessLog, RunRecord
116
+ from fivefold.harness.mcp_link import Link
117
+ from fivefold.harness.models import (
118
+ SCRIPTED,
119
+ Branch,
120
+ ModelSpec,
121
+ Parent,
122
+ RunModels,
123
+ branch_run_id,
124
+ parent_of,
125
+ )
126
+ from fivefold.harness.registry import RegistryEntry, database_of, register
127
+ from fivefold.harness.reports import ReportSink, sink_endpoint
128
+ from fivefold.harness.rng import Generators
129
+ from fivefold.harness.scheduler import Scheduler
130
+ from fivefold.harness.sim_points import SimulatorPoints
131
+ from fivefold.harness.transport import Envelope, Outgoing, Stage, Transport
132
+ from fivefold.recorder.cassette import RunHeader, plain_object
133
+ from fivefold.sim import SimConfig, Simulator
134
+ from fivefold.sim.endpoint import endpoint
135
+
136
+ Config = Literal["B0", "G1", "G2"]
137
+
138
+ GATEWAY_CONFIGS = ("G1", "G2")
139
+ """The configurations that write through the gateway (0008). G1 is authorization only."""
140
+
141
+ CONFIGS: tuple[Config, ...] = ("B0", "G1", "G2")
142
+ """Every configuration a run can have (0008)."""
143
+
144
+ GATEWAY = "gateway"
145
+ SIMULATOR = "simulator"
146
+ REPORT_SINK = "report_sink"
147
+
148
+ POSSIBLY_SENT_POINT = "gw.dispatch.after_record"
149
+ """A gateway crash here leaves a request that may or may not have left (0006, crashes)."""
150
+
151
+ VOIDING_POINT = "gw.dispatch.after_answer"
152
+ """A gateway crash here voids the answer it just received (0006, crashes)."""
153
+
154
+ GATEWAY_TOOL_NAMES = frozenset(
155
+ {
156
+ "get_quote",
157
+ "list_suppliers",
158
+ "get_budget",
159
+ "get_inventory",
160
+ "submit_purchase_order",
161
+ "reserve_inventory",
162
+ "get_action_status",
163
+ "submit_final_report",
164
+ }
165
+ )
166
+ BASELINE_TOOL_NAMES = frozenset(
167
+ {
168
+ "get_quote",
169
+ "list_suppliers",
170
+ "get_budget",
171
+ "get_inventory",
172
+ "request_approval",
173
+ "get_approval_status",
174
+ "submit_purchase_order",
175
+ "reserve_inventory",
176
+ "get_order_status",
177
+ "get_reservation_status",
178
+ "submit_final_report",
179
+ }
180
+ )
181
+
182
+
183
+ @dataclass(frozen=True)
184
+ class RunSpec:
185
+ """What to run: `plan` defaults to the scenario's fault plan. A plan is a list of
186
+ faults, or a random plan, `{random: {from: [...], max: N}}`, which the run draws from
187
+ its seed (`run_plan`)."""
188
+
189
+ scenario: Scenario
190
+ config: Config
191
+ contract: Contract
192
+ seed: int
193
+ plan: Sequence[Mapping[str, Any]] | Mapping[str, Any] | None = None
194
+ workers: int = 1
195
+ model: ModelSpec = SCRIPTED
196
+ """The model adapter: scripted, scripted with recording, replay, live, or a branch's
197
+ (0009)."""
198
+
199
+ @property
200
+ def parent(self) -> Parent | None:
201
+ """For an experimental branch, or a replay of one, the run it branched from and
202
+ its branch point (0009); None for any other run."""
203
+ return parent_of(self.model)
204
+
205
+ @property
206
+ def run_id(self) -> str:
207
+ """A readable id that names the run's choices. A branch's is its parent's run id
208
+ with `.branch-K` added, K being its branch point (`branch_run_id`)."""
209
+ parent = self.parent
210
+ if parent is not None:
211
+ return branch_run_id(parent)
212
+ return f"{self.scenario.id}.{self.config}.{self.contract}.{self.seed}"
213
+
214
+
215
+ @dataclass
216
+ class RunResult:
217
+ """How a run ended: each worker's end, the virtual time the run stopped, and the
218
+ evaluator's result."""
219
+
220
+ run_id: str
221
+ ends: dict[str, tuple[WorkerEnd, str]] = field(default_factory=dict)
222
+ stopped_vt: int = 0
223
+ evaluation: Evaluation | None = None
224
+
225
+
226
+ def drain_window_ms(scenario: Scenario) -> int:
227
+ """`2 x (D + max_hop_latency) + L` (0009)."""
228
+ params = scenario.params
229
+ return 2 * (params.commit_deadline_ms + params.max_hop_latency_ms) + params.status_lag_ms
230
+
231
+
232
+ def run_plan(spec: RunSpec) -> tuple[PlannedFault, ...]:
233
+ """The run's fault plan, as it fires and as `harness.run` records it (0006, Plans).
234
+
235
+ The run's own plan, or else the scenario's, is either a list of faults, which must
236
+ fit the run's configuration, timing and contract level, or a random plan, from which
237
+ the run's `faults` generator draws such a list (`fivefold.faults.plan.draw_plan`).
238
+ Then the same generator draws each seeded parameter the list leaves unset (0009). So
239
+ a seed always gives the same plan, and a list with no random part draws what it
240
+ always did. Raises `ValueError` for a plan the run cannot carry out.
241
+ """
242
+ faults = spec.plan if spec.plan is not None else spec.scenario.faults
243
+ draws = Generators(spec.seed).get("faults")
244
+ if isinstance(faults, Mapping):
245
+ plan = draw_plan(
246
+ parse_random_plan(faults),
247
+ draws,
248
+ scenario=spec.scenario,
249
+ config=spec.config,
250
+ contract=spec.contract,
251
+ )
252
+ else:
253
+ plan = parse_plan(faults)
254
+ check_configuration(plan, gateway=spec.config != "B0")
255
+ check_run(plan, params=spec.scenario.params, contract=spec.contract)
256
+ return seed_parameters(plan, draws)
257
+
258
+
259
+ class InprocRun:
260
+ """One run of a scenario on one scheduler, in a migrated run database."""
261
+
262
+ def __init__(self, spec: RunSpec, dsn: str) -> None:
263
+ """`dsn` names the run database; each component connects with its own role."""
264
+ if spec.config not in ("B0", *GATEWAY_CONFIGS):
265
+ raise ValueError(f"this build runs B0, G1 and G2, not {spec.config}")
266
+ self.spec = spec
267
+ self._plan = run_plan(spec)
268
+ self._dsn = dsn
269
+ self.scheduler = Scheduler()
270
+ self.result = RunResult(spec.run_id)
271
+ self._workers = list(_worker_names(spec.workers))
272
+ self._ranks: dict[str, int] = {}
273
+ self._agent_workers = spec.workers + sum(
274
+ 1 for entry in self._plan if entry.kind.action is Action.START_WORKER
275
+ )
276
+ gateway = spec.config in GATEWAY_CONFIGS
277
+ self._tool_names = GATEWAY_TOOL_NAMES if gateway else BASELINE_TOOL_NAMES
278
+ self._closing: list[Any] = []
279
+ self._last_write_answers: dict[tuple[str, str], int] = {}
280
+ self._params: dict[str, object] = {
281
+ "params": spec.scenario.params.model_dump(mode="json"),
282
+ "plan": [entry.model_dump(mode="json") for entry in self._plan],
283
+ "workers": spec.workers,
284
+ "agent_workers": self._agent_workers,
285
+ }
286
+ """The values the run uses, as `harness.run.params` records them (0009)."""
287
+ if isinstance(spec.model, Branch):
288
+ _check_branch(spec, self._params, spec.model.parent.run)
289
+ self.models = RunModels(spec.model)
290
+ self._model_calls = 0
291
+
292
+ def _conninfo(self, role: str) -> str:
293
+ return make_conninfo(self._dsn, user=role)
294
+
295
+ async def execute(self) -> RunResult:
296
+ """Run to the end of the drain window; return how each worker ended."""
297
+ spec, scenario = self.spec, self.spec.scenario
298
+ params = scenario.params
299
+ harness_conn = psycopg.connect(self._conninfo("ff_harness"), autocommit=True)
300
+ self._closing.append(harness_conn)
301
+ self.log = PgHarnessLog(harness_conn)
302
+ plan = self._plan
303
+ parent = spec.parent
304
+ self.log.run_recorded(
305
+ RunRecord(
306
+ run_id=spec.run_id,
307
+ scenario_id=scenario.id,
308
+ scenario_digest=scenario_digest(scenario),
309
+ config=spec.config,
310
+ contract=str(spec.contract), # type: ignore[arg-type]
311
+ seed=spec.seed,
312
+ model_adapter=self.models.adapter,
313
+ model=self.models.name,
314
+ cassette_digest=self.models.cassette_digest,
315
+ params=self._params,
316
+ parent_run_id=None if parent is None else parent.run_id,
317
+ branch_point=None if parent is None else parent.branch_point,
318
+ )
319
+ )
320
+ register(
321
+ self._dsn,
322
+ RegistryEntry(
323
+ database=database_of(self._dsn),
324
+ run_id=spec.run_id,
325
+ scenario_id=scenario.id,
326
+ config=spec.config,
327
+ contract=str(spec.contract),
328
+ seed=spec.seed,
329
+ ),
330
+ )
331
+ self.faults = FaultInjector(plan)
332
+ generators = Generators(spec.seed)
333
+ self.transport = Transport(
334
+ self.scheduler,
335
+ self.log,
336
+ generators.get("latency"),
337
+ max_hop_latency_ms=params.max_hop_latency_ms,
338
+ faults=self.faults,
339
+ )
340
+ self.transport.observe(self._agent_points)
341
+ self.transport.observe(self._gateway_answers)
342
+ self.transport.before_send(self._before_tool_call)
343
+ gateway = spec.config in GATEWAY_CONFIGS
344
+ credentials = (GATEWAY,) if gateway else _worker_names(self._agent_workers)
345
+ config = SimConfig.preset(
346
+ spec.contract, credentials=credentials, params=params.contract_params()
347
+ )
348
+ self.simulator = Simulator.setup(self._conninfo("ff_sim"), scenario, config, vt=0)
349
+ self._closing.append(self.simulator)
350
+ self.approver = ScriptedApprover(
351
+ self.scheduler, self.simulator.control, self.log, scenario.authority
352
+ )
353
+ self.resolver = self._resolver(PgEffectsLedger(harness_conn))
354
+ self.effects = SimulatorFaults(self.scheduler, self.simulator.control, self.log, scenario)
355
+ self.transport.apply_faults_with(
356
+ lambda fired, envelope: self.effects.apply(fired, message_context(envelope.body))
357
+ )
358
+ self.transport.serve(
359
+ SIMULATOR,
360
+ endpoint(
361
+ self.simulator,
362
+ self.scheduler,
363
+ poll_ms=params.approval_poll_interval_ms,
364
+ max_wait_ms=params.max_wait_ms,
365
+ served=[self.approver.served],
366
+ points=self._simulator_points(),
367
+ ),
368
+ )
369
+ intents = {intent.id: str(intent.operation) for intent in scenario.intents}
370
+ self._intents = intents
371
+ self.sink = ReportSink(self.scheduler, self.log, intents)
372
+ if spec.config == "B0":
373
+ self.transport.serve(REPORT_SINK, sink_endpoint(self.sink))
374
+ checkpoint_conn = connect(self._conninfo("ff_agent"))
375
+ self._closing.append(checkpoint_conn)
376
+ self.checkpointer = InlinePostgresSaver(checkpoint_conn)
377
+ self.checkpointer.setup()
378
+ self.timeouts = Timeouts(
379
+ t_ms=params.dispatch_timeout_ms,
380
+ tool_wait_ms=params.tool_wait_ms,
381
+ max_wait_ms=params.max_wait_ms,
382
+ )
383
+ if gateway:
384
+ self._start_gateway()
385
+ for worker in self._workers:
386
+ self._start_worker(worker, resume=False)
387
+ await self.scheduler.run(until=self._agents_done)
388
+ self._close_run()
389
+ drain_end = self.scheduler.now + drain_window_ms(scenario)
390
+ await self.scheduler.run(deadline_ms=drain_end)
391
+ # The drain lasts its whole window (0009). The scheduler returns as soon as nothing
392
+ # is queued and leaves the clock there, so a quiet drain moves it to the end here.
393
+ self.scheduler.clock.advance_to(max(self.scheduler.now, drain_end))
394
+ self.simulator.control.abort_overdue(vt=self.scheduler.now)
395
+ self.result.stopped_vt = self.scheduler.now
396
+ self.log.run_stopped(self.scheduler.now)
397
+ self._models_stopped()
398
+ await self._stop()
399
+ self.models.check()
400
+ self.result.evaluation = evaluate_run(self._conninfo("ff_eval"), scenario)
401
+ return self.result
402
+
403
+ def _agents_done(self) -> bool:
404
+ return len(self.result.ends) == len(self._workers)
405
+
406
+ def _simulator_points(self) -> SimulatorPoints:
407
+ """F5 and F10, at the simulator's fault points (0006). They also watch the status
408
+ answers go out, to log each one F10 changed (0014)."""
409
+ params = self.spec.scenario.params
410
+ points = SimulatorPoints(
411
+ self.scheduler,
412
+ self.faults,
413
+ self.log,
414
+ dispatch_timeout_ms=params.dispatch_timeout_ms,
415
+ status_lag_ms=params.status_lag_ms,
416
+ )
417
+ self.transport.observe(points.answer_sent)
418
+ return points
419
+
420
+ # The gateway --------------------------------------------------------------------------
421
+
422
+ def _start_gateway(self) -> None:
423
+ """Start the gateway: its store, startup recovery and its dispatcher (0003).
424
+
425
+ Also how a crashed gateway restarts: everything it had in memory is new.
426
+ """
427
+ scenario, params = self.spec.scenario, self.spec.scenario.params
428
+ intents = self._intents
429
+ settings = GatewaySettings(
430
+ approval_wait_ms=scenario.authority.approval_wait_ms,
431
+ params=params.contract_params(),
432
+ dispatch_timeout_ms=params.dispatch_timeout_ms,
433
+ max_attempts=params.max_attempts,
434
+ max_probes=params.max_probes,
435
+ agent_workers=self._agent_workers,
436
+ )
437
+ self.store = PgStore(
438
+ self._conninfo("ff_gw"),
439
+ task_id=scenario.task_id,
440
+ intents=intents,
441
+ contract=self.spec.contract,
442
+ settings=settings,
443
+ identity=self.spec.config == "G2",
444
+ )
445
+ self._closing.append(self.store)
446
+ client = SimClient(self.transport, timeout_ms=params.dispatch_timeout_ms)
447
+ self.dispatcher = Dispatcher(
448
+ self.scheduler,
449
+ self.store,
450
+ client,
451
+ RuleFreshness(self.store),
452
+ component=GATEWAY,
453
+ timing=Timing(
454
+ approval_poll_ms=params.approval_poll_interval_ms,
455
+ probe_interval_ms=params.status_probe_interval_ms,
456
+ ),
457
+ faults=self.faults,
458
+ on_fault=self._gateway_fault,
459
+ on_escalated=self._escalated,
460
+ )
461
+ self.tools = GatewayTools(
462
+ self.scheduler,
463
+ self.store,
464
+ self.dispatcher,
465
+ client,
466
+ self.sink,
467
+ task_id=scenario.task_id,
468
+ intents=intents,
469
+ timing=ToolTiming(tool_wait_ms=params.tool_wait_ms, max_wait_ms=params.max_wait_ms),
470
+ )
471
+ self.dispatcher.start()
472
+
473
+ def _close_run(self) -> None:
474
+ """In G2, once the agents' phase is over, close the run if no report did (0003).
475
+
476
+ A gateway that is down then closes it when it restarts.
477
+ """
478
+ if self.spec.config != "G2" or not self.scheduler.is_up(GATEWAY):
479
+ return
480
+ if not self.store.snapshot().run_closed:
481
+ self.store.close_run(self.scheduler.now)
482
+
483
+ def _gateway_fault(self, fired: Fired, point: str, context: Mapping[str, object]) -> None:
484
+ """A fault at a dispatch point: record it, then apply it: F6 and F7 early through
485
+ the simulator, F1 and F3 by crashing the gateway.
486
+
487
+ What the crash changes for the evidence goes in the fault's record (0006, crashes;
488
+ 0004, property 5). A crash after the attempt is recorded leaves a request that may
489
+ have left: the record names it. A crash after the answer arrived voids that answer,
490
+ which the gateway never recorded.
491
+ """
492
+ action = fired.planned.kind.action
493
+ params: dict[str, object] = dict(fired.planned.params)
494
+ voided: tuple[int, ...] = ()
495
+ if action is Action.KILL and point == POSSIBLY_SENT_POINT:
496
+ params["request"] = {
497
+ "intent_id": context["intent_id"],
498
+ "api": context["api"],
499
+ "idempotency_key": context["idempotency_key"],
500
+ }
501
+ if action is Action.KILL and point == VOIDING_POINT:
502
+ answer = self._last_write_answers.get((str(context["api"]), str(context["intent_id"])))
503
+ if answer is None:
504
+ raise RuntimeError(f"{fired.fault} at {point}: no answer reached the gateway")
505
+ voided = (answer,)
506
+ self.log.fault_fired(
507
+ FaultRecord(
508
+ seq=self.log.next_seq(),
509
+ fault=str(fired.fault),
510
+ variant=fired.planned.variant,
511
+ point=point,
512
+ occurrence=fired.occurrence,
513
+ action=str(action),
514
+ vt=self.scheduler.now,
515
+ component=GATEWAY,
516
+ voided=voided,
517
+ params=params,
518
+ intent_id=_text(context.get("intent_id")),
519
+ )
520
+ )
521
+ if action is Action.KILL:
522
+ self._crash_gateway(fired.planned.param("restart_delay_ms"))
523
+ else:
524
+ self.effects.apply(fired, context)
525
+
526
+ def _crash_gateway(self, restart_after_ms: int) -> None:
527
+ """Stop the gateway now and start it again later (0006, crashes).
528
+
529
+ Its memory goes, and so does its database connection, with any open
530
+ transaction and the run's lock; its tasks are cancelled. The restart builds a new
531
+ gateway, which recovers from its records before it serves anything (0003).
532
+ """
533
+ self.store.close()
534
+ self.scheduler.crash(
535
+ GATEWAY, restart_after_ms=restart_after_ms, restart=self._restart_gateway
536
+ )
537
+
538
+ def _restart_gateway(self) -> None:
539
+ self._start_gateway()
540
+ if self._agents_done():
541
+ self._close_run()
542
+
543
+ def _connect(self, worker: str) -> Link:
544
+ """Open a new MCP connection from `worker` to the gateway.
545
+
546
+ A gateway that is up serves it, with the tools of its current incarnation; one
547
+ that is down never does.
548
+ """
549
+ link = Link(self.scheduler, self.transport, agent=worker, gateway=GATEWAY)
550
+ if self.scheduler.is_up(GATEWAY):
551
+ self.scheduler.spawn(GATEWAY, serve(self.tools, link), name=f"mcp server for {worker}")
552
+ return link
553
+
554
+ # The scripted resolver (0003, 0009) -----------------------------------------------------
555
+
556
+ def _resolver(self, ledger: PgEffectsLedger) -> ScriptedResolver | None:
557
+ """The scripted resolver, in G modes when the scenario has one (0009).
558
+
559
+ It reads the simulator's ledger out of band through `ledger`, as `ff_harness`
560
+ (0008). B0 has no gateway, so nothing there escalates.
561
+ """
562
+ rule = self.spec.scenario.authority.resolver
563
+ if self.spec.config == "B0" or rule == "none":
564
+ return None
565
+ settle_ms = self.spec.scenario.params.contract_params().settle_ms
566
+ return ScriptedResolver(
567
+ self.scheduler, self._admin, ledger, self.log, rule, settle_ms=settle_ms
568
+ )
569
+
570
+ def _admin(self) -> GatewayAdmin | None:
571
+ """The admin path of the gateway's current incarnation, or None while it is down."""
572
+ if not self.scheduler.is_up(GATEWAY):
573
+ return None
574
+ return GatewayAdmin(self.scheduler, self.store, self.dispatcher)
575
+
576
+ def _escalated(self, action_id: str) -> None:
577
+ """The gateway left an escalated action to a human: tell the resolver, if any."""
578
+ if self.resolver is not None:
579
+ self.resolver.escalated(action_id)
580
+
581
+ # Agent workers ------------------------------------------------------------------------
582
+
583
+ def _start_worker(self, worker: str, *, resume: bool) -> None:
584
+ label = "resumes" if resume else "starts"
585
+ self.scheduler.spawn(worker, self._worker(worker, resume=resume), name=f"{worker} {label}")
586
+
587
+ async def _worker(self, worker: str, *, resume: bool) -> None:
588
+ scenario = self.spec.scenario
589
+ model = self._chat_model(worker)
590
+ budget = scenario.budget
591
+ spec = WorkerSpec(worker, scenario.task, budget.max_agent_steps, budget.max_tokens)
592
+ on_call = self._call_log(worker)
593
+
594
+ def on_step(step: int) -> None:
595
+ self._agent_step(worker, step)
596
+
597
+ if self.spec.config == "B0":
598
+ tools = baseline_tools(
599
+ self.transport,
600
+ self.scheduler,
601
+ self.timeouts,
602
+ worker=worker,
603
+ status=self.spec.contract.has(Capability.STATUS),
604
+ )
605
+ graph = build_graph(
606
+ model,
607
+ tools,
608
+ self.checkpointer,
609
+ max_steps=spec.max_steps,
610
+ max_tokens=spec.max_tokens,
611
+ on_step=on_step,
612
+ on_call=on_call,
613
+ )
614
+ self.result.ends[worker] = await run_worker(graph, spec, resume=resume)
615
+ return
616
+ connection = GatewayConnection(
617
+ self.scheduler,
618
+ lambda: self._connect(worker),
619
+ worker=worker,
620
+ timeout_ms=scenario.params.dispatch_timeout_ms,
621
+ )
622
+ try:
623
+ listed = await connection.list_tools()
624
+ tools = gateway_tools(connection, listed, self.timeouts)
625
+ graph = build_graph(
626
+ model,
627
+ tools,
628
+ self.checkpointer,
629
+ max_steps=spec.max_steps,
630
+ max_tokens=spec.max_tokens,
631
+ on_step=on_step,
632
+ on_call=on_call,
633
+ )
634
+ self.result.ends[worker] = await run_worker(graph, spec, resume=resume)
635
+ except asyncio.CancelledError:
636
+ raise
637
+ except Exception as error:
638
+ # A worker that cannot connect, or fails otherwise, ends without a report (0008).
639
+ self.result.ends[worker] = (WorkerEnd.ERROR, _describe(error))
640
+ finally:
641
+ # After a crash the link ends the connection itself, once in-flight requests
642
+ # have reached the gateway; this closes it only when the worker finishes.
643
+ connection.close()
644
+
645
+ # Fault points of the agent (0006) -----------------------------------------------------
646
+
647
+ def _agent_step(self, worker: str, step: int) -> None:
648
+ """`agent.step`: F9 starts another worker on the same task (0006).
649
+
650
+ The new worker runs the same scripted policy from the start, with its own
651
+ checkpoint thread and step budget; in the `different_args` variant it chooses the
652
+ second-cheapest acceptable option (0008).
653
+ """
654
+ for fired in self.faults.at("agent.step", {"worker": worker, "step": step}):
655
+ if fired.planned.kind.action is not Action.START_WORKER:
656
+ continue
657
+ started = f"agent-{len(self._workers) + 1}"
658
+ self.log.fault_fired(
659
+ FaultRecord(
660
+ seq=self.log.next_seq(),
661
+ fault=str(fired.fault),
662
+ variant=fired.planned.variant,
663
+ point="agent.step",
664
+ occurrence=fired.occurrence,
665
+ action=str(Action.START_WORKER),
666
+ vt=self.scheduler.now,
667
+ component=started,
668
+ params={**fired.planned.params, "worker": worker, "step": step},
669
+ intent_id=None,
670
+ )
671
+ )
672
+ self._workers.append(started)
673
+ self._ranks[started] = 1 if fired.planned.variant == "different_args" else 0
674
+ self._start_worker(started, resume=False)
675
+
676
+ def _before_tool_call(self, outgoing: Outgoing) -> None:
677
+ """F1's B0 equivalent: kill a worker just before it sends a tool call (0006).
678
+
679
+ The call is never sent. The worker resumes from its last checkpoint, which comes
680
+ before the tool node, so it sends the call once it resumes.
681
+ """
682
+ worker = outgoing.sender
683
+ if outgoing.kind != "request" or worker not in self._workers:
684
+ return
685
+ if outgoing.api not in self._tool_names:
686
+ return
687
+ fields = {"api": outgoing.api, "tool": outgoing.api, "intent_id": outgoing.intent_id}
688
+ for fired in self.faults.at("agent.before_tool_call", fields):
689
+ if fired.planned.kind.action is not Action.KILL:
690
+ continue
691
+ self.log.fault_fired(
692
+ FaultRecord(
693
+ seq=self.log.next_seq(),
694
+ fault=str(fired.fault),
695
+ variant=fired.planned.variant,
696
+ point="agent.before_tool_call",
697
+ occurrence=fired.occurrence,
698
+ action=str(Action.KILL),
699
+ vt=self.scheduler.now,
700
+ component=worker,
701
+ params=dict(fired.planned.params),
702
+ intent_id=outgoing.intent_id,
703
+ )
704
+ )
705
+ self._kill(worker, fired.planned.param("restart_delay_ms"))
706
+ return # The worker is down, so it reaches no further fault (`_agent_points`).
707
+
708
+ def _gateway_answers(self, envelope: Envelope, stage: Stage) -> None:
709
+ """Note each write answer delivered to the gateway, by API and intent, so that a
710
+ crash right after it (F3) can void it."""
711
+ if (
712
+ stage == "delivered"
713
+ and envelope.hop == "gw_sim"
714
+ and envelope.kind == "answer"
715
+ and envelope.receiver == GATEWAY
716
+ and envelope.api is not None
717
+ and envelope.intent_id is not None
718
+ ):
719
+ self._last_write_answers[(envelope.api, envelope.intent_id)] = envelope.seq
720
+
721
+ def _agent_points(self, envelope: Envelope, stage: Stage) -> None:
722
+ """F11, and F3's B0 equivalent: kill a worker after it sends a tool call, or as a
723
+ tool result reaches it."""
724
+ if stage == "sent" and envelope.kind == "request":
725
+ worker, point = envelope.sender, "agent.after_tool_call"
726
+ elif stage == "delivered" and envelope.kind == "answer":
727
+ worker, point = envelope.receiver, "agent.after_tool_result"
728
+ else:
729
+ return
730
+ if worker not in self._workers or envelope.api not in self._tool_names:
731
+ return
732
+ fields = {"api": envelope.api, "tool": envelope.api, "intent_id": envelope.intent_id}
733
+ for fired in self.faults.at(point, fields):
734
+ if fired.planned.kind.action is not Action.KILL:
735
+ continue
736
+ delay = fired.planned.param("restart_delay_ms")
737
+ voided = (envelope.seq,) if point == "agent.after_tool_result" else ()
738
+ self.log.fault_fired(
739
+ FaultRecord(
740
+ seq=self.log.next_seq(),
741
+ fault=str(fired.fault),
742
+ variant=fired.planned.variant,
743
+ point=point,
744
+ occurrence=fired.occurrence,
745
+ action=str(Action.KILL),
746
+ vt=self.scheduler.now,
747
+ target_seq=envelope.seq,
748
+ component=worker,
749
+ voided=voided,
750
+ params=dict(fired.planned.params),
751
+ intent_id=envelope.intent_id,
752
+ )
753
+ )
754
+ # The kill is immediate. After a tool call, the request is already on the
755
+ # wire; after a result, the worker must not see it: the transport hands a
756
+ # delivery to its receiver only after the observers ran, and only if the
757
+ # receiver is still up (0006, voided results).
758
+ self._kill(worker, delay)
759
+ # A worker that is down reaches no fault point, as the gateway's dispatcher
760
+ # stops at the first crash: a second kill at this event does not fire, and
761
+ # the worker restarts once, after this kill's delay.
762
+ return
763
+
764
+ def _kill(self, worker: str, restart_after_ms: int) -> None:
765
+ self.scheduler.crash(
766
+ worker,
767
+ restart_after_ms=restart_after_ms,
768
+ restart=lambda: self._start_worker(worker, resume=True),
769
+ )
770
+
771
+ # The model adapter (0009) -------------------------------------------------------------
772
+
773
+ def _chat_model(self, worker: str) -> BaseChatModel:
774
+ """The chat model of `worker`, built each time it starts or resumes, by the run's
775
+ model adapter (`fivefold.harness.models`)."""
776
+ scenario = self.spec.scenario
777
+ mode = Mode.GATEWAY if self.spec.config in GATEWAY_CONFIGS else Mode.BASELINE
778
+ rank = self._ranks.get(worker, 0)
779
+
780
+ def think() -> Awaitable[None]:
781
+ return self.scheduler.sleep(scenario.params.think_ms)
782
+
783
+ def scripted(its_think: Think) -> BaseChatModel:
784
+ return ScriptedModel(
785
+ lambda: scripted_policy(
786
+ scenario.intents, mode=mode, rank=rank, max_wait_ms=scenario.params.max_wait_ms
787
+ ),
788
+ its_think,
789
+ )
790
+
791
+ return self.models.chat_model(
792
+ worker, scripted=scripted, think=think, hold=self.scheduler.hold
793
+ )
794
+
795
+ def _call_log(self, worker: str) -> CallLog:
796
+ """Log each model call of `worker` once it is answered (0004): its number in the
797
+ run, its virtual times, the run's model, and the reply's token usage."""
798
+
799
+ def asked() -> Callable[[AIMessage], None]:
800
+ asked_vt, started = self.scheduler.now, wall_ms()
801
+
802
+ def answered(reply: AIMessage) -> None:
803
+ self._model_calls += 1
804
+ usage = reply.usage_metadata
805
+ self.log.model_called(
806
+ ModelCallRecord(
807
+ call_no=self._model_calls,
808
+ worker=worker,
809
+ asked_vt=asked_vt,
810
+ answered_vt=self.scheduler.now,
811
+ model=self.models.name,
812
+ input_tokens=None if usage is None else usage["input_tokens"],
813
+ output_tokens=None if usage is None else usage["output_tokens"],
814
+ refused=refusal(reply),
815
+ wall_ms=wall_ms() - started,
816
+ )
817
+ )
818
+
819
+ return answered
820
+
821
+ return asked
822
+
823
+ def _models_stopped(self) -> None:
824
+ """The run has stopped: a recording's cassette is complete, and `harness.run`
825
+ records its digest."""
826
+ if self.log.run is None:
827
+ raise RuntimeError("the run was never recorded")
828
+ digest = self.models.stopped(self.log.run)
829
+ if digest is not None:
830
+ self.log.cassette_recorded(digest)
831
+
832
+ # Stopping -------------------------------------------------------------------------------
833
+
834
+ async def _stop(self) -> None:
835
+ """Stop every component, let their tasks unwind, and close their connections."""
836
+ for component in (*self._workers, GATEWAY, SIMULATOR, REPORT_SINK):
837
+ if self.scheduler.is_up(component):
838
+ self.scheduler.crash(component)
839
+ await settle()
840
+ for resource in reversed(self._closing):
841
+ resource.close()
842
+
843
+
844
+ def _worker_names(count: int) -> tuple[str, ...]:
845
+ """The names of a run's agent workers: `agent-1` to `agent-{count}`."""
846
+ return tuple(f"agent-{number}" for number in range(1, count + 1))
847
+
848
+
849
+ def _text(value: object) -> str | None:
850
+ return value if isinstance(value, str) else None
851
+
852
+
853
+ def _describe(error: BaseException) -> str:
854
+ """The error that ended a worker; the first one inside an exception group, as the
855
+ MCP session's task group wraps them."""
856
+ while isinstance(error, BaseExceptionGroup) and error.exceptions:
857
+ error = error.exceptions[0]
858
+ return f"{type(error).__name__}: {error}"
859
+
860
+
861
+ def _check_branch(spec: RunSpec, params: Mapping[str, object], parent: RunHeader) -> None:
862
+ """Refuse a branch that would not run its parent's run (0009): the same scenario, with
863
+ the same digest, configuration, contract level, seed and topology, and the same
864
+ values, which hold the fault plan with the parameters drawn from the seed, and the
865
+ workers. Otherwise the branch would name a parent it does not continue."""
866
+ scenario = spec.scenario
867
+ mine: dict[str, object] = {
868
+ "scenario_id": scenario.id,
869
+ "scenario_digest": scenario_digest(scenario),
870
+ "config": spec.config,
871
+ "contract": str(spec.contract),
872
+ "seed": spec.seed,
873
+ "topology": "inproc",
874
+ "params": plain_object(params, "the run's params"),
875
+ }
876
+ differ = [name for name, value in mine.items() if getattr(parent, name) != value]
877
+ if differ:
878
+ raise ValueError(
879
+ f"a branch runs its parent's run, but it differs from {parent.run_id} in"
880
+ f" {', '.join(differ)}"
881
+ )