fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,441 @@
1
+ """The deterministic scheduler for the inproc topology (0009).
2
+
3
+ One process, one thread, one asyncio event loop. The scheduler owns the run's virtual
4
+ clock and a queue of timed events, ordered by (time, enqueue sequence). It repeats one
5
+ step: let every task run until each waits on the scheduler (`settle`), then take the
6
+ next event, move the clock to it and fire it. Message deliveries, timeouts, probes,
7
+ polls, approver delays and restart delays are all such events, so nothing depends on
8
+ wall-clock time, and a run full of long timeouts takes milliseconds.
9
+
10
+ Components (the agent workers, the gateway, the simulator and the harness's scripted
11
+ humans) run their asynchronous work as tasks spawned through the scheduler, under the
12
+ component's name. Libraries that hand synchronous work to the event loop's default
13
+ executor, as LangChain and LangGraph do for synchronous callables, run it at once in the
14
+ loop's thread instead (`InlineExecutor`), so a run never has a second thread.
15
+
16
+ Crashing a component cancels its tasks and discards their memory; it stays down until
17
+ its restart event fires (0006). Each crash starts a new incarnation, so the transport can
18
+ drop answers to requests sent before the crash (0010, change 3). A restart belongs to the
19
+ incarnation its crash started. When the component crashes again before that restart
20
+ fires, the earlier restart does nothing, so a component never runs as two copies at
21
+ once.
22
+
23
+ A component that waits on the real world, as the live model adapter waits on its
24
+ provider, awaits it through `hold`. Until the wait ends, the scheduler fires no event: it
25
+ waits in wall time, with virtual time held still. So every timer fires at the virtual time
26
+ it would have without the wait, and a model call takes exactly its `think_ms` (0009).
27
+
28
+ `HostScheduler` is the split topology's scheduler. It has the same interface, so the
29
+ gateway, the simulator and the fault points run on it unchanged, but its virtual time
30
+ follows the host's monotonic clock (`HostClock`, 0009): its events are the event loop's
31
+ timers, its components' tasks run as the event loop schedules them, and nothing it does
32
+ is deterministic.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import asyncio
38
+ import heapq
39
+ import logging
40
+ from collections.abc import Awaitable, Callable, Coroutine
41
+ from concurrent.futures import Future, ThreadPoolExecutor
42
+ from dataclasses import dataclass, field
43
+ from enum import StrEnum
44
+ from typing import Any, TypeVar
45
+
46
+ from fivefold.harness.clock import HostClock, RealWaits, VirtualClock, settle
47
+
48
+ T = TypeVar("T")
49
+
50
+ logger = logging.getLogger(__name__)
51
+
52
+
53
+ class SchedulerTimeout(Exception):
54
+ """A wait ended because its virtual-time limit passed."""
55
+
56
+
57
+ class ComponentFailed(RuntimeError):
58
+ """A component task raised an exception that nothing handled."""
59
+
60
+
61
+ class RunEnd(StrEnum):
62
+ """Why `Scheduler.run` returned."""
63
+
64
+ UNTIL = "until"
65
+ IDLE = "idle"
66
+ DEADLINE = "deadline"
67
+
68
+
69
+ @dataclass(order=True)
70
+ class _Event:
71
+ at: int
72
+ seq: int
73
+ callback: Callable[[], None] = field(compare=False)
74
+ label: str = field(compare=False, default="")
75
+ cancelled: bool = field(compare=False, default=False)
76
+
77
+
78
+ class Timer:
79
+ """A scheduled event that can be cancelled before it fires."""
80
+
81
+ def __init__(self, event: _Event) -> None:
82
+ self._event = event
83
+
84
+ @property
85
+ def at(self) -> int:
86
+ """The virtual time it fires at."""
87
+ return self._event.at
88
+
89
+ @property
90
+ def label(self) -> str:
91
+ """What the event is for, for debugging."""
92
+ return self._event.label
93
+
94
+ @property
95
+ def cancelled(self) -> bool:
96
+ """Whether it was cancelled."""
97
+ return self._event.cancelled
98
+
99
+ def cancel(self) -> None:
100
+ """Keep the event from firing. Cancelling twice, or after it fired, does nothing."""
101
+ self._event.cancelled = True
102
+
103
+
104
+ class InlineExecutor(ThreadPoolExecutor):
105
+ """Runs submitted work at once, in the calling thread, and never starts a thread.
106
+
107
+ asyncio's `loop.run_in_executor(None, ...)` uses the loop's default executor, which
108
+ must be a `ThreadPoolExecutor` (CPython 3.12, `set_default_executor`). LangGraph runs
109
+ synchronous callables through it in asynchronous runs (langchain-core 1.8.0,
110
+ `run_in_executor`; langgraph 1.2.22, `pregel/_call.py`). A thread would finish at a
111
+ wall-clock moment the scheduler cannot see, so this runs the work inline instead.
112
+ """
113
+
114
+ def submit(self, fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> Future[T]:
115
+ future: Future[T] = Future()
116
+ try:
117
+ future.set_result(fn(*args, **kwargs))
118
+ except BaseException as error: # handed to the awaiting task, as a thread would
119
+ future.set_exception(error)
120
+ return future
121
+
122
+
123
+ def _resolve(future: asyncio.Future[None]) -> None:
124
+ if not future.done():
125
+ future.set_result(None)
126
+
127
+
128
+ class Scheduler:
129
+ """Runs a run's components one step at a time on a virtual clock."""
130
+
131
+ def __init__(self, clock: VirtualClock | None = None) -> None:
132
+ self.clock = clock if clock is not None else VirtualClock()
133
+ self._events: list[_Event] = []
134
+ self._next_seq = 0
135
+ self._next_task = 0
136
+ self._tasks: dict[str, dict[asyncio.Task[Any], int]] = {}
137
+ self._incarnations: dict[str, int] = {}
138
+ self._down: set[str] = set()
139
+ self._failures: list[tuple[str, BaseException]] = []
140
+ self._crash_listeners: list[Callable[[str], None]] = []
141
+ self._real = RealWaits()
142
+
143
+ @property
144
+ def now(self) -> int:
145
+ """The run's virtual time, in milliseconds."""
146
+ return self.clock.now
147
+
148
+ # Timed events --------------------------------------------------------------------------
149
+
150
+ def call_at(self, at: int, callback: Callable[[], None], *, label: str = "") -> Timer:
151
+ """Fire `callback` at virtual time `at`, after earlier and same-time events."""
152
+ if at < self.now:
153
+ raise ValueError(f"cannot schedule {label!r} at {at}, before now ({self.now})")
154
+ event = _Event(at, self._next_seq, callback, label)
155
+ self._next_seq += 1
156
+ heapq.heappush(self._events, event)
157
+ return Timer(event)
158
+
159
+ def call_later(self, delay: int, callback: Callable[[], None], *, label: str = "") -> Timer:
160
+ """Fire `callback` `delay` virtual milliseconds from now."""
161
+ if delay < 0:
162
+ raise ValueError(f"a delay cannot be negative: {delay}")
163
+ return self.call_at(self.now + delay, callback, label=label)
164
+
165
+ async def sleep(self, ms: int) -> None:
166
+ """Suspend the calling task for `ms` virtual milliseconds."""
167
+ if ms < 0:
168
+ raise ValueError(f"cannot sleep for a negative time: {ms}")
169
+ future: asyncio.Future[None] = asyncio.get_running_loop().create_future()
170
+ timer = self.call_later(ms, lambda: _resolve(future), label="sleep")
171
+ try:
172
+ await future
173
+ finally:
174
+ timer.cancel()
175
+
176
+ async def wait_for(self, awaitable: Awaitable[T], timeout_ms: int) -> T:
177
+ """Await `awaitable`; after `timeout_ms` virtual ms, cancel it and raise.
178
+
179
+ Raises `SchedulerTimeout` when the limit passes first.
180
+ """
181
+ if timeout_ms < 0:
182
+ raise ValueError(f"a timeout cannot be negative: {timeout_ms}")
183
+ inner = asyncio.ensure_future(awaitable)
184
+ expired = False
185
+
186
+ def expire() -> None:
187
+ nonlocal expired
188
+ if not inner.done():
189
+ expired = True
190
+ inner.cancel()
191
+
192
+ timer = self.call_later(timeout_ms, expire, label="timeout")
193
+ try:
194
+ return await inner
195
+ except asyncio.CancelledError:
196
+ if expired:
197
+ raise SchedulerTimeout(f"no result within {timeout_ms} virtual ms") from None
198
+ raise
199
+ finally:
200
+ timer.cancel()
201
+
202
+ async def hold(self, awaitable: Awaitable[T]) -> T:
203
+ """Await `awaitable`, a wait on the real world such as a live model call, with
204
+ virtual time held still.
205
+
206
+ Until it ends, `run` fires no event and waits for it in wall time, so the wait
207
+ takes no virtual time. `awaitable` must not wait on the scheduler: nothing the
208
+ scheduler controls can happen until it ends.
209
+ """
210
+ return await self._real.hold(awaitable)
211
+
212
+ # Components ----------------------------------------------------------------------------
213
+
214
+ def spawn(self, component: str, coro: Coroutine[Any, Any, T], *, name: str) -> asyncio.Task[T]:
215
+ """Run `coro` as a task of `component`. A crash of the component cancels it."""
216
+ if component in self._down:
217
+ coro.close()
218
+ raise RuntimeError(f"{component} is down")
219
+ self._next_task += 1
220
+ task = asyncio.get_running_loop().create_task(
221
+ coro, name=f"{component} {self._next_task}: {name}"
222
+ )
223
+ self._tasks.setdefault(component, {})[task] = self._next_task
224
+ task.add_done_callback(lambda done: self._task_finished(component, done))
225
+ return task
226
+
227
+ def _task_finished(self, component: str, task: asyncio.Task[Any]) -> None:
228
+ self._tasks.get(component, {}).pop(task, None)
229
+ if task.cancelled():
230
+ return
231
+ error = task.exception()
232
+ if error is not None:
233
+ self._failures.append((task.get_name(), error))
234
+
235
+ def is_up(self, component: str) -> bool:
236
+ """Whether `component` is running (not crashed and waiting for its restart)."""
237
+ return component not in self._down
238
+
239
+ def incarnation(self, component: str) -> int:
240
+ """How many times `component` has crashed."""
241
+ return self._incarnations.get(component, 0)
242
+
243
+ def on_crash(self, listener: Callable[[str], None]) -> None:
244
+ """Call `listener(component)` whenever a component crashes."""
245
+ self._crash_listeners.append(listener)
246
+
247
+ def crash(
248
+ self,
249
+ component: str,
250
+ *,
251
+ restart_after_ms: int | None = None,
252
+ restart: Callable[[], None] | None = None,
253
+ ) -> None:
254
+ """Stop `component` now, discarding its memory, and maybe restart it later (0006).
255
+
256
+ Its tasks are cancelled in the order they were spawned. `restart`, called when the
257
+ restart event fires, rebuilds the component, for example through startup
258
+ recovery (0003) or from the agent's last checkpoint. It is called only if this
259
+ crash is still the component's latest when the event fires.
260
+ """
261
+ if (restart_after_ms is None) != (restart is None):
262
+ raise ValueError("give both restart_after_ms and restart, or neither")
263
+ self._down.add(component)
264
+ incarnation = self.incarnation(component) + 1
265
+ self._incarnations[component] = incarnation
266
+ spawned = self._tasks.pop(component, {})
267
+ for task in sorted(spawned, key=spawned.__getitem__):
268
+ task.cancel()
269
+ for listener in self._crash_listeners:
270
+ listener(component)
271
+ if restart_after_ms is not None and restart is not None:
272
+ self.call_later(
273
+ restart_after_ms,
274
+ lambda: self._restart(component, incarnation, restart),
275
+ label=f"restart {component}",
276
+ )
277
+
278
+ def _restart(self, component: str, incarnation: int, restart: Callable[[], None]) -> None:
279
+ """Bring `component` back as `incarnation`, unless a later crash has started a
280
+ newer one: then that crash's own restart brings it back, and this one does
281
+ nothing."""
282
+ if self.incarnation(component) != incarnation:
283
+ return
284
+ self._down.discard(component)
285
+ restart()
286
+
287
+ # Running -------------------------------------------------------------------------------
288
+
289
+ async def run(
290
+ self, *, until: Callable[[], bool] | None = None, deadline_ms: int | None = None
291
+ ) -> RunEnd:
292
+ """Fire events in order until `until()` holds, nothing is left, or the deadline.
293
+
294
+ Raises `ComponentFailed` if a component task raised an exception that nothing
295
+ handled. At the deadline, the clock moves to it and later events stay queued.
296
+ """
297
+ loop = asyncio.get_running_loop()
298
+ if not isinstance(getattr(loop, "_default_executor", None), InlineExecutor):
299
+ loop.set_default_executor(InlineExecutor())
300
+ while True:
301
+ await settle(self._real)
302
+ self._raise_failures()
303
+ if until is not None and until():
304
+ return RunEnd.UNTIL
305
+ event = self._next_event()
306
+ if event is None:
307
+ return RunEnd.IDLE
308
+ if deadline_ms is not None and event.at > deadline_ms:
309
+ heapq.heappush(self._events, event)
310
+ self.clock.advance_to(max(self.now, deadline_ms))
311
+ return RunEnd.DEADLINE
312
+ self.clock.advance_to(event.at)
313
+ event.callback()
314
+
315
+ def pending(self) -> list[Timer]:
316
+ """The events still queued, in firing order."""
317
+ return [Timer(event) for event in sorted(self._events) if not event.cancelled]
318
+
319
+ def _next_event(self) -> _Event | None:
320
+ while self._events:
321
+ event = heapq.heappop(self._events)
322
+ if not event.cancelled:
323
+ return event
324
+ return None
325
+
326
+ def _raise_failures(self) -> None:
327
+ if self._failures:
328
+ name, error = self._failures[0]
329
+ raise ComponentFailed(f"task {name} failed: {error!r}") from error
330
+
331
+
332
+ class SchedulerStopped(RuntimeError):
333
+ """A timer was asked for after the split topology's scheduler stopped."""
334
+
335
+
336
+ class _HostTimer(Timer):
337
+ """A host timer: cancelling it also takes it off the event loop, and forgets it."""
338
+
339
+ def __init__(
340
+ self,
341
+ event: _Event,
342
+ handles: dict[int, asyncio.TimerHandle],
343
+ live: dict[int, _Event],
344
+ ) -> None:
345
+ super().__init__(event)
346
+ self._handles = handles
347
+ self._live = live
348
+
349
+ def cancel(self) -> None:
350
+ super().cancel()
351
+ self._live.pop(self._event.seq, None)
352
+ handle = self._handles.pop(self._event.seq, None)
353
+ if handle is not None:
354
+ handle.cancel()
355
+
356
+
357
+ class HostScheduler(Scheduler):
358
+ """The scheduler of the split topology, whose virtual time follows the host's clock.
359
+
360
+ Each event is a timer of the running event loop, which fires it in wall time once the
361
+ host's clock reaches its virtual time; an event due in the past fires at once, since
362
+ the host's clock moves on while a caller computes a time. Components spawn tasks and
363
+ crash as they do on the deterministic scheduler. A task or event that raises is logged
364
+ and kept in `failures`, since nothing runs the scheduler to raise it. `run` is refused:
365
+ the services' event loops run everything. `stop` ends the run's timers and tasks.
366
+ """
367
+
368
+ def __init__(self, clock: HostClock) -> None:
369
+ super().__init__(clock)
370
+ self._host = clock
371
+ self._handles: dict[int, asyncio.TimerHandle] = {}
372
+ self._live: dict[int, _Event] = {}
373
+ self._stopped = False
374
+
375
+ @property
376
+ def stopped(self) -> bool:
377
+ """Whether `stop` has run."""
378
+ return self._stopped
379
+
380
+ @property
381
+ def failures(self) -> list[tuple[str, BaseException]]:
382
+ """The tasks and events that raised, with their errors, oldest first."""
383
+ return list(self._failures)
384
+
385
+ def call_at(self, at: int, callback: Callable[[], None], *, label: str = "") -> Timer:
386
+ """Fire `callback` once the host's clock reaches virtual time `at`, or at once if it
387
+ has passed. Raises `SchedulerStopped` after `stop`."""
388
+ if self._stopped:
389
+ raise SchedulerStopped(f"cannot schedule {label!r}: the scheduler has stopped")
390
+ event = _Event(max(at, self.now), self._next_seq, callback, label)
391
+ self._next_seq += 1
392
+ self._live[event.seq] = event
393
+
394
+ def fire() -> None:
395
+ self._handles.pop(event.seq, None)
396
+ self._live.pop(event.seq, None)
397
+ if event.cancelled:
398
+ return
399
+ try:
400
+ callback()
401
+ except Exception as error:
402
+ self._failed(f"event {label or event.seq}", error)
403
+
404
+ self._handles[event.seq] = self._host.call_at(event.at, fire)
405
+ return _HostTimer(event, self._handles, self._live)
406
+
407
+ def pending(self) -> list[Timer]:
408
+ """The timers still due to fire, in firing order. A timer is forgotten once it
409
+ fires or is cancelled."""
410
+ return [
411
+ _HostTimer(event, self._handles, self._live) for event in sorted(self._live.values())
412
+ ]
413
+
414
+ async def run(
415
+ self, *, until: Callable[[], bool] | None = None, deadline_ms: int | None = None
416
+ ) -> RunEnd:
417
+ """Refused: the split topology's services run their own event loops."""
418
+ raise TypeError("a HostScheduler has no run loop: the event loop runs its components")
419
+
420
+ def stop(self) -> None:
421
+ """End the run here: cancel every timer, crash every component, refuse new timers."""
422
+ self._stopped = True
423
+ for handle in self._handles.values():
424
+ handle.cancel()
425
+ self._handles.clear()
426
+ self._live.clear()
427
+ for component in sorted(self._tasks):
428
+ if self.is_up(component):
429
+ self.crash(component)
430
+
431
+ def _task_finished(self, component: str, task: asyncio.Task[Any]) -> None:
432
+ self._tasks.get(component, {}).pop(task, None)
433
+ if task.cancelled():
434
+ return
435
+ error = task.exception()
436
+ if error is not None:
437
+ self._failed(task.get_name(), error)
438
+
439
+ def _failed(self, name: str, error: BaseException) -> None:
440
+ self._failures.append((name, error))
441
+ logger.error("%s failed", name, exc_info=error)
@@ -0,0 +1,215 @@
1
+ """Faults at the simulator's fault points: F5 and F10 (0005, 0006, 0013).
2
+
3
+ The simulator's endpoint (`fivefold.sim.endpoint`) asks a `SimulatorPoints` what fires
4
+ at its two points:
5
+
6
+ - `sim.write.received`, as a write request reaches the destination.
7
+ - F5 holds the write and commits it `delay_ms` later, through the simulator's two-phase
8
+ write. In contract, T < `delay_ms` < D, so the caller has timed out and the commit
9
+ still meets the deadline. Beyond the deadline, `delay_ms` > D and the deadline is
10
+ waived: a contract violation. The answer goes back when the write commits.
11
+ - F10 `stale_beyond_lag` freezes the state as it was just before this write arrived
12
+ (0013). Each of the next `count` status answers lists the effects committed before
13
+ the virtual millisecond in which the write arrived, the same in every answer, but
14
+ claims to be accurate to the time it is produced: a contract violation. So the write
15
+ is in none of them, even when it commits in that millisecond. The status lookup
16
+ counts time in whole milliseconds, so an effect committed earlier in that millisecond
17
+ is left out too.
18
+ - `sim.status.answer`, as a status answer is produced. F10 `delayed` and
19
+ `stale_within_lag` fire here, and shape this answer and the next ones, `count` in all:
20
+ - `delayed`: the answer is held for T (`dispatch_timeout_ms`) before it is sent, so it
21
+ arrives after the caller's timeout, and the probe counts as unanswered;
22
+ - `stale_within_lag`: the answer is accurate to L (`status_lag_ms`) before it is
23
+ produced, as stale as the destination's promise allows, and says so.
24
+
25
+ Every status answer counts toward each F10 that has fired, wherever it fired. When two
26
+ F10 entries overlap, every one counts the answer, and the one that fired first shapes it.
27
+
28
+ Each fault that fires gets a row in the fault log, naming the request it acted on, the
29
+ request's intent, and the simulator as the component. Its params hold the values it
30
+ used: `delay_ms` for F5, `count` for F10. F10 also gets one row for each status answer it
31
+ changes, whichever variant and wherever it fired, once that answer is sent
32
+ (`answer_sent`): the answer is the row's target, its intent the row's intent, and
33
+ `answer_no` counts the answers F10 changed, from 1 (0014).
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ from collections.abc import Mapping
39
+ from dataclasses import dataclass
40
+
41
+ from fivefold.faults.catalog import (
42
+ F5_BEYOND_DEADLINE,
43
+ F10_DELAYED,
44
+ F10_STALE_BEYOND_LAG,
45
+ F10_STALE_WITHIN_LAG,
46
+ Action,
47
+ )
48
+ from fivefold.faults.plan import FaultInjector, Fired
49
+ from fivefold.harness.log import FaultRecord, HarnessLog
50
+ from fivefold.harness.scheduler import Scheduler
51
+ from fivefold.harness.transport import Envelope, Stage
52
+ from fivefold.sim.endpoint import StatusShape, WriteDelay
53
+
54
+ SIMULATOR = "simulator"
55
+
56
+ WRITE_RECEIVED = "sim.write.received"
57
+ STATUS_ANSWER = "sim.status.answer"
58
+
59
+ ACTIONS: Mapping[str, frozenset[Action]] = {
60
+ WRITE_RECEIVED: frozenset({Action.DELAY_COMMIT, Action.DEGRADE_STATUS}),
61
+ STATUS_ANSWER: frozenset({Action.DEGRADE_STATUS}),
62
+ }
63
+ """What a fault may do at each simulator point: F5 holds a write, F10 shapes status."""
64
+
65
+
66
+ class SimulatorPointError(RuntimeError):
67
+ """A fault fired at a simulator point where it has nothing to do: a plan bug."""
68
+
69
+
70
+ @dataclass
71
+ class _Shaping:
72
+ """An F10 that fired, with the answers it still shapes."""
73
+
74
+ fired: Fired
75
+ left: int
76
+ fired_vt: int
77
+ changed: int = 0
78
+ """The status answers it has changed so far."""
79
+
80
+
81
+ class SimulatorPoints:
82
+ """Fires F5 and F10 at the simulator's points, and logs them (0006)."""
83
+
84
+ def __init__(
85
+ self,
86
+ scheduler: Scheduler,
87
+ faults: FaultInjector,
88
+ log: HarnessLog,
89
+ *,
90
+ dispatch_timeout_ms: int,
91
+ status_lag_ms: int,
92
+ ) -> None:
93
+ self._scheduler = scheduler
94
+ self._faults = faults
95
+ self._log = log
96
+ self._t_ms = dispatch_timeout_ms
97
+ self._lag_ms = status_lag_ms
98
+ self._shaping: list[_Shaping] = []
99
+ self._changed: dict[int, tuple[_Shaping, int]] = {}
100
+ """The status requests whose answers F10 changed and has not logged yet: the F10
101
+ that shapes each answer, and the answer's number among those it changed."""
102
+
103
+ def write_received(self, request: Envelope) -> WriteDelay | None:
104
+ """`sim.write.received`: F5 holds the write and commits it `delay_ms` later, and
105
+ F10 `stale_beyond_lag` freezes the state the next `count` status answers list.
106
+
107
+ A write is held once. Two F5 entries that fire at the same write are a plan bug:
108
+ the second would be recorded and take no effect, so the run stops instead.
109
+ """
110
+ fired = self._faults.at(WRITE_RECEIVED, _fields(request))
111
+ for one in fired:
112
+ _check(one, WRITE_RECEIVED)
113
+ holds = [one for one in fired if one.planned.kind.action is Action.DELAY_COMMIT]
114
+ if len(holds) > 1:
115
+ raise SimulatorPointError(
116
+ f"{len(holds)} F5 entries fired at one write (message {request.seq});"
117
+ " a write is held once"
118
+ )
119
+ delay: WriteDelay | None = None
120
+ for one in fired:
121
+ if one.planned.kind.action is Action.DEGRADE_STATUS:
122
+ self._start_shaping(one, WRITE_RECEIVED, request)
123
+ continue
124
+ delay_ms = one.planned.param("delay_ms")
125
+ self._record(one, WRITE_RECEIVED, request, {"delay_ms": delay_ms})
126
+ delay = WriteDelay(delay_ms, waive_deadline=one.planned.variant == F5_BEYOND_DEADLINE)
127
+ return delay
128
+
129
+ def status_answer(self, request: Envelope) -> StatusShape | None:
130
+ """`sim.status.answer`: F10 `delayed` and `stale_within_lag` fire here. Every F10
131
+ that has fired shapes this answer and the next, `count` in all."""
132
+ now = self._scheduler.now
133
+ fired = self._faults.at(STATUS_ANSWER, _fields(request))
134
+ for one in fired:
135
+ _check(one, STATUS_ANSWER)
136
+ for one in fired:
137
+ self._start_shaping(one, STATUS_ANSWER, request)
138
+ if not self._shaping:
139
+ return None
140
+ first = self._shaping[0]
141
+ first.changed += 1
142
+ self._changed[request.seq] = (first, first.changed)
143
+ for shaping in self._shaping:
144
+ shaping.left -= 1
145
+ self._shaping = [shaping for shaping in self._shaping if shaping.left > 0]
146
+ variant = first.fired.planned.variant
147
+ if variant == F10_DELAYED:
148
+ return StatusShape(hold_ms=self._t_ms)
149
+ if variant == F10_STALE_WITHIN_LAG:
150
+ return StatusShape(as_of_vt=max(0, now - self._lag_ms))
151
+ if variant == F10_STALE_BEYOND_LAG:
152
+ # The simulator lists effects committed at or before `reflect_vt`. The write F10
153
+ # fired at commits in the millisecond it arrived in at the earliest, so freezing
154
+ # at the millisecond before leaves it out of every shaped answer.
155
+ frozen_vt = max(0, first.fired_vt - 1)
156
+ return StatusShape(reflect_vt=frozen_vt, contract_violation=True)
157
+ raise SimulatorPointError(f"F10 has no variant {variant!r}")
158
+
159
+ def answer_sent(self, envelope: Envelope, stage: Stage) -> None:
160
+ """A transport observer: once a status answer F10 changed is sent, log one more
161
+ row for that F10, naming the answer and its intent (0014). F10 acts on the next
162
+ `count` status answers, which can concern several intents, so each answer it
163
+ changes touches the answer's intent."""
164
+ if stage != "sent" or envelope.kind != "answer" or envelope.correlation is None:
165
+ return
166
+ changed = self._changed.pop(envelope.correlation, None)
167
+ if changed is None:
168
+ return
169
+ shaping, number = changed
170
+ count = shaping.fired.planned.param("count")
171
+ self._record(shaping.fired, STATUS_ANSWER, envelope, {"count": count, "answer_no": number})
172
+
173
+ def _start_shaping(self, fired: Fired, point: str, request: Envelope) -> None:
174
+ """Log an F10 that fired, and shape the next `count` status answers with it."""
175
+ count = fired.planned.param("count")
176
+ self._record(fired, point, request, {"count": count})
177
+ self._shaping.append(_Shaping(fired, count, self._scheduler.now))
178
+
179
+ def _record(
180
+ self, fired: Fired, point: str, message: Envelope, used: Mapping[str, object]
181
+ ) -> None:
182
+ self._log.fault_fired(
183
+ FaultRecord(
184
+ seq=self._log.next_seq(),
185
+ fault=str(fired.fault),
186
+ variant=fired.planned.variant,
187
+ point=point,
188
+ occurrence=fired.occurrence,
189
+ action=str(fired.planned.kind.action),
190
+ vt=self._scheduler.now,
191
+ target_seq=message.seq,
192
+ component=SIMULATOR,
193
+ params={**fired.planned.params, **used},
194
+ intent_id=message.intent_id,
195
+ )
196
+ )
197
+
198
+
199
+ def _check(fired: Fired, point: str) -> None:
200
+ """Refuse a fault with nothing to do at `point`, before anything is logged.
201
+
202
+ The catalog already refuses such a plan. Without this check, the fault would be
203
+ logged as fired and change nothing, or, for an F10 variant away from its own point,
204
+ shape answers its variant does not describe (0013).
205
+ """
206
+ planned = fired.planned
207
+ points = None if planned.variant is None else planned.kind.variants.get(planned.variant)
208
+ if planned.kind.action not in ACTIONS[point] or (points is not None and point not in points):
209
+ name = " ".join(part for part in (str(fired.fault), planned.variant) if part)
210
+ raise SimulatorPointError(f"{name} cannot act at {point}")
211
+
212
+
213
+ def _fields(request: Envelope) -> dict[str, object]:
214
+ """What a plan's match can name at a simulator point: the call and its intent."""
215
+ return {"api": request.api, "tool": request.api, "intent_id": request.intent_id}
@@ -0,0 +1,34 @@
1
+ """The split topology: the simulator and the gateway as HTTP services, for people who bring
2
+ their own agent (0009).
3
+
4
+ Record 0009's topology table: "split | Bringing your own agent: the gateway (MCP over
5
+ streamable HTTP), the simulator and Postgres run as separate Compose services |
6
+ Deterministic: No; faults still fire on counters". What runs where:
7
+
8
+ - The simulator's service (`simulator`) serves the simulator's API over HTTP: a POST to
9
+ `/api/{call}` for each call of `fivefold.domain.api`, with the caller's credential as a
10
+ bearer token. It runs the scripted approver and the faults that change the simulator's
11
+ state, and, for a baseline's agent, the harness's report sink.
12
+ - The gateway's service (`gateway`) serves the gateway's tools over MCP's streamable HTTP
13
+ transport at `/mcp`, and reaches the simulator only through the simulator's API, with
14
+ its own credential (0008).
15
+ - The run's lifecycle (`lifecycle`) starts a run, ends it after the drain window, and
16
+ scores it with the evaluator. `python -m fivefold.harness.split` runs it, and serves
17
+ either service.
18
+
19
+ Each service runs on a `HostScheduler`, whose virtual time follows the host's monotonic
20
+ clock (0009). Every component reads time from it, so the gateway's, the simulator's and
21
+ the fault points' code runs as it does in the inproc topology. Each service writes the
22
+ harness's records of the traffic it sees (`hops`), and fires the plan's faults at the
23
+ points it owns, on the plan's counters (`faults`). Nothing here is deterministic.
24
+ """
25
+
26
+ SERVICE_STOPPED = -32019
27
+ """The JSON-RPC error code the gateway answers an agent's MCP request with once the run's
28
+ agents' phase is over, or the run has stopped.
29
+
30
+ It is an implementation-defined server error: JSON-RPC leaves -32000 to -32099 to servers,
31
+ and MCP keeps -32020 to -32099 for its own codes (mcp 2.3.0, `mcp_types/jsonrpc.py`). The
32
+ SDK uses -32000 and -32001 for its own ends of a call; -32000, `CONNECTION_CLOSED`, is how
33
+ a client reads a closed connection, so it cannot also mean that the run is over. The SDK
34
+ uses -32019 nowhere (checked in mcp 2.3.0 and mcp-types 2.3.0, and by a test)."""