fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
"""The deterministic scheduler for the inproc topology (0009).
|
|
2
|
+
|
|
3
|
+
One process, one thread, one asyncio event loop. The scheduler owns the run's virtual
|
|
4
|
+
clock and a queue of timed events, ordered by (time, enqueue sequence). It repeats one
|
|
5
|
+
step: let every task run until each waits on the scheduler (`settle`), then take the
|
|
6
|
+
next event, move the clock to it and fire it. Message deliveries, timeouts, probes,
|
|
7
|
+
polls, approver delays and restart delays are all such events, so nothing depends on
|
|
8
|
+
wall-clock time, and a run full of long timeouts takes milliseconds.
|
|
9
|
+
|
|
10
|
+
Components (the agent workers, the gateway, the simulator and the harness's scripted
|
|
11
|
+
humans) run their asynchronous work as tasks spawned through the scheduler, under the
|
|
12
|
+
component's name. Libraries that hand synchronous work to the event loop's default
|
|
13
|
+
executor, as LangChain and LangGraph do for synchronous callables, run it at once in the
|
|
14
|
+
loop's thread instead (`InlineExecutor`), so a run never has a second thread.
|
|
15
|
+
|
|
16
|
+
Crashing a component cancels its tasks and discards their memory; it stays down until
|
|
17
|
+
its restart event fires (0006). Each crash starts a new incarnation, so the transport can
|
|
18
|
+
drop answers to requests sent before the crash (0010, change 3). A restart belongs to the
|
|
19
|
+
incarnation its crash started. When the component crashes again before that restart
|
|
20
|
+
fires, the earlier restart does nothing, so a component never runs as two copies at
|
|
21
|
+
once.
|
|
22
|
+
|
|
23
|
+
A component that waits on the real world, as the live model adapter waits on its
|
|
24
|
+
provider, awaits it through `hold`. Until the wait ends, the scheduler fires no event: it
|
|
25
|
+
waits in wall time, with virtual time held still. So every timer fires at the virtual time
|
|
26
|
+
it would have without the wait, and a model call takes exactly its `think_ms` (0009).
|
|
27
|
+
|
|
28
|
+
`HostScheduler` is the split topology's scheduler. It has the same interface, so the
|
|
29
|
+
gateway, the simulator and the fault points run on it unchanged, but its virtual time
|
|
30
|
+
follows the host's monotonic clock (`HostClock`, 0009): its events are the event loop's
|
|
31
|
+
timers, its components' tasks run as the event loop schedules them, and nothing it does
|
|
32
|
+
is deterministic.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import asyncio
|
|
38
|
+
import heapq
|
|
39
|
+
import logging
|
|
40
|
+
from collections.abc import Awaitable, Callable, Coroutine
|
|
41
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
42
|
+
from dataclasses import dataclass, field
|
|
43
|
+
from enum import StrEnum
|
|
44
|
+
from typing import Any, TypeVar
|
|
45
|
+
|
|
46
|
+
from fivefold.harness.clock import HostClock, RealWaits, VirtualClock, settle
|
|
47
|
+
|
|
48
|
+
T = TypeVar("T")
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger(__name__)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class SchedulerTimeout(Exception):
|
|
54
|
+
"""A wait ended because its virtual-time limit passed."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ComponentFailed(RuntimeError):
|
|
58
|
+
"""A component task raised an exception that nothing handled."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class RunEnd(StrEnum):
|
|
62
|
+
"""Why `Scheduler.run` returned."""
|
|
63
|
+
|
|
64
|
+
UNTIL = "until"
|
|
65
|
+
IDLE = "idle"
|
|
66
|
+
DEADLINE = "deadline"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(order=True)
|
|
70
|
+
class _Event:
|
|
71
|
+
at: int
|
|
72
|
+
seq: int
|
|
73
|
+
callback: Callable[[], None] = field(compare=False)
|
|
74
|
+
label: str = field(compare=False, default="")
|
|
75
|
+
cancelled: bool = field(compare=False, default=False)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Timer:
|
|
79
|
+
"""A scheduled event that can be cancelled before it fires."""
|
|
80
|
+
|
|
81
|
+
def __init__(self, event: _Event) -> None:
|
|
82
|
+
self._event = event
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def at(self) -> int:
|
|
86
|
+
"""The virtual time it fires at."""
|
|
87
|
+
return self._event.at
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def label(self) -> str:
|
|
91
|
+
"""What the event is for, for debugging."""
|
|
92
|
+
return self._event.label
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def cancelled(self) -> bool:
|
|
96
|
+
"""Whether it was cancelled."""
|
|
97
|
+
return self._event.cancelled
|
|
98
|
+
|
|
99
|
+
def cancel(self) -> None:
|
|
100
|
+
"""Keep the event from firing. Cancelling twice, or after it fired, does nothing."""
|
|
101
|
+
self._event.cancelled = True
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class InlineExecutor(ThreadPoolExecutor):
|
|
105
|
+
"""Runs submitted work at once, in the calling thread, and never starts a thread.
|
|
106
|
+
|
|
107
|
+
asyncio's `loop.run_in_executor(None, ...)` uses the loop's default executor, which
|
|
108
|
+
must be a `ThreadPoolExecutor` (CPython 3.12, `set_default_executor`). LangGraph runs
|
|
109
|
+
synchronous callables through it in asynchronous runs (langchain-core 1.8.0,
|
|
110
|
+
`run_in_executor`; langgraph 1.2.22, `pregel/_call.py`). A thread would finish at a
|
|
111
|
+
wall-clock moment the scheduler cannot see, so this runs the work inline instead.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def submit(self, fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> Future[T]:
|
|
115
|
+
future: Future[T] = Future()
|
|
116
|
+
try:
|
|
117
|
+
future.set_result(fn(*args, **kwargs))
|
|
118
|
+
except BaseException as error: # handed to the awaiting task, as a thread would
|
|
119
|
+
future.set_exception(error)
|
|
120
|
+
return future
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _resolve(future: asyncio.Future[None]) -> None:
|
|
124
|
+
if not future.done():
|
|
125
|
+
future.set_result(None)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class Scheduler:
|
|
129
|
+
"""Runs a run's components one step at a time on a virtual clock."""
|
|
130
|
+
|
|
131
|
+
def __init__(self, clock: VirtualClock | None = None) -> None:
|
|
132
|
+
self.clock = clock if clock is not None else VirtualClock()
|
|
133
|
+
self._events: list[_Event] = []
|
|
134
|
+
self._next_seq = 0
|
|
135
|
+
self._next_task = 0
|
|
136
|
+
self._tasks: dict[str, dict[asyncio.Task[Any], int]] = {}
|
|
137
|
+
self._incarnations: dict[str, int] = {}
|
|
138
|
+
self._down: set[str] = set()
|
|
139
|
+
self._failures: list[tuple[str, BaseException]] = []
|
|
140
|
+
self._crash_listeners: list[Callable[[str], None]] = []
|
|
141
|
+
self._real = RealWaits()
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def now(self) -> int:
|
|
145
|
+
"""The run's virtual time, in milliseconds."""
|
|
146
|
+
return self.clock.now
|
|
147
|
+
|
|
148
|
+
# Timed events --------------------------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
def call_at(self, at: int, callback: Callable[[], None], *, label: str = "") -> Timer:
|
|
151
|
+
"""Fire `callback` at virtual time `at`, after earlier and same-time events."""
|
|
152
|
+
if at < self.now:
|
|
153
|
+
raise ValueError(f"cannot schedule {label!r} at {at}, before now ({self.now})")
|
|
154
|
+
event = _Event(at, self._next_seq, callback, label)
|
|
155
|
+
self._next_seq += 1
|
|
156
|
+
heapq.heappush(self._events, event)
|
|
157
|
+
return Timer(event)
|
|
158
|
+
|
|
159
|
+
def call_later(self, delay: int, callback: Callable[[], None], *, label: str = "") -> Timer:
|
|
160
|
+
"""Fire `callback` `delay` virtual milliseconds from now."""
|
|
161
|
+
if delay < 0:
|
|
162
|
+
raise ValueError(f"a delay cannot be negative: {delay}")
|
|
163
|
+
return self.call_at(self.now + delay, callback, label=label)
|
|
164
|
+
|
|
165
|
+
async def sleep(self, ms: int) -> None:
|
|
166
|
+
"""Suspend the calling task for `ms` virtual milliseconds."""
|
|
167
|
+
if ms < 0:
|
|
168
|
+
raise ValueError(f"cannot sleep for a negative time: {ms}")
|
|
169
|
+
future: asyncio.Future[None] = asyncio.get_running_loop().create_future()
|
|
170
|
+
timer = self.call_later(ms, lambda: _resolve(future), label="sleep")
|
|
171
|
+
try:
|
|
172
|
+
await future
|
|
173
|
+
finally:
|
|
174
|
+
timer.cancel()
|
|
175
|
+
|
|
176
|
+
async def wait_for(self, awaitable: Awaitable[T], timeout_ms: int) -> T:
|
|
177
|
+
"""Await `awaitable`; after `timeout_ms` virtual ms, cancel it and raise.
|
|
178
|
+
|
|
179
|
+
Raises `SchedulerTimeout` when the limit passes first.
|
|
180
|
+
"""
|
|
181
|
+
if timeout_ms < 0:
|
|
182
|
+
raise ValueError(f"a timeout cannot be negative: {timeout_ms}")
|
|
183
|
+
inner = asyncio.ensure_future(awaitable)
|
|
184
|
+
expired = False
|
|
185
|
+
|
|
186
|
+
def expire() -> None:
|
|
187
|
+
nonlocal expired
|
|
188
|
+
if not inner.done():
|
|
189
|
+
expired = True
|
|
190
|
+
inner.cancel()
|
|
191
|
+
|
|
192
|
+
timer = self.call_later(timeout_ms, expire, label="timeout")
|
|
193
|
+
try:
|
|
194
|
+
return await inner
|
|
195
|
+
except asyncio.CancelledError:
|
|
196
|
+
if expired:
|
|
197
|
+
raise SchedulerTimeout(f"no result within {timeout_ms} virtual ms") from None
|
|
198
|
+
raise
|
|
199
|
+
finally:
|
|
200
|
+
timer.cancel()
|
|
201
|
+
|
|
202
|
+
async def hold(self, awaitable: Awaitable[T]) -> T:
|
|
203
|
+
"""Await `awaitable`, a wait on the real world such as a live model call, with
|
|
204
|
+
virtual time held still.
|
|
205
|
+
|
|
206
|
+
Until it ends, `run` fires no event and waits for it in wall time, so the wait
|
|
207
|
+
takes no virtual time. `awaitable` must not wait on the scheduler: nothing the
|
|
208
|
+
scheduler controls can happen until it ends.
|
|
209
|
+
"""
|
|
210
|
+
return await self._real.hold(awaitable)
|
|
211
|
+
|
|
212
|
+
# Components ----------------------------------------------------------------------------
|
|
213
|
+
|
|
214
|
+
def spawn(self, component: str, coro: Coroutine[Any, Any, T], *, name: str) -> asyncio.Task[T]:
|
|
215
|
+
"""Run `coro` as a task of `component`. A crash of the component cancels it."""
|
|
216
|
+
if component in self._down:
|
|
217
|
+
coro.close()
|
|
218
|
+
raise RuntimeError(f"{component} is down")
|
|
219
|
+
self._next_task += 1
|
|
220
|
+
task = asyncio.get_running_loop().create_task(
|
|
221
|
+
coro, name=f"{component} {self._next_task}: {name}"
|
|
222
|
+
)
|
|
223
|
+
self._tasks.setdefault(component, {})[task] = self._next_task
|
|
224
|
+
task.add_done_callback(lambda done: self._task_finished(component, done))
|
|
225
|
+
return task
|
|
226
|
+
|
|
227
|
+
def _task_finished(self, component: str, task: asyncio.Task[Any]) -> None:
|
|
228
|
+
self._tasks.get(component, {}).pop(task, None)
|
|
229
|
+
if task.cancelled():
|
|
230
|
+
return
|
|
231
|
+
error = task.exception()
|
|
232
|
+
if error is not None:
|
|
233
|
+
self._failures.append((task.get_name(), error))
|
|
234
|
+
|
|
235
|
+
def is_up(self, component: str) -> bool:
|
|
236
|
+
"""Whether `component` is running (not crashed and waiting for its restart)."""
|
|
237
|
+
return component not in self._down
|
|
238
|
+
|
|
239
|
+
def incarnation(self, component: str) -> int:
|
|
240
|
+
"""How many times `component` has crashed."""
|
|
241
|
+
return self._incarnations.get(component, 0)
|
|
242
|
+
|
|
243
|
+
def on_crash(self, listener: Callable[[str], None]) -> None:
|
|
244
|
+
"""Call `listener(component)` whenever a component crashes."""
|
|
245
|
+
self._crash_listeners.append(listener)
|
|
246
|
+
|
|
247
|
+
def crash(
|
|
248
|
+
self,
|
|
249
|
+
component: str,
|
|
250
|
+
*,
|
|
251
|
+
restart_after_ms: int | None = None,
|
|
252
|
+
restart: Callable[[], None] | None = None,
|
|
253
|
+
) -> None:
|
|
254
|
+
"""Stop `component` now, discarding its memory, and maybe restart it later (0006).
|
|
255
|
+
|
|
256
|
+
Its tasks are cancelled in the order they were spawned. `restart`, called when the
|
|
257
|
+
restart event fires, rebuilds the component, for example through startup
|
|
258
|
+
recovery (0003) or from the agent's last checkpoint. It is called only if this
|
|
259
|
+
crash is still the component's latest when the event fires.
|
|
260
|
+
"""
|
|
261
|
+
if (restart_after_ms is None) != (restart is None):
|
|
262
|
+
raise ValueError("give both restart_after_ms and restart, or neither")
|
|
263
|
+
self._down.add(component)
|
|
264
|
+
incarnation = self.incarnation(component) + 1
|
|
265
|
+
self._incarnations[component] = incarnation
|
|
266
|
+
spawned = self._tasks.pop(component, {})
|
|
267
|
+
for task in sorted(spawned, key=spawned.__getitem__):
|
|
268
|
+
task.cancel()
|
|
269
|
+
for listener in self._crash_listeners:
|
|
270
|
+
listener(component)
|
|
271
|
+
if restart_after_ms is not None and restart is not None:
|
|
272
|
+
self.call_later(
|
|
273
|
+
restart_after_ms,
|
|
274
|
+
lambda: self._restart(component, incarnation, restart),
|
|
275
|
+
label=f"restart {component}",
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
def _restart(self, component: str, incarnation: int, restart: Callable[[], None]) -> None:
|
|
279
|
+
"""Bring `component` back as `incarnation`, unless a later crash has started a
|
|
280
|
+
newer one: then that crash's own restart brings it back, and this one does
|
|
281
|
+
nothing."""
|
|
282
|
+
if self.incarnation(component) != incarnation:
|
|
283
|
+
return
|
|
284
|
+
self._down.discard(component)
|
|
285
|
+
restart()
|
|
286
|
+
|
|
287
|
+
# Running -------------------------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
async def run(
|
|
290
|
+
self, *, until: Callable[[], bool] | None = None, deadline_ms: int | None = None
|
|
291
|
+
) -> RunEnd:
|
|
292
|
+
"""Fire events in order until `until()` holds, nothing is left, or the deadline.
|
|
293
|
+
|
|
294
|
+
Raises `ComponentFailed` if a component task raised an exception that nothing
|
|
295
|
+
handled. At the deadline, the clock moves to it and later events stay queued.
|
|
296
|
+
"""
|
|
297
|
+
loop = asyncio.get_running_loop()
|
|
298
|
+
if not isinstance(getattr(loop, "_default_executor", None), InlineExecutor):
|
|
299
|
+
loop.set_default_executor(InlineExecutor())
|
|
300
|
+
while True:
|
|
301
|
+
await settle(self._real)
|
|
302
|
+
self._raise_failures()
|
|
303
|
+
if until is not None and until():
|
|
304
|
+
return RunEnd.UNTIL
|
|
305
|
+
event = self._next_event()
|
|
306
|
+
if event is None:
|
|
307
|
+
return RunEnd.IDLE
|
|
308
|
+
if deadline_ms is not None and event.at > deadline_ms:
|
|
309
|
+
heapq.heappush(self._events, event)
|
|
310
|
+
self.clock.advance_to(max(self.now, deadline_ms))
|
|
311
|
+
return RunEnd.DEADLINE
|
|
312
|
+
self.clock.advance_to(event.at)
|
|
313
|
+
event.callback()
|
|
314
|
+
|
|
315
|
+
def pending(self) -> list[Timer]:
|
|
316
|
+
"""The events still queued, in firing order."""
|
|
317
|
+
return [Timer(event) for event in sorted(self._events) if not event.cancelled]
|
|
318
|
+
|
|
319
|
+
def _next_event(self) -> _Event | None:
|
|
320
|
+
while self._events:
|
|
321
|
+
event = heapq.heappop(self._events)
|
|
322
|
+
if not event.cancelled:
|
|
323
|
+
return event
|
|
324
|
+
return None
|
|
325
|
+
|
|
326
|
+
def _raise_failures(self) -> None:
|
|
327
|
+
if self._failures:
|
|
328
|
+
name, error = self._failures[0]
|
|
329
|
+
raise ComponentFailed(f"task {name} failed: {error!r}") from error
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class SchedulerStopped(RuntimeError):
|
|
333
|
+
"""A timer was asked for after the split topology's scheduler stopped."""
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
class _HostTimer(Timer):
|
|
337
|
+
"""A host timer: cancelling it also takes it off the event loop, and forgets it."""
|
|
338
|
+
|
|
339
|
+
def __init__(
|
|
340
|
+
self,
|
|
341
|
+
event: _Event,
|
|
342
|
+
handles: dict[int, asyncio.TimerHandle],
|
|
343
|
+
live: dict[int, _Event],
|
|
344
|
+
) -> None:
|
|
345
|
+
super().__init__(event)
|
|
346
|
+
self._handles = handles
|
|
347
|
+
self._live = live
|
|
348
|
+
|
|
349
|
+
def cancel(self) -> None:
|
|
350
|
+
super().cancel()
|
|
351
|
+
self._live.pop(self._event.seq, None)
|
|
352
|
+
handle = self._handles.pop(self._event.seq, None)
|
|
353
|
+
if handle is not None:
|
|
354
|
+
handle.cancel()
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
class HostScheduler(Scheduler):
|
|
358
|
+
"""The scheduler of the split topology, whose virtual time follows the host's clock.
|
|
359
|
+
|
|
360
|
+
Each event is a timer of the running event loop, which fires it in wall time once the
|
|
361
|
+
host's clock reaches its virtual time; an event due in the past fires at once, since
|
|
362
|
+
the host's clock moves on while a caller computes a time. Components spawn tasks and
|
|
363
|
+
crash as they do on the deterministic scheduler. A task or event that raises is logged
|
|
364
|
+
and kept in `failures`, since nothing runs the scheduler to raise it. `run` is refused:
|
|
365
|
+
the services' event loops run everything. `stop` ends the run's timers and tasks.
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
def __init__(self, clock: HostClock) -> None:
|
|
369
|
+
super().__init__(clock)
|
|
370
|
+
self._host = clock
|
|
371
|
+
self._handles: dict[int, asyncio.TimerHandle] = {}
|
|
372
|
+
self._live: dict[int, _Event] = {}
|
|
373
|
+
self._stopped = False
|
|
374
|
+
|
|
375
|
+
@property
|
|
376
|
+
def stopped(self) -> bool:
|
|
377
|
+
"""Whether `stop` has run."""
|
|
378
|
+
return self._stopped
|
|
379
|
+
|
|
380
|
+
@property
|
|
381
|
+
def failures(self) -> list[tuple[str, BaseException]]:
|
|
382
|
+
"""The tasks and events that raised, with their errors, oldest first."""
|
|
383
|
+
return list(self._failures)
|
|
384
|
+
|
|
385
|
+
def call_at(self, at: int, callback: Callable[[], None], *, label: str = "") -> Timer:
|
|
386
|
+
"""Fire `callback` once the host's clock reaches virtual time `at`, or at once if it
|
|
387
|
+
has passed. Raises `SchedulerStopped` after `stop`."""
|
|
388
|
+
if self._stopped:
|
|
389
|
+
raise SchedulerStopped(f"cannot schedule {label!r}: the scheduler has stopped")
|
|
390
|
+
event = _Event(max(at, self.now), self._next_seq, callback, label)
|
|
391
|
+
self._next_seq += 1
|
|
392
|
+
self._live[event.seq] = event
|
|
393
|
+
|
|
394
|
+
def fire() -> None:
|
|
395
|
+
self._handles.pop(event.seq, None)
|
|
396
|
+
self._live.pop(event.seq, None)
|
|
397
|
+
if event.cancelled:
|
|
398
|
+
return
|
|
399
|
+
try:
|
|
400
|
+
callback()
|
|
401
|
+
except Exception as error:
|
|
402
|
+
self._failed(f"event {label or event.seq}", error)
|
|
403
|
+
|
|
404
|
+
self._handles[event.seq] = self._host.call_at(event.at, fire)
|
|
405
|
+
return _HostTimer(event, self._handles, self._live)
|
|
406
|
+
|
|
407
|
+
def pending(self) -> list[Timer]:
|
|
408
|
+
"""The timers still due to fire, in firing order. A timer is forgotten once it
|
|
409
|
+
fires or is cancelled."""
|
|
410
|
+
return [
|
|
411
|
+
_HostTimer(event, self._handles, self._live) for event in sorted(self._live.values())
|
|
412
|
+
]
|
|
413
|
+
|
|
414
|
+
async def run(
|
|
415
|
+
self, *, until: Callable[[], bool] | None = None, deadline_ms: int | None = None
|
|
416
|
+
) -> RunEnd:
|
|
417
|
+
"""Refused: the split topology's services run their own event loops."""
|
|
418
|
+
raise TypeError("a HostScheduler has no run loop: the event loop runs its components")
|
|
419
|
+
|
|
420
|
+
def stop(self) -> None:
|
|
421
|
+
"""End the run here: cancel every timer, crash every component, refuse new timers."""
|
|
422
|
+
self._stopped = True
|
|
423
|
+
for handle in self._handles.values():
|
|
424
|
+
handle.cancel()
|
|
425
|
+
self._handles.clear()
|
|
426
|
+
self._live.clear()
|
|
427
|
+
for component in sorted(self._tasks):
|
|
428
|
+
if self.is_up(component):
|
|
429
|
+
self.crash(component)
|
|
430
|
+
|
|
431
|
+
def _task_finished(self, component: str, task: asyncio.Task[Any]) -> None:
|
|
432
|
+
self._tasks.get(component, {}).pop(task, None)
|
|
433
|
+
if task.cancelled():
|
|
434
|
+
return
|
|
435
|
+
error = task.exception()
|
|
436
|
+
if error is not None:
|
|
437
|
+
self._failed(task.get_name(), error)
|
|
438
|
+
|
|
439
|
+
def _failed(self, name: str, error: BaseException) -> None:
|
|
440
|
+
self._failures.append((name, error))
|
|
441
|
+
logger.error("%s failed", name, exc_info=error)
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Faults at the simulator's fault points: F5 and F10 (0005, 0006, 0013).
|
|
2
|
+
|
|
3
|
+
The simulator's endpoint (`fivefold.sim.endpoint`) asks a `SimulatorPoints` what fires
|
|
4
|
+
at its two points:
|
|
5
|
+
|
|
6
|
+
- `sim.write.received`, as a write request reaches the destination.
|
|
7
|
+
- F5 holds the write and commits it `delay_ms` later, through the simulator's two-phase
|
|
8
|
+
write. In contract, T < `delay_ms` < D, so the caller has timed out and the commit
|
|
9
|
+
still meets the deadline. Beyond the deadline, `delay_ms` > D and the deadline is
|
|
10
|
+
waived: a contract violation. The answer goes back when the write commits.
|
|
11
|
+
- F10 `stale_beyond_lag` freezes the state as it was just before this write arrived
|
|
12
|
+
(0013). Each of the next `count` status answers lists the effects committed before
|
|
13
|
+
the virtual millisecond in which the write arrived, the same in every answer, but
|
|
14
|
+
claims to be accurate to the time it is produced: a contract violation. So the write
|
|
15
|
+
is in none of them, even when it commits in that millisecond. The status lookup
|
|
16
|
+
counts time in whole milliseconds, so an effect committed earlier in that millisecond
|
|
17
|
+
is left out too.
|
|
18
|
+
- `sim.status.answer`, as a status answer is produced. F10 `delayed` and
|
|
19
|
+
`stale_within_lag` fire here, and shape this answer and the next ones, `count` in all:
|
|
20
|
+
- `delayed`: the answer is held for T (`dispatch_timeout_ms`) before it is sent, so it
|
|
21
|
+
arrives after the caller's timeout, and the probe counts as unanswered;
|
|
22
|
+
- `stale_within_lag`: the answer is accurate to L (`status_lag_ms`) before it is
|
|
23
|
+
produced, as stale as the destination's promise allows, and says so.
|
|
24
|
+
|
|
25
|
+
Every status answer counts toward each F10 that has fired, wherever it fired. When two
|
|
26
|
+
F10 entries overlap, every one counts the answer, and the one that fired first shapes it.
|
|
27
|
+
|
|
28
|
+
Each fault that fires gets a row in the fault log, naming the request it acted on, the
|
|
29
|
+
request's intent, and the simulator as the component. Its params hold the values it
|
|
30
|
+
used: `delay_ms` for F5, `count` for F10. F10 also gets one row for each status answer it
|
|
31
|
+
changes, whichever variant and wherever it fired, once that answer is sent
|
|
32
|
+
(`answer_sent`): the answer is the row's target, its intent the row's intent, and
|
|
33
|
+
`answer_no` counts the answers F10 changed, from 1 (0014).
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from collections.abc import Mapping
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
|
|
41
|
+
from fivefold.faults.catalog import (
|
|
42
|
+
F5_BEYOND_DEADLINE,
|
|
43
|
+
F10_DELAYED,
|
|
44
|
+
F10_STALE_BEYOND_LAG,
|
|
45
|
+
F10_STALE_WITHIN_LAG,
|
|
46
|
+
Action,
|
|
47
|
+
)
|
|
48
|
+
from fivefold.faults.plan import FaultInjector, Fired
|
|
49
|
+
from fivefold.harness.log import FaultRecord, HarnessLog
|
|
50
|
+
from fivefold.harness.scheduler import Scheduler
|
|
51
|
+
from fivefold.harness.transport import Envelope, Stage
|
|
52
|
+
from fivefold.sim.endpoint import StatusShape, WriteDelay
|
|
53
|
+
|
|
54
|
+
SIMULATOR = "simulator"
|
|
55
|
+
|
|
56
|
+
WRITE_RECEIVED = "sim.write.received"
|
|
57
|
+
STATUS_ANSWER = "sim.status.answer"
|
|
58
|
+
|
|
59
|
+
ACTIONS: Mapping[str, frozenset[Action]] = {
|
|
60
|
+
WRITE_RECEIVED: frozenset({Action.DELAY_COMMIT, Action.DEGRADE_STATUS}),
|
|
61
|
+
STATUS_ANSWER: frozenset({Action.DEGRADE_STATUS}),
|
|
62
|
+
}
|
|
63
|
+
"""What a fault may do at each simulator point: F5 holds a write, F10 shapes status."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class SimulatorPointError(RuntimeError):
|
|
67
|
+
"""A fault fired at a simulator point where it has nothing to do: a plan bug."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class _Shaping:
|
|
72
|
+
"""An F10 that fired, with the answers it still shapes."""
|
|
73
|
+
|
|
74
|
+
fired: Fired
|
|
75
|
+
left: int
|
|
76
|
+
fired_vt: int
|
|
77
|
+
changed: int = 0
|
|
78
|
+
"""The status answers it has changed so far."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class SimulatorPoints:
|
|
82
|
+
"""Fires F5 and F10 at the simulator's points, and logs them (0006)."""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
scheduler: Scheduler,
|
|
87
|
+
faults: FaultInjector,
|
|
88
|
+
log: HarnessLog,
|
|
89
|
+
*,
|
|
90
|
+
dispatch_timeout_ms: int,
|
|
91
|
+
status_lag_ms: int,
|
|
92
|
+
) -> None:
|
|
93
|
+
self._scheduler = scheduler
|
|
94
|
+
self._faults = faults
|
|
95
|
+
self._log = log
|
|
96
|
+
self._t_ms = dispatch_timeout_ms
|
|
97
|
+
self._lag_ms = status_lag_ms
|
|
98
|
+
self._shaping: list[_Shaping] = []
|
|
99
|
+
self._changed: dict[int, tuple[_Shaping, int]] = {}
|
|
100
|
+
"""The status requests whose answers F10 changed and has not logged yet: the F10
|
|
101
|
+
that shapes each answer, and the answer's number among those it changed."""
|
|
102
|
+
|
|
103
|
+
def write_received(self, request: Envelope) -> WriteDelay | None:
|
|
104
|
+
"""`sim.write.received`: F5 holds the write and commits it `delay_ms` later, and
|
|
105
|
+
F10 `stale_beyond_lag` freezes the state the next `count` status answers list.
|
|
106
|
+
|
|
107
|
+
A write is held once. Two F5 entries that fire at the same write are a plan bug:
|
|
108
|
+
the second would be recorded and take no effect, so the run stops instead.
|
|
109
|
+
"""
|
|
110
|
+
fired = self._faults.at(WRITE_RECEIVED, _fields(request))
|
|
111
|
+
for one in fired:
|
|
112
|
+
_check(one, WRITE_RECEIVED)
|
|
113
|
+
holds = [one for one in fired if one.planned.kind.action is Action.DELAY_COMMIT]
|
|
114
|
+
if len(holds) > 1:
|
|
115
|
+
raise SimulatorPointError(
|
|
116
|
+
f"{len(holds)} F5 entries fired at one write (message {request.seq});"
|
|
117
|
+
" a write is held once"
|
|
118
|
+
)
|
|
119
|
+
delay: WriteDelay | None = None
|
|
120
|
+
for one in fired:
|
|
121
|
+
if one.planned.kind.action is Action.DEGRADE_STATUS:
|
|
122
|
+
self._start_shaping(one, WRITE_RECEIVED, request)
|
|
123
|
+
continue
|
|
124
|
+
delay_ms = one.planned.param("delay_ms")
|
|
125
|
+
self._record(one, WRITE_RECEIVED, request, {"delay_ms": delay_ms})
|
|
126
|
+
delay = WriteDelay(delay_ms, waive_deadline=one.planned.variant == F5_BEYOND_DEADLINE)
|
|
127
|
+
return delay
|
|
128
|
+
|
|
129
|
+
def status_answer(self, request: Envelope) -> StatusShape | None:
|
|
130
|
+
"""`sim.status.answer`: F10 `delayed` and `stale_within_lag` fire here. Every F10
|
|
131
|
+
that has fired shapes this answer and the next, `count` in all."""
|
|
132
|
+
now = self._scheduler.now
|
|
133
|
+
fired = self._faults.at(STATUS_ANSWER, _fields(request))
|
|
134
|
+
for one in fired:
|
|
135
|
+
_check(one, STATUS_ANSWER)
|
|
136
|
+
for one in fired:
|
|
137
|
+
self._start_shaping(one, STATUS_ANSWER, request)
|
|
138
|
+
if not self._shaping:
|
|
139
|
+
return None
|
|
140
|
+
first = self._shaping[0]
|
|
141
|
+
first.changed += 1
|
|
142
|
+
self._changed[request.seq] = (first, first.changed)
|
|
143
|
+
for shaping in self._shaping:
|
|
144
|
+
shaping.left -= 1
|
|
145
|
+
self._shaping = [shaping for shaping in self._shaping if shaping.left > 0]
|
|
146
|
+
variant = first.fired.planned.variant
|
|
147
|
+
if variant == F10_DELAYED:
|
|
148
|
+
return StatusShape(hold_ms=self._t_ms)
|
|
149
|
+
if variant == F10_STALE_WITHIN_LAG:
|
|
150
|
+
return StatusShape(as_of_vt=max(0, now - self._lag_ms))
|
|
151
|
+
if variant == F10_STALE_BEYOND_LAG:
|
|
152
|
+
# The simulator lists effects committed at or before `reflect_vt`. The write F10
|
|
153
|
+
# fired at commits in the millisecond it arrived in at the earliest, so freezing
|
|
154
|
+
# at the millisecond before leaves it out of every shaped answer.
|
|
155
|
+
frozen_vt = max(0, first.fired_vt - 1)
|
|
156
|
+
return StatusShape(reflect_vt=frozen_vt, contract_violation=True)
|
|
157
|
+
raise SimulatorPointError(f"F10 has no variant {variant!r}")
|
|
158
|
+
|
|
159
|
+
def answer_sent(self, envelope: Envelope, stage: Stage) -> None:
|
|
160
|
+
"""A transport observer: once a status answer F10 changed is sent, log one more
|
|
161
|
+
row for that F10, naming the answer and its intent (0014). F10 acts on the next
|
|
162
|
+
`count` status answers, which can concern several intents, so each answer it
|
|
163
|
+
changes touches the answer's intent."""
|
|
164
|
+
if stage != "sent" or envelope.kind != "answer" or envelope.correlation is None:
|
|
165
|
+
return
|
|
166
|
+
changed = self._changed.pop(envelope.correlation, None)
|
|
167
|
+
if changed is None:
|
|
168
|
+
return
|
|
169
|
+
shaping, number = changed
|
|
170
|
+
count = shaping.fired.planned.param("count")
|
|
171
|
+
self._record(shaping.fired, STATUS_ANSWER, envelope, {"count": count, "answer_no": number})
|
|
172
|
+
|
|
173
|
+
def _start_shaping(self, fired: Fired, point: str, request: Envelope) -> None:
|
|
174
|
+
"""Log an F10 that fired, and shape the next `count` status answers with it."""
|
|
175
|
+
count = fired.planned.param("count")
|
|
176
|
+
self._record(fired, point, request, {"count": count})
|
|
177
|
+
self._shaping.append(_Shaping(fired, count, self._scheduler.now))
|
|
178
|
+
|
|
179
|
+
def _record(
|
|
180
|
+
self, fired: Fired, point: str, message: Envelope, used: Mapping[str, object]
|
|
181
|
+
) -> None:
|
|
182
|
+
self._log.fault_fired(
|
|
183
|
+
FaultRecord(
|
|
184
|
+
seq=self._log.next_seq(),
|
|
185
|
+
fault=str(fired.fault),
|
|
186
|
+
variant=fired.planned.variant,
|
|
187
|
+
point=point,
|
|
188
|
+
occurrence=fired.occurrence,
|
|
189
|
+
action=str(fired.planned.kind.action),
|
|
190
|
+
vt=self._scheduler.now,
|
|
191
|
+
target_seq=message.seq,
|
|
192
|
+
component=SIMULATOR,
|
|
193
|
+
params={**fired.planned.params, **used},
|
|
194
|
+
intent_id=message.intent_id,
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _check(fired: Fired, point: str) -> None:
|
|
200
|
+
"""Refuse a fault with nothing to do at `point`, before anything is logged.
|
|
201
|
+
|
|
202
|
+
The catalog already refuses such a plan. Without this check, the fault would be
|
|
203
|
+
logged as fired and change nothing, or, for an F10 variant away from its own point,
|
|
204
|
+
shape answers its variant does not describe (0013).
|
|
205
|
+
"""
|
|
206
|
+
planned = fired.planned
|
|
207
|
+
points = None if planned.variant is None else planned.kind.variants.get(planned.variant)
|
|
208
|
+
if planned.kind.action not in ACTIONS[point] or (points is not None and point not in points):
|
|
209
|
+
name = " ".join(part for part in (str(fired.fault), planned.variant) if part)
|
|
210
|
+
raise SimulatorPointError(f"{name} cannot act at {point}")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _fields(request: Envelope) -> dict[str, object]:
|
|
214
|
+
"""What a plan's match can name at a simulator point: the call and its intent."""
|
|
215
|
+
return {"api": request.api, "tool": request.api, "intent_id": request.intent_id}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""The split topology: the simulator and the gateway as HTTP services, for people who bring
|
|
2
|
+
their own agent (0009).
|
|
3
|
+
|
|
4
|
+
Record 0009's topology table: "split | Bringing your own agent: the gateway (MCP over
|
|
5
|
+
streamable HTTP), the simulator and Postgres run as separate Compose services |
|
|
6
|
+
Deterministic: No; faults still fire on counters". What runs where:
|
|
7
|
+
|
|
8
|
+
- The simulator's service (`simulator`) serves the simulator's API over HTTP: a POST to
|
|
9
|
+
`/api/{call}` for each call of `fivefold.domain.api`, with the caller's credential as a
|
|
10
|
+
bearer token. It runs the scripted approver and the faults that change the simulator's
|
|
11
|
+
state, and, for a baseline's agent, the harness's report sink.
|
|
12
|
+
- The gateway's service (`gateway`) serves the gateway's tools over MCP's streamable HTTP
|
|
13
|
+
transport at `/mcp`, and reaches the simulator only through the simulator's API, with
|
|
14
|
+
its own credential (0008).
|
|
15
|
+
- The run's lifecycle (`lifecycle`) starts a run, ends it after the drain window, and
|
|
16
|
+
scores it with the evaluator. `python -m fivefold.harness.split` runs it, and serves
|
|
17
|
+
either service.
|
|
18
|
+
|
|
19
|
+
Each service runs on a `HostScheduler`, whose virtual time follows the host's monotonic
|
|
20
|
+
clock (0009). Every component reads time from it, so the gateway's, the simulator's and
|
|
21
|
+
the fault points' code runs as it does in the inproc topology. Each service writes the
|
|
22
|
+
harness's records of the traffic it sees (`hops`), and fires the plan's faults at the
|
|
23
|
+
points it owns, on the plan's counters (`faults`). Nothing here is deterministic.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
SERVICE_STOPPED = -32019
|
|
27
|
+
"""The JSON-RPC error code the gateway answers an agent's MCP request with once the run's
|
|
28
|
+
agents' phase is over, or the run has stopped.
|
|
29
|
+
|
|
30
|
+
It is an implementation-defined server error: JSON-RPC leaves -32000 to -32099 to servers,
|
|
31
|
+
and MCP keeps -32020 to -32099 for its own codes (mcp 2.3.0, `mcp_types/jsonrpc.py`). The
|
|
32
|
+
SDK uses -32000 and -32001 for its own ends of a call; -32000, `CONNECTION_CLOSED`, is how
|
|
33
|
+
a client reads a closed connection, so it cannot also mean that the run is over. The SDK
|
|
34
|
+
uses -32019 nowhere (checked in mcp 2.3.0 and mcp-types 2.3.0, and by a test)."""
|