fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
"""Messages between components, through the scheduler (0006, 0009).
|
|
2
|
+
|
|
3
|
+
Every interaction between components is a message on a hop: `agent_gw` and `gw_sim` in
|
|
4
|
+
gateway configurations, `agent_sim` in baselines. Each message takes a latency drawn from
|
|
5
|
+
the run's `latency` generator, between 5 and `max_hop_latency` virtual ms, and arrives in
|
|
6
|
+
order of (arrival time, send order). The harness log records every message as it is sent
|
|
7
|
+
and then its fate: delivered, with the number of the delivery, or dropped, with why.
|
|
8
|
+
|
|
9
|
+
Transport rules (0006, as amended by 0010):
|
|
10
|
+
|
|
11
|
+
- A component that is down sends nothing: what its cancelled tasks try to send while
|
|
12
|
+
they unwind is discarded, and never logged.
|
|
13
|
+
- A message addressed to a component that is down is dropped.
|
|
14
|
+
- When a caller times out, every later answer to that request is dropped, as when an
|
|
15
|
+
HTTP client closes its connection.
|
|
16
|
+
- When a component crashes, every later answer to a request it sent before the crash is
|
|
17
|
+
dropped.
|
|
18
|
+
- Faults fire at `net.{hop}.request` and `net.{hop}.answer`, before the message travels.
|
|
19
|
+
The fault's record names the message and its intent (0014).
|
|
20
|
+
- When F4 fires on a request, the transport delivers it a second time, `copy_delay_ms`
|
|
21
|
+
after the first delivery. The copy is a message of its own, logged with `copy_of`
|
|
22
|
+
naming the request: its sender sent the request once. A copy reaches no fault point
|
|
23
|
+
and no before-send hook. The answer to a copy is never delivered: it is dropped on
|
|
24
|
+
arrival with reason `answer_to_copy`, as a retrying proxy drops it (0006). An MCP
|
|
25
|
+
notification passes the same point, as with F12, and is copied the same way.
|
|
26
|
+
|
|
27
|
+
A message's body is copied when it is sent: the log and the receiver each get their own
|
|
28
|
+
copy, so nothing done to a body later changes what was sent.
|
|
29
|
+
|
|
30
|
+
`call` is a request and its answer, like an HTTP call: the agent's tools in baselines and
|
|
31
|
+
the gateway's calls to the simulator use it. `post` sends one message and runs a callback
|
|
32
|
+
when it arrives; stream transports, such as MCP's in-process connection, build on it.
|
|
33
|
+
`Calls` names what `call` does, so the split topology's HTTP clients can stand in for it
|
|
34
|
+
(`fivefold.harness.split`).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import asyncio
|
|
40
|
+
import json
|
|
41
|
+
from collections.abc import Awaitable, Callable
|
|
42
|
+
from dataclasses import dataclass, replace
|
|
43
|
+
from typing import Literal, Protocol
|
|
44
|
+
|
|
45
|
+
from fivefold.faults.catalog import HOPS, Action
|
|
46
|
+
from fivefold.faults.plan import FaultInjector, Fired
|
|
47
|
+
from fivefold.harness.log import (
|
|
48
|
+
DropReason,
|
|
49
|
+
FaultRecord,
|
|
50
|
+
HarnessLog,
|
|
51
|
+
JsonValue,
|
|
52
|
+
Kind,
|
|
53
|
+
MessageRecord,
|
|
54
|
+
)
|
|
55
|
+
from fivefold.harness.rng import Generator
|
|
56
|
+
from fivefold.harness.scheduler import Scheduler, SchedulerTimeout
|
|
57
|
+
|
|
58
|
+
Hop = Literal["agent_gw", "gw_sim", "agent_sim"]
|
|
59
|
+
|
|
60
|
+
MIN_LATENCY_MS = 5
|
|
61
|
+
"""The shortest hop latency, in virtual ms (0009)."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class TransportTimeout(Exception):
|
|
65
|
+
"""No answer arrived within the caller's timeout.
|
|
66
|
+
|
|
67
|
+
Not an `OSError`, so LangGraph's default retry predicate retries it (0008).
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class Envelope:
|
|
73
|
+
"""One message, as the receiver gets it."""
|
|
74
|
+
|
|
75
|
+
seq: int
|
|
76
|
+
hop: Hop
|
|
77
|
+
kind: Kind
|
|
78
|
+
sender: str
|
|
79
|
+
receiver: str
|
|
80
|
+
api: str | None
|
|
81
|
+
correlation: int | None
|
|
82
|
+
intent_id: str | None
|
|
83
|
+
idempotency_key: str | None
|
|
84
|
+
answer_class: str | None
|
|
85
|
+
body: JsonValue
|
|
86
|
+
sent_vt: int
|
|
87
|
+
copy_of: int | None = None
|
|
88
|
+
"""For a copy the transport made (F4), the seq of the request it repeats."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class Reply:
|
|
93
|
+
"""A handler's answer to a request: a body and its answer class, if it has one."""
|
|
94
|
+
|
|
95
|
+
body: JsonValue
|
|
96
|
+
answer_class: str | None = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
Handler = Callable[[Envelope], Awaitable[Reply]]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class Calls(Protocol):
|
|
103
|
+
"""Sends a request and waits for its answer, as `Transport.call` does: what the
|
|
104
|
+
gateway's simulator client and the baseline's tools need from a transport."""
|
|
105
|
+
|
|
106
|
+
async def call(
|
|
107
|
+
self,
|
|
108
|
+
*,
|
|
109
|
+
hop: Hop,
|
|
110
|
+
sender: str,
|
|
111
|
+
receiver: str,
|
|
112
|
+
api: str,
|
|
113
|
+
body: JsonValue,
|
|
114
|
+
timeout_ms: int,
|
|
115
|
+
intent_id: str | None = None,
|
|
116
|
+
idempotency_key: str | None = None,
|
|
117
|
+
sent_vt: int | None = None,
|
|
118
|
+
) -> Reply:
|
|
119
|
+
"""Send a request and return its answer. Raises `TransportTimeout` when no answer
|
|
120
|
+
arrives within `timeout_ms`.
|
|
121
|
+
|
|
122
|
+
`sent_vt` is when the sender recorded the send, if it did: the gateway records
|
|
123
|
+
each write attempt with its send time before it sends it (0003). The harness's
|
|
124
|
+
log then gives the request that time, so that the gateway's settle bounds and the
|
|
125
|
+
evaluator's, which reads the log, start from one number (0004, 0005).
|
|
126
|
+
"""
|
|
127
|
+
...
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
Effect = Callable[[Fired, "Envelope"], None]
|
|
131
|
+
"""Applies a fault that fired on a message and does more than drop it (F6, F7)."""
|
|
132
|
+
|
|
133
|
+
Stage = Literal["sent", "delivered"]
|
|
134
|
+
Observer = Callable[[Envelope, Stage], None]
|
|
135
|
+
"""Called after a message is sent, and again after it is delivered."""
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass(frozen=True)
|
|
139
|
+
class Outgoing:
|
|
140
|
+
"""A message about to be sent: what a fault point before a send can match on."""
|
|
141
|
+
|
|
142
|
+
hop: Hop
|
|
143
|
+
kind: Kind
|
|
144
|
+
sender: str
|
|
145
|
+
receiver: str
|
|
146
|
+
api: str | None
|
|
147
|
+
intent_id: str | None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
BeforeSend = Callable[[Outgoing], None]
|
|
151
|
+
"""Called before a message is sent, for the fault point `agent.before_tool_call`."""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class Transport:
|
|
155
|
+
"""Carries messages between a run's components, on the scheduler's clock."""
|
|
156
|
+
|
|
157
|
+
def __init__(
|
|
158
|
+
self,
|
|
159
|
+
scheduler: Scheduler,
|
|
160
|
+
log: HarnessLog,
|
|
161
|
+
latency: Generator,
|
|
162
|
+
*,
|
|
163
|
+
max_hop_latency_ms: int,
|
|
164
|
+
faults: FaultInjector | None = None,
|
|
165
|
+
) -> None:
|
|
166
|
+
if max_hop_latency_ms < MIN_LATENCY_MS:
|
|
167
|
+
raise ValueError(f"max_hop_latency_ms must be at least {MIN_LATENCY_MS}")
|
|
168
|
+
self._scheduler = scheduler
|
|
169
|
+
self._log = log
|
|
170
|
+
self._latency = latency
|
|
171
|
+
self._max_latency = max_hop_latency_ms
|
|
172
|
+
self._faults = faults if faults is not None else FaultInjector()
|
|
173
|
+
self._handlers: dict[str, Handler] = {}
|
|
174
|
+
self._observers: list[Observer] = []
|
|
175
|
+
self._before_send: list[BeforeSend] = []
|
|
176
|
+
self._effect: Effect | None = None
|
|
177
|
+
self._last_due: dict[tuple[str, str], int] = {}
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def max_hop_latency_ms(self) -> int:
|
|
181
|
+
"""The longest one-way latency a message can take on any hop."""
|
|
182
|
+
return self._max_latency
|
|
183
|
+
|
|
184
|
+
def last_due_vt(self, hop: Hop, sender: str) -> int | None:
|
|
185
|
+
"""When the last message `sender` has sent on `hop` is due to arrive, or None.
|
|
186
|
+
|
|
187
|
+
A message arrives within `max_hop_latency_ms`, but a copy F4 delivers can arrive
|
|
188
|
+
later, `copy_delay_ms` after the first delivery. A connection waits for this
|
|
189
|
+
before its receiving end closes, so nothing it was due to deliver is lost.
|
|
190
|
+
"""
|
|
191
|
+
return self._last_due.get((hop, sender))
|
|
192
|
+
|
|
193
|
+
def _due(self, envelope: Envelope, delay_ms: int) -> None:
|
|
194
|
+
"""Note that `envelope` is due to arrive `delay_ms` from now."""
|
|
195
|
+
key = (envelope.hop, envelope.sender)
|
|
196
|
+
due = self._scheduler.now + delay_ms
|
|
197
|
+
self._last_due[key] = max(due, self._last_due.get(key, due))
|
|
198
|
+
|
|
199
|
+
def apply_faults_with(self, effect: Effect) -> None:
|
|
200
|
+
"""Apply faults that change state, such as F6 and F7, with `effect` (0006).
|
|
201
|
+
|
|
202
|
+
The effect runs when the fault fires, before the message travels on.
|
|
203
|
+
"""
|
|
204
|
+
self._effect = effect
|
|
205
|
+
|
|
206
|
+
def observe(self, observer: Observer) -> None:
|
|
207
|
+
"""Call `observer` after each message is sent, and after each is delivered.
|
|
208
|
+
|
|
209
|
+
The harness uses it for fault points that belong to a component rather than a
|
|
210
|
+
hop, such as F11's `agent.after_tool_call` (0006). It runs inside the step that
|
|
211
|
+
sent or delivered the message, after the message's fate so far is logged. For a
|
|
212
|
+
delivery it runs before the receiver gets the message, and a receiver that an
|
|
213
|
+
observer crashed never gets it: the delivery happened, and the crash voided it.
|
|
214
|
+
"""
|
|
215
|
+
self._observers.append(observer)
|
|
216
|
+
|
|
217
|
+
def before_send(self, hook: BeforeSend) -> None:
|
|
218
|
+
"""Call `hook` before each message is sent.
|
|
219
|
+
|
|
220
|
+
The harness uses it for `agent.before_tool_call`, F1's B0 equivalent (0006). A
|
|
221
|
+
hook that crashes the sender stops the message: a component that is down sends
|
|
222
|
+
nothing, so the message is never sent or logged.
|
|
223
|
+
"""
|
|
224
|
+
self._before_send.append(hook)
|
|
225
|
+
|
|
226
|
+
def draw_latency(self) -> int:
|
|
227
|
+
"""One hop's latency, from the run's latency generator (0009)."""
|
|
228
|
+
return self._latency.randint(MIN_LATENCY_MS, self._max_latency)
|
|
229
|
+
|
|
230
|
+
# One message ----------------------------------------------------------------------------
|
|
231
|
+
|
|
232
|
+
def post(
|
|
233
|
+
self,
|
|
234
|
+
*,
|
|
235
|
+
hop: Hop,
|
|
236
|
+
kind: Kind,
|
|
237
|
+
sender: str,
|
|
238
|
+
receiver: str,
|
|
239
|
+
body: JsonValue,
|
|
240
|
+
on_arrival: Callable[[Envelope], None],
|
|
241
|
+
api: str | None = None,
|
|
242
|
+
correlation: int | None = None,
|
|
243
|
+
intent_id: str | None = None,
|
|
244
|
+
idempotency_key: str | None = None,
|
|
245
|
+
answer_class: str | None = None,
|
|
246
|
+
wanted: Callable[[], DropReason | None] | None = None,
|
|
247
|
+
) -> Envelope | None:
|
|
248
|
+
"""Send one message; `on_arrival` runs when it is delivered.
|
|
249
|
+
|
|
250
|
+
`wanted`, checked on arrival, names a reason to drop the message instead, for
|
|
251
|
+
example that the caller has timed out. Returns None, sending nothing, when the
|
|
252
|
+
sender is down, or a before-send hook crashed it.
|
|
253
|
+
"""
|
|
254
|
+
if hop not in HOPS:
|
|
255
|
+
raise ValueError(f"unknown hop {hop!r}")
|
|
256
|
+
if not self._scheduler.is_up(sender):
|
|
257
|
+
return None
|
|
258
|
+
if self._before_send:
|
|
259
|
+
outgoing = Outgoing(hop, kind, sender, receiver, api, intent_id)
|
|
260
|
+
for hook in self._before_send:
|
|
261
|
+
hook(outgoing)
|
|
262
|
+
if not self._scheduler.is_up(sender):
|
|
263
|
+
return None
|
|
264
|
+
text = json.dumps(body, sort_keys=True)
|
|
265
|
+
envelope = Envelope(
|
|
266
|
+
seq=self._log.next_seq(),
|
|
267
|
+
hop=hop,
|
|
268
|
+
kind=kind,
|
|
269
|
+
sender=sender,
|
|
270
|
+
receiver=receiver,
|
|
271
|
+
api=api,
|
|
272
|
+
correlation=correlation,
|
|
273
|
+
intent_id=intent_id,
|
|
274
|
+
idempotency_key=idempotency_key,
|
|
275
|
+
answer_class=answer_class,
|
|
276
|
+
body=json.loads(text),
|
|
277
|
+
sent_vt=self._scheduler.now,
|
|
278
|
+
)
|
|
279
|
+
self._log.message_sent(
|
|
280
|
+
MessageRecord(
|
|
281
|
+
seq=envelope.seq,
|
|
282
|
+
hop=hop,
|
|
283
|
+
kind=kind,
|
|
284
|
+
sender=sender,
|
|
285
|
+
receiver=receiver,
|
|
286
|
+
api=api,
|
|
287
|
+
correlation=correlation,
|
|
288
|
+
intent_id=intent_id,
|
|
289
|
+
idempotency_key=idempotency_key,
|
|
290
|
+
answer_class=answer_class,
|
|
291
|
+
body=json.loads(text),
|
|
292
|
+
sent_vt=envelope.sent_vt,
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
point = f"net.{hop}.{'answer' if kind == 'answer' else 'request'}"
|
|
296
|
+
fields = {"api": api, "tool": api, "answer": answer_class, "intent_id": intent_id}
|
|
297
|
+
dropped = False
|
|
298
|
+
copies: list[Fired] = []
|
|
299
|
+
for fired in self._faults.at(point, fields):
|
|
300
|
+
action = fired.planned.kind.action
|
|
301
|
+
if action is Action.DUPLICATE:
|
|
302
|
+
copies.append(fired)
|
|
303
|
+
continue
|
|
304
|
+
self._record(fired, point, envelope)
|
|
305
|
+
if action is Action.DROP:
|
|
306
|
+
dropped = True
|
|
307
|
+
elif self._effect is not None:
|
|
308
|
+
self._effect(fired, envelope)
|
|
309
|
+
else:
|
|
310
|
+
raise RuntimeError(f"{fired.fault} fired at {point} with nothing to apply it")
|
|
311
|
+
latency: int | None = None
|
|
312
|
+
if dropped:
|
|
313
|
+
self._log.message_fate(envelope.seq, "dropped", vt=self._scheduler.now, reason="fault")
|
|
314
|
+
else:
|
|
315
|
+
latency = self.draw_latency()
|
|
316
|
+
self._due(envelope, latency)
|
|
317
|
+
self._scheduler.call_later(
|
|
318
|
+
latency,
|
|
319
|
+
lambda: self._arrive(envelope, on_arrival, wanted),
|
|
320
|
+
label=f"deliver {envelope.seq}",
|
|
321
|
+
)
|
|
322
|
+
for fired in copies:
|
|
323
|
+
self._copy(fired, point, envelope, text, on_arrival, wanted, latency)
|
|
324
|
+
for observer in self._observers:
|
|
325
|
+
observer(envelope, "sent")
|
|
326
|
+
return envelope
|
|
327
|
+
|
|
328
|
+
def _copy(
|
|
329
|
+
self,
|
|
330
|
+
fired: Fired,
|
|
331
|
+
point: str,
|
|
332
|
+
original: Envelope,
|
|
333
|
+
text: str,
|
|
334
|
+
on_arrival: Callable[[Envelope], None],
|
|
335
|
+
wanted: Callable[[], DropReason | None] | None,
|
|
336
|
+
latency: int | None,
|
|
337
|
+
) -> None:
|
|
338
|
+
"""F4: deliver `original` a second time, `copy_delay_ms` after its first delivery.
|
|
339
|
+
|
|
340
|
+
The fault's record names the copy (`copy_seq`), and the copy's names the request
|
|
341
|
+
it repeats (`copy_of`). When a fault dropped the request itself, the copy still
|
|
342
|
+
travels, with a latency of its own.
|
|
343
|
+
"""
|
|
344
|
+
if latency is None:
|
|
345
|
+
latency = self.draw_latency()
|
|
346
|
+
fault_seq = self._log.next_seq()
|
|
347
|
+
copy = replace(
|
|
348
|
+
original, seq=self._log.next_seq(), body=json.loads(text), copy_of=original.seq
|
|
349
|
+
)
|
|
350
|
+
self._log.fault_fired(
|
|
351
|
+
FaultRecord(
|
|
352
|
+
seq=fault_seq,
|
|
353
|
+
fault=str(fired.fault),
|
|
354
|
+
variant=fired.planned.variant,
|
|
355
|
+
point=point,
|
|
356
|
+
occurrence=fired.occurrence,
|
|
357
|
+
action=str(Action.DUPLICATE),
|
|
358
|
+
vt=self._scheduler.now,
|
|
359
|
+
target_seq=original.seq,
|
|
360
|
+
params={**fired.planned.params, "copy_seq": copy.seq},
|
|
361
|
+
intent_id=original.intent_id,
|
|
362
|
+
)
|
|
363
|
+
)
|
|
364
|
+
self._log.message_sent(
|
|
365
|
+
MessageRecord(
|
|
366
|
+
seq=copy.seq,
|
|
367
|
+
hop=copy.hop,
|
|
368
|
+
kind=copy.kind,
|
|
369
|
+
sender=copy.sender,
|
|
370
|
+
receiver=copy.receiver,
|
|
371
|
+
api=copy.api,
|
|
372
|
+
correlation=copy.correlation,
|
|
373
|
+
intent_id=copy.intent_id,
|
|
374
|
+
idempotency_key=copy.idempotency_key,
|
|
375
|
+
answer_class=copy.answer_class,
|
|
376
|
+
body=json.loads(text),
|
|
377
|
+
sent_vt=copy.sent_vt,
|
|
378
|
+
copy_of=original.seq,
|
|
379
|
+
)
|
|
380
|
+
)
|
|
381
|
+
delay_ms = latency + fired.planned.param("copy_delay_ms")
|
|
382
|
+
self._due(copy, delay_ms)
|
|
383
|
+
self._scheduler.call_later(
|
|
384
|
+
delay_ms,
|
|
385
|
+
lambda: self._arrive(copy, on_arrival, wanted),
|
|
386
|
+
label=f"deliver copy {copy.seq}",
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
def _arrive(
|
|
390
|
+
self,
|
|
391
|
+
envelope: Envelope,
|
|
392
|
+
on_arrival: Callable[[Envelope], None],
|
|
393
|
+
wanted: Callable[[], DropReason | None] | None,
|
|
394
|
+
) -> None:
|
|
395
|
+
reason: DropReason | None = None
|
|
396
|
+
if not self._scheduler.is_up(envelope.receiver):
|
|
397
|
+
reason = "receiver_down"
|
|
398
|
+
elif wanted is not None:
|
|
399
|
+
reason = wanted()
|
|
400
|
+
now = self._scheduler.now
|
|
401
|
+
if reason is not None:
|
|
402
|
+
self._log.message_fate(envelope.seq, "dropped", vt=now, reason=reason)
|
|
403
|
+
return
|
|
404
|
+
self._log.message_fate(
|
|
405
|
+
envelope.seq, "delivered", vt=now, delivered_seq=self._log.next_seq()
|
|
406
|
+
)
|
|
407
|
+
for observer in self._observers:
|
|
408
|
+
observer(envelope, "delivered")
|
|
409
|
+
if self._scheduler.is_up(envelope.receiver):
|
|
410
|
+
on_arrival(envelope)
|
|
411
|
+
|
|
412
|
+
def _record(self, fired: Fired, point: str, envelope: Envelope) -> None:
|
|
413
|
+
self._log.fault_fired(
|
|
414
|
+
FaultRecord(
|
|
415
|
+
seq=self._log.next_seq(),
|
|
416
|
+
fault=str(fired.fault),
|
|
417
|
+
variant=fired.planned.variant,
|
|
418
|
+
point=point,
|
|
419
|
+
occurrence=fired.occurrence,
|
|
420
|
+
action=str(fired.planned.kind.action),
|
|
421
|
+
vt=self._scheduler.now,
|
|
422
|
+
target_seq=envelope.seq,
|
|
423
|
+
params=dict(fired.planned.params),
|
|
424
|
+
intent_id=envelope.intent_id,
|
|
425
|
+
)
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
# A request and its answer ------------------------------------------------------------------
|
|
429
|
+
|
|
430
|
+
def serve(self, component: str, handler: Handler, *, replace: bool = False) -> None:
|
|
431
|
+
"""Answer requests addressed to `component` with `handler`.
|
|
432
|
+
|
|
433
|
+
Each request runs as a task of the component, so a crash cancels it. A component
|
|
434
|
+
rebuilt after a crash registers its new handler with `replace`, so no request
|
|
435
|
+
reaches the memory the crash discarded. The handler is looked up when a request
|
|
436
|
+
arrives.
|
|
437
|
+
"""
|
|
438
|
+
if component in self._handlers and not replace:
|
|
439
|
+
raise ValueError(f"{component} already has a handler")
|
|
440
|
+
self._handlers[component] = handler
|
|
441
|
+
|
|
442
|
+
async def call(
|
|
443
|
+
self,
|
|
444
|
+
*,
|
|
445
|
+
hop: Hop,
|
|
446
|
+
sender: str,
|
|
447
|
+
receiver: str,
|
|
448
|
+
api: str,
|
|
449
|
+
body: JsonValue,
|
|
450
|
+
timeout_ms: int,
|
|
451
|
+
intent_id: str | None = None,
|
|
452
|
+
idempotency_key: str | None = None,
|
|
453
|
+
sent_vt: int | None = None,
|
|
454
|
+
) -> Reply:
|
|
455
|
+
"""Send a request and wait for its answer, at most `timeout_ms` virtual ms.
|
|
456
|
+
|
|
457
|
+
Raises `TransportTimeout` if no answer arrives in time. A later answer is dropped.
|
|
458
|
+
`sent_vt` changes nothing here: the request is sent in its sender's step, and the
|
|
459
|
+
scheduler moves virtual time only between steps, so a send the sender recorded in
|
|
460
|
+
that step is sent at the time it recorded.
|
|
461
|
+
"""
|
|
462
|
+
if receiver not in self._handlers:
|
|
463
|
+
raise ValueError(f"{receiver} serves no requests")
|
|
464
|
+
answered: asyncio.Future[Envelope] = asyncio.get_running_loop().create_future()
|
|
465
|
+
incarnation = self._scheduler.incarnation(sender)
|
|
466
|
+
abandoned = False
|
|
467
|
+
|
|
468
|
+
def still_wanted() -> DropReason | None:
|
|
469
|
+
if self._scheduler.incarnation(sender) != incarnation:
|
|
470
|
+
return "caller_crashed"
|
|
471
|
+
if abandoned or answered.cancelled():
|
|
472
|
+
return "caller_timed_out"
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
def on_answer(answer: Envelope) -> None:
|
|
476
|
+
if not answered.done():
|
|
477
|
+
answered.set_result(answer)
|
|
478
|
+
|
|
479
|
+
def on_request(request: Envelope) -> None:
|
|
480
|
+
wanted = answer_to_copy if request.copy_of is not None else still_wanted
|
|
481
|
+
self._scheduler.spawn(
|
|
482
|
+
receiver,
|
|
483
|
+
self._answer(request, self._handlers[receiver], on_answer, wanted),
|
|
484
|
+
name=f"{api} {request.seq}",
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
self.post(
|
|
488
|
+
hop=hop,
|
|
489
|
+
kind="request",
|
|
490
|
+
sender=sender,
|
|
491
|
+
receiver=receiver,
|
|
492
|
+
body=body,
|
|
493
|
+
on_arrival=on_request,
|
|
494
|
+
api=api,
|
|
495
|
+
intent_id=intent_id,
|
|
496
|
+
idempotency_key=idempotency_key,
|
|
497
|
+
)
|
|
498
|
+
try:
|
|
499
|
+
answer = await self._scheduler.wait_for(answered, timeout_ms)
|
|
500
|
+
except SchedulerTimeout:
|
|
501
|
+
abandoned = True
|
|
502
|
+
raise TransportTimeout(
|
|
503
|
+
f"{api} from {sender} to {receiver}: no answer within {timeout_ms} ms"
|
|
504
|
+
) from None
|
|
505
|
+
return Reply(answer.body, answer.answer_class)
|
|
506
|
+
|
|
507
|
+
async def _answer(
|
|
508
|
+
self,
|
|
509
|
+
request: Envelope,
|
|
510
|
+
handler: Handler,
|
|
511
|
+
on_answer: Callable[[Envelope], None],
|
|
512
|
+
still_wanted: Callable[[], DropReason | None],
|
|
513
|
+
) -> None:
|
|
514
|
+
reply = await handler(request)
|
|
515
|
+
self.post(
|
|
516
|
+
hop=request.hop,
|
|
517
|
+
kind="answer",
|
|
518
|
+
sender=request.receiver,
|
|
519
|
+
receiver=request.sender,
|
|
520
|
+
body=reply.body,
|
|
521
|
+
on_arrival=on_answer,
|
|
522
|
+
api=request.api,
|
|
523
|
+
correlation=request.seq,
|
|
524
|
+
intent_id=request.intent_id,
|
|
525
|
+
idempotency_key=request.idempotency_key,
|
|
526
|
+
answer_class=reply.answer_class,
|
|
527
|
+
wanted=still_wanted,
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def answer_to_copy() -> DropReason:
|
|
532
|
+
"""The arrival check of an answer to a copy (F4): it never reaches the sender (0006)."""
|
|
533
|
+
return "answer_to_copy"
|
fivefold/paths.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Paths to the runtime data in a checkout or an installed distribution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import atexit
|
|
6
|
+
from contextlib import ExitStack
|
|
7
|
+
from importlib.resources import as_file, files
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
# The wheel contains one data directory. Keep its extraction alive for callers that
|
|
11
|
+
# need real paths, including when the package is imported from an archive.
|
|
12
|
+
_resources = files("fivefold").joinpath("_data")
|
|
13
|
+
_stack = ExitStack()
|
|
14
|
+
atexit.register(_stack.close)
|
|
15
|
+
if _resources.is_dir():
|
|
16
|
+
_data = _stack.enter_context(as_file(_resources))
|
|
17
|
+
else:
|
|
18
|
+
# Editable installs use the single copies at the repository root.
|
|
19
|
+
_data = Path(__file__).resolve().parents[2]
|
|
20
|
+
|
|
21
|
+
DB_DIR = _data / "db"
|
|
22
|
+
DEV_001 = _data / "scenarios" / "dev" / "dev-001-single-po.yaml"
|
|
23
|
+
DEV_002 = _data / "scenarios" / "dev" / "dev-002-human-approval.yaml"
|
|
24
|
+
EXPECTED = _data / "faults" / "expected.yaml"
|
fivefold/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Model cassettes, run playback and experimental branches (0009).
|
|
2
|
+
|
|
3
|
+
A run can record every model request and response into a cassette: scripted runs on
|
|
4
|
+
request, and live runs always (`fivefold.agents.live`). Replay serves them in order, so
|
|
5
|
+
anyone can re-execute a recorded run without an API key or a network, and check that it
|
|
6
|
+
does the same as before. An experimental branch replays a recorded run's first K model
|
|
7
|
+
calls, then switches to the scripted model or a live one, as a new run with its own
|
|
8
|
+
cassette.
|
|
9
|
+
|
|
10
|
+
- `cassette`: the cassette format, its digest, and the request hash.
|
|
11
|
+
- `recording`: the recording wrapper, a chat model that records the model it wraps.
|
|
12
|
+
- `playback`: record a run, re-execute a recorded run from its cassette and compare its
|
|
13
|
+
records with the original's, and build a branch of a recorded run;
|
|
14
|
+
`python -m fivefold.recorder` on the command line.
|
|
15
|
+
|
|
16
|
+
The replay and branch adapters are model adapters, in `fivefold.agents.replay` and
|
|
17
|
+
`fivefold.agents.branch`. The harness picks a run's adapter in `fivefold.harness.models`.
|
|
18
|
+
"""
|