fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""An MCP connection between an agent worker and the gateway, through the scheduler (0009).
|
|
2
|
+
|
|
3
|
+
The inproc topology keeps the MCP layer between the agent and the gateway (0009): the
|
|
4
|
+
agent's MCP client and the gateway's MCP server exchange JSON-RPC messages, and each one
|
|
5
|
+
is a message on the `agent_gw` hop, with its latency, fate and fault points (0006). The
|
|
6
|
+
two ends are stream pairs that satisfy the MCP SDK's `ReadStream` and `WriteStream`
|
|
7
|
+
protocols (mcp 2.3.0, `mcp/shared/_stream_protocols.py`), so the SDK's `ClientSession`
|
|
8
|
+
and low-level `Server.run` use them as they would any transport.
|
|
9
|
+
|
|
10
|
+
What a crash means for a connection (0006, 0010 change 3):
|
|
11
|
+
|
|
12
|
+
- A crashed agent's connection is gone. Its requests already on the wire still reach
|
|
13
|
+
the gateway, which serves them, as a server serves a request whose client died after
|
|
14
|
+
sending it; the answers are dropped. The gateway's end reads end-of-stream once every
|
|
15
|
+
such request has arrived: `max_hop_latency_ms` after the crash, or later when a copy
|
|
16
|
+
that F4 makes is still due. A restarted agent opens a new connection.
|
|
17
|
+
- A crashed gateway's connections are gone too (F1). Answers it sent before the crash
|
|
18
|
+
still arrive; then the agent's end reads end-of-stream, `max_hop_latency_ms` after the
|
|
19
|
+
crash, and a call still waiting fails as a closed connection. What the agent sends on
|
|
20
|
+
the old connection is dropped, even after the gateway restarts: the restarted gateway
|
|
21
|
+
never knew it. The agent opens a new connection, with a new MCP handshake.
|
|
22
|
+
- A connection opened while the gateway is down is never served: what the agent sends
|
|
23
|
+
on it is dropped, as messages to a component that is down are.
|
|
24
|
+
- A component that is down sends nothing (`Transport.post`).
|
|
25
|
+
|
|
26
|
+
When the transport delivers a request twice (F4), the copy reaches the gateway with a
|
|
27
|
+
JSON-RPC id of its own, `copy-<seq>`, made from its message seq. The gateway answers
|
|
28
|
+
both, and each answer carries the id of the delivery it answers, so the agent always
|
|
29
|
+
gets the answer to its own request and the transport drops the copy's (0006), whichever
|
|
30
|
+
the gateway sends first. The logged copy keeps the request's body; only the delivered
|
|
31
|
+
message carries the new id. The gateway never uses the JSON-RPC id itself.
|
|
32
|
+
|
|
33
|
+
Each request delivered to the gateway carries its own message seq, as a transport
|
|
34
|
+
attaches its per-request data: `ServerMessageMetadata(request_context=...)`, which the
|
|
35
|
+
SDK hands to the handler as `ServerRequestContext.request` (mcp 2.3.0,
|
|
36
|
+
`mcp/shared/message.py`, `mcp/server/runner.py` `_make_context`). So the gateway names
|
|
37
|
+
the delivery it served, the copy included, when it records a final report (0004).
|
|
38
|
+
|
|
39
|
+
A message of the agent's that holds a NUL character, in a value or a key, is refused
|
|
40
|
+
before anything logs it, since the harness's log cannot store one
|
|
41
|
+
(`fivefold.harness.log.holds_nul`). As the split gateway does, the link answers a request
|
|
42
|
+
with invalid params, at once and in the agent's own inbox, and drops a notification.
|
|
43
|
+
Nothing goes on the wire, so the refusal takes no seq, no latency draw and no virtual time.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import asyncio
|
|
49
|
+
from collections import deque
|
|
50
|
+
from collections.abc import Callable
|
|
51
|
+
from dataclasses import dataclass, field
|
|
52
|
+
from types import TracebackType
|
|
53
|
+
|
|
54
|
+
import anyio
|
|
55
|
+
import mcp_types as types
|
|
56
|
+
from mcp.shared.message import ServerMessageMetadata, SessionMessage
|
|
57
|
+
|
|
58
|
+
from fivefold.harness.log import NUL_REFUSED, DropReason, Kind, holds_nul
|
|
59
|
+
from fivefold.harness.scheduler import Scheduler
|
|
60
|
+
from fivefold.harness.transport import Envelope, Transport, answer_to_copy
|
|
61
|
+
|
|
62
|
+
Item = SessionMessage | Exception
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Delivery:
|
|
67
|
+
"""The delivery of a request to the gateway: the seq of the message that carried it."""
|
|
68
|
+
|
|
69
|
+
seq: int
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def delivered_seq(request: object) -> int | None:
|
|
73
|
+
"""The message seq of the delivery a gateway handler serves.
|
|
74
|
+
|
|
75
|
+
`request` is the handler context's `request`, which the link sets on each request it
|
|
76
|
+
delivers to the gateway. When the transport delivered a request twice (F4), each
|
|
77
|
+
delivery has its own seq, so a handler serving the copy names the copy.
|
|
78
|
+
"""
|
|
79
|
+
return request.seq if isinstance(request, Delivery) else None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class Inbox:
|
|
83
|
+
"""The receiving end of one direction of a connection."""
|
|
84
|
+
|
|
85
|
+
def __init__(self) -> None:
|
|
86
|
+
self._items: deque[Item] = deque()
|
|
87
|
+
self._waiter: asyncio.Future[None] | None = None
|
|
88
|
+
self._closed = False
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def closed(self) -> bool:
|
|
92
|
+
"""Whether the sending end has closed, or the connection is gone."""
|
|
93
|
+
return self._closed
|
|
94
|
+
|
|
95
|
+
def deliver(self, item: Item) -> None:
|
|
96
|
+
"""Hand over an arrived message."""
|
|
97
|
+
if not self._closed:
|
|
98
|
+
self._items.append(item)
|
|
99
|
+
self._wake()
|
|
100
|
+
|
|
101
|
+
def close(self) -> None:
|
|
102
|
+
"""End the stream: once the delivered messages are read, `receive` ends."""
|
|
103
|
+
self._closed = True
|
|
104
|
+
self._wake()
|
|
105
|
+
|
|
106
|
+
def _wake(self) -> None:
|
|
107
|
+
if self._waiter is not None and not self._waiter.done():
|
|
108
|
+
self._waiter.set_result(None)
|
|
109
|
+
|
|
110
|
+
async def receive(self) -> Item:
|
|
111
|
+
while not self._items:
|
|
112
|
+
if self._closed:
|
|
113
|
+
raise anyio.EndOfStream
|
|
114
|
+
self._waiter = asyncio.get_running_loop().create_future()
|
|
115
|
+
await self._waiter
|
|
116
|
+
return self._items.popleft()
|
|
117
|
+
|
|
118
|
+
def __aiter__(self) -> Inbox:
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
async def __anext__(self) -> Item:
|
|
122
|
+
try:
|
|
123
|
+
return await self.receive()
|
|
124
|
+
except anyio.EndOfStream:
|
|
125
|
+
raise StopAsyncIteration from None
|
|
126
|
+
|
|
127
|
+
async def aclose(self) -> None:
|
|
128
|
+
self.close()
|
|
129
|
+
|
|
130
|
+
async def __aenter__(self) -> Inbox:
|
|
131
|
+
return self
|
|
132
|
+
|
|
133
|
+
async def __aexit__(
|
|
134
|
+
self,
|
|
135
|
+
exc_type: type[BaseException] | None,
|
|
136
|
+
exc_val: BaseException | None,
|
|
137
|
+
exc_tb: TracebackType | None,
|
|
138
|
+
) -> None:
|
|
139
|
+
self.close()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass
|
|
143
|
+
class _Requests:
|
|
144
|
+
"""What one side's requests were: their message seq, tool or method, and intent, and
|
|
145
|
+
the deliveries of each that the other side has not answered yet, oldest first."""
|
|
146
|
+
|
|
147
|
+
sent: dict[object, tuple[int, str | None, str | None]] = field(default_factory=dict)
|
|
148
|
+
arrived: dict[object, deque[Envelope]] = field(default_factory=dict)
|
|
149
|
+
|
|
150
|
+
def answering(self, message_id: object) -> Envelope | None:
|
|
151
|
+
"""The delivery that an answer with this JSON-RPC id answers: the oldest one."""
|
|
152
|
+
waiting = self.arrived.get(message_id)
|
|
153
|
+
if not waiting:
|
|
154
|
+
return None
|
|
155
|
+
delivery = waiting.popleft()
|
|
156
|
+
if not waiting:
|
|
157
|
+
del self.arrived[message_id]
|
|
158
|
+
return delivery
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class Outbox:
|
|
162
|
+
"""The sending end of one direction: each message goes through the transport."""
|
|
163
|
+
|
|
164
|
+
def __init__(
|
|
165
|
+
self,
|
|
166
|
+
link: Link,
|
|
167
|
+
transport: Transport,
|
|
168
|
+
*,
|
|
169
|
+
sender: str,
|
|
170
|
+
receiver: str,
|
|
171
|
+
peer: Inbox,
|
|
172
|
+
own: _Requests,
|
|
173
|
+
theirs: _Requests,
|
|
174
|
+
) -> None:
|
|
175
|
+
self._link = link
|
|
176
|
+
self._transport = transport
|
|
177
|
+
self._sender = sender
|
|
178
|
+
self._receiver = receiver
|
|
179
|
+
self._peer = peer
|
|
180
|
+
self._own = own
|
|
181
|
+
self._theirs = theirs
|
|
182
|
+
|
|
183
|
+
async def send(self, item: SessionMessage, /) -> None:
|
|
184
|
+
body = item.message.model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
185
|
+
method = body.get("method")
|
|
186
|
+
message_id = body.get("id")
|
|
187
|
+
if self._sender == self._link.agent and holds_nul(body):
|
|
188
|
+
self._refuse(method, message_id)
|
|
189
|
+
return
|
|
190
|
+
kind: Kind
|
|
191
|
+
correlation: int | None = None
|
|
192
|
+
api: str | None = None
|
|
193
|
+
intent_id: str | None = None
|
|
194
|
+
answer_class: str | None = None
|
|
195
|
+
if isinstance(method, str) and message_id is not None:
|
|
196
|
+
kind, api = "request", method
|
|
197
|
+
params = body.get("params")
|
|
198
|
+
if method == "tools/call" and isinstance(params, dict):
|
|
199
|
+
api = str(params.get("name"))
|
|
200
|
+
arguments = params.get("arguments")
|
|
201
|
+
if isinstance(arguments, dict) and isinstance(arguments.get("intent_id"), str):
|
|
202
|
+
intent_id = arguments["intent_id"]
|
|
203
|
+
elif isinstance(method, str):
|
|
204
|
+
kind, api = "notification", method
|
|
205
|
+
else:
|
|
206
|
+
kind = "answer"
|
|
207
|
+
correlation, api, intent_id = self._theirs.sent.get(message_id, (None, None, None))
|
|
208
|
+
answer_class = answer_label(body)
|
|
209
|
+
wanted = self._link.wanted(self._sender, kind)
|
|
210
|
+
if kind == "answer":
|
|
211
|
+
answered = self._theirs.answering(message_id)
|
|
212
|
+
if answered is not None:
|
|
213
|
+
correlation, api, intent_id = answered.seq, answered.api, answered.intent_id
|
|
214
|
+
if answered.copy_of is not None:
|
|
215
|
+
wanted = answer_to_copy
|
|
216
|
+
|
|
217
|
+
def arrive(envelope: Envelope) -> None:
|
|
218
|
+
delivered = item
|
|
219
|
+
if kind == "request":
|
|
220
|
+
# Deliver the transport's copied JSON, the same body that was logged.
|
|
221
|
+
# The sender's object can have a different mapping order or change in flight.
|
|
222
|
+
key = message_id
|
|
223
|
+
message = type(item.message).model_validate(envelope.body)
|
|
224
|
+
if envelope.copy_of is not None:
|
|
225
|
+
key = _copy_id(envelope.seq)
|
|
226
|
+
message = message.model_copy(update={"id": key})
|
|
227
|
+
delivered = SessionMessage(message, metadata=item.metadata)
|
|
228
|
+
self._own.arrived.setdefault(key, deque()).append(envelope)
|
|
229
|
+
if self._receiver == self._link.gateway:
|
|
230
|
+
metadata = ServerMessageMetadata(request_context=Delivery(envelope.seq))
|
|
231
|
+
delivered = SessionMessage(message, metadata=metadata)
|
|
232
|
+
self._peer.deliver(delivered)
|
|
233
|
+
|
|
234
|
+
envelope = self._transport.post(
|
|
235
|
+
hop="agent_gw",
|
|
236
|
+
kind=kind,
|
|
237
|
+
sender=self._sender,
|
|
238
|
+
receiver=self._receiver,
|
|
239
|
+
body=body,
|
|
240
|
+
on_arrival=arrive,
|
|
241
|
+
api=api,
|
|
242
|
+
correlation=correlation,
|
|
243
|
+
intent_id=intent_id,
|
|
244
|
+
answer_class=answer_class,
|
|
245
|
+
wanted=wanted,
|
|
246
|
+
)
|
|
247
|
+
if envelope is not None and kind == "request":
|
|
248
|
+
self._own.sent[message_id] = (envelope.seq, api, intent_id)
|
|
249
|
+
|
|
250
|
+
def _refuse(self, method: object, message_id: object) -> None:
|
|
251
|
+
"""Refuse a message of the agent's that holds a NUL character, which the harness's
|
|
252
|
+
log cannot store (`fivefold.harness.log.holds_nul`): nothing is logged or sent. A
|
|
253
|
+
request gets the split gateway's answer, invalid params, at once, in the agent's
|
|
254
|
+
own inbox; a notification is dropped."""
|
|
255
|
+
if not isinstance(method, str) or not isinstance(message_id, int | str):
|
|
256
|
+
return
|
|
257
|
+
error = types.ErrorData(
|
|
258
|
+
code=types.INVALID_PARAMS, message=f"the message is refused: {NUL_REFUSED}"
|
|
259
|
+
)
|
|
260
|
+
refusal = types.JSONRPCError(jsonrpc="2.0", id=message_id, error=error)
|
|
261
|
+
self._link.client_read.deliver(SessionMessage(refusal))
|
|
262
|
+
|
|
263
|
+
async def aclose(self) -> None:
|
|
264
|
+
"""End this direction, after the messages already on the wire arrive.
|
|
265
|
+
|
|
266
|
+
A component that is down closes nothing: its crash ends the connection.
|
|
267
|
+
"""
|
|
268
|
+
self._link.close_after_flight(self._sender, self._peer)
|
|
269
|
+
|
|
270
|
+
async def __aenter__(self) -> Outbox:
|
|
271
|
+
return self
|
|
272
|
+
|
|
273
|
+
async def __aexit__(
|
|
274
|
+
self,
|
|
275
|
+
exc_type: type[BaseException] | None,
|
|
276
|
+
exc_val: BaseException | None,
|
|
277
|
+
exc_tb: TracebackType | None,
|
|
278
|
+
) -> None:
|
|
279
|
+
await self.aclose()
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _copy_id(seq: int) -> str:
|
|
283
|
+
"""The JSON-RPC id a copy carries into the receiver: its own, never the request's."""
|
|
284
|
+
return f"copy-{seq}"
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def answer_label(body: dict[str, object]) -> str | None:
|
|
288
|
+
"""A short label for an answer: `error`, or the tool result's `result` field."""
|
|
289
|
+
if "error" in body:
|
|
290
|
+
return "error"
|
|
291
|
+
result = body.get("result")
|
|
292
|
+
if not isinstance(result, dict):
|
|
293
|
+
return None
|
|
294
|
+
if result.get("isError"):
|
|
295
|
+
return "error"
|
|
296
|
+
structured = result.get("structuredContent")
|
|
297
|
+
if isinstance(structured, dict) and isinstance(structured.get("result"), str):
|
|
298
|
+
return str(structured["result"])
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class Link:
|
|
303
|
+
"""One MCP connection from an agent worker to the gateway.
|
|
304
|
+
|
|
305
|
+
The agent's `ClientSession` uses (`client_read`, `client_write`); the gateway's
|
|
306
|
+
`Server.run` uses (`server_read`, `server_write`). Open a new link after a restart.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
def __init__(
|
|
310
|
+
self, scheduler: Scheduler, transport: Transport, *, agent: str, gateway: str
|
|
311
|
+
) -> None:
|
|
312
|
+
self._scheduler = scheduler
|
|
313
|
+
self._transport = transport
|
|
314
|
+
self.agent = agent
|
|
315
|
+
self.gateway = gateway
|
|
316
|
+
self._agent_incarnation = scheduler.incarnation(agent)
|
|
317
|
+
self._gateway_incarnation = scheduler.incarnation(gateway)
|
|
318
|
+
self._served = scheduler.is_up(gateway)
|
|
319
|
+
self.client_read = Inbox()
|
|
320
|
+
self.server_read = Inbox()
|
|
321
|
+
agent_requests, gateway_requests = _Requests(), _Requests()
|
|
322
|
+
self.client_write = Outbox(
|
|
323
|
+
self,
|
|
324
|
+
transport,
|
|
325
|
+
sender=agent,
|
|
326
|
+
receiver=gateway,
|
|
327
|
+
peer=self.server_read,
|
|
328
|
+
own=agent_requests,
|
|
329
|
+
theirs=gateway_requests,
|
|
330
|
+
)
|
|
331
|
+
self.server_write = Outbox(
|
|
332
|
+
self,
|
|
333
|
+
transport,
|
|
334
|
+
sender=gateway,
|
|
335
|
+
receiver=agent,
|
|
336
|
+
peer=self.client_read,
|
|
337
|
+
own=gateway_requests,
|
|
338
|
+
theirs=agent_requests,
|
|
339
|
+
)
|
|
340
|
+
scheduler.on_crash(self._crashed)
|
|
341
|
+
|
|
342
|
+
@property
|
|
343
|
+
def ended(self) -> bool:
|
|
344
|
+
"""Whether the agent's end of the connection has ended, or is about to."""
|
|
345
|
+
return self.client_read.closed
|
|
346
|
+
|
|
347
|
+
def close(self) -> None:
|
|
348
|
+
"""End the connection from the agent's side, once its messages have arrived."""
|
|
349
|
+
self.close_after_flight(self.agent, self.server_read)
|
|
350
|
+
|
|
351
|
+
def wanted(self, sender: str, kind: Kind) -> Callable[[], DropReason | None]:
|
|
352
|
+
"""The check on arrival for a message `sender` sends on this connection."""
|
|
353
|
+
if kind == "answer":
|
|
354
|
+
return self.answer_wanted
|
|
355
|
+
if sender == self.agent:
|
|
356
|
+
return self.gateway_wanted
|
|
357
|
+
return _always_wanted
|
|
358
|
+
|
|
359
|
+
def answer_wanted(self) -> DropReason | None:
|
|
360
|
+
"""Drop answers sent to an agent that has crashed since it opened this link."""
|
|
361
|
+
if self._scheduler.incarnation(self.agent) != self._agent_incarnation:
|
|
362
|
+
return "caller_crashed"
|
|
363
|
+
return None
|
|
364
|
+
|
|
365
|
+
def gateway_wanted(self) -> DropReason | None:
|
|
366
|
+
"""Drop what the agent sends once the gateway's end is gone: the gateway was down
|
|
367
|
+
when the connection opened, or has crashed since, restarted or not."""
|
|
368
|
+
if (
|
|
369
|
+
not self._served
|
|
370
|
+
or self._scheduler.incarnation(self.gateway) != self._gateway_incarnation
|
|
371
|
+
):
|
|
372
|
+
return "receiver_down"
|
|
373
|
+
return None
|
|
374
|
+
|
|
375
|
+
def close_after_flight(self, sender: str, inbox: Inbox) -> None:
|
|
376
|
+
"""Close `inbox` once every message `sender` has sent can have arrived."""
|
|
377
|
+
if not self._scheduler.is_up(sender):
|
|
378
|
+
return
|
|
379
|
+
self._scheduler.call_later(
|
|
380
|
+
self._in_flight_ms(sender), inbox.close, label=f"{sender} closes its end"
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
def _in_flight_ms(self, sender: str) -> int:
|
|
384
|
+
"""How long until every message `sender` has sent on this hop can have arrived:
|
|
385
|
+
`max_hop_latency_ms`, or longer while a copy that F4 makes is still due."""
|
|
386
|
+
wait = self._transport.max_hop_latency_ms + 1
|
|
387
|
+
due = self._transport.last_due_vt("agent_gw", sender)
|
|
388
|
+
if due is not None:
|
|
389
|
+
wait = max(wait, due - self._scheduler.now + 1)
|
|
390
|
+
return wait
|
|
391
|
+
|
|
392
|
+
def _crashed(self, component: str) -> None:
|
|
393
|
+
if component == self.agent:
|
|
394
|
+
self.client_read.close()
|
|
395
|
+
self._scheduler.call_later(
|
|
396
|
+
self._in_flight_ms(self.agent),
|
|
397
|
+
self.server_read.close,
|
|
398
|
+
label=f"{self.agent}'s connection ends",
|
|
399
|
+
)
|
|
400
|
+
elif component == self.gateway and self._served and not self.client_read.closed:
|
|
401
|
+
self._scheduler.call_later(
|
|
402
|
+
self._in_flight_ms(self.gateway),
|
|
403
|
+
self.client_read.close,
|
|
404
|
+
label=f"{self.agent}'s connection to the gateway ends",
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _always_wanted() -> DropReason | None:
|
|
409
|
+
return None
|