fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,409 @@
1
+ """An MCP connection between an agent worker and the gateway, through the scheduler (0009).
2
+
3
+ The inproc topology keeps the MCP layer between the agent and the gateway (0009): the
4
+ agent's MCP client and the gateway's MCP server exchange JSON-RPC messages, and each one
5
+ is a message on the `agent_gw` hop, with its latency, fate and fault points (0006). The
6
+ two ends are stream pairs that satisfy the MCP SDK's `ReadStream` and `WriteStream`
7
+ protocols (mcp 2.3.0, `mcp/shared/_stream_protocols.py`), so the SDK's `ClientSession`
8
+ and low-level `Server.run` use them as they would any transport.
9
+
10
+ What a crash means for a connection (0006, 0010 change 3):
11
+
12
+ - A crashed agent's connection is gone. Its requests already on the wire still reach
13
+ the gateway, which serves them, as a server serves a request whose client died after
14
+ sending it; the answers are dropped. The gateway's end reads end-of-stream once every
15
+ such request has arrived: `max_hop_latency_ms` after the crash, or later when a copy
16
+ that F4 makes is still due. A restarted agent opens a new connection.
17
+ - A crashed gateway's connections are gone too (F1). Answers it sent before the crash
18
+ still arrive; then the agent's end reads end-of-stream, `max_hop_latency_ms` after the
19
+ crash, and a call still waiting fails as a closed connection. What the agent sends on
20
+ the old connection is dropped, even after the gateway restarts: the restarted gateway
21
+ never knew it. The agent opens a new connection, with a new MCP handshake.
22
+ - A connection opened while the gateway is down is never served: what the agent sends
23
+ on it is dropped, as messages to a component that is down are.
24
+ - A component that is down sends nothing (`Transport.post`).
25
+
26
+ When the transport delivers a request twice (F4), the copy reaches the gateway with a
27
+ JSON-RPC id of its own, `copy-<seq>`, made from its message seq. The gateway answers
28
+ both, and each answer carries the id of the delivery it answers, so the agent always
29
+ gets the answer to its own request and the transport drops the copy's (0006), whichever
30
+ the gateway sends first. The logged copy keeps the request's body; only the delivered
31
+ message carries the new id. The gateway never uses the JSON-RPC id itself.
32
+
33
+ Each request delivered to the gateway carries its own message seq, as a transport
34
+ attaches its per-request data: `ServerMessageMetadata(request_context=...)`, which the
35
+ SDK hands to the handler as `ServerRequestContext.request` (mcp 2.3.0,
36
+ `mcp/shared/message.py`, `mcp/server/runner.py` `_make_context`). So the gateway names
37
+ the delivery it served, the copy included, when it records a final report (0004).
38
+
39
+ A message of the agent's that holds a NUL character, in a value or a key, is refused
40
+ before anything logs it, since the harness's log cannot store one
41
+ (`fivefold.harness.log.holds_nul`). As the split gateway does, the link answers a request
42
+ with invalid params, at once and in the agent's own inbox, and drops a notification.
43
+ Nothing goes on the wire, so the refusal takes no seq, no latency draw and no virtual time.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import asyncio
49
+ from collections import deque
50
+ from collections.abc import Callable
51
+ from dataclasses import dataclass, field
52
+ from types import TracebackType
53
+
54
+ import anyio
55
+ import mcp_types as types
56
+ from mcp.shared.message import ServerMessageMetadata, SessionMessage
57
+
58
+ from fivefold.harness.log import NUL_REFUSED, DropReason, Kind, holds_nul
59
+ from fivefold.harness.scheduler import Scheduler
60
+ from fivefold.harness.transport import Envelope, Transport, answer_to_copy
61
+
62
+ Item = SessionMessage | Exception
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Delivery:
67
+ """The delivery of a request to the gateway: the seq of the message that carried it."""
68
+
69
+ seq: int
70
+
71
+
72
+ def delivered_seq(request: object) -> int | None:
73
+ """The message seq of the delivery a gateway handler serves.
74
+
75
+ `request` is the handler context's `request`, which the link sets on each request it
76
+ delivers to the gateway. When the transport delivered a request twice (F4), each
77
+ delivery has its own seq, so a handler serving the copy names the copy.
78
+ """
79
+ return request.seq if isinstance(request, Delivery) else None
80
+
81
+
82
+ class Inbox:
83
+ """The receiving end of one direction of a connection."""
84
+
85
+ def __init__(self) -> None:
86
+ self._items: deque[Item] = deque()
87
+ self._waiter: asyncio.Future[None] | None = None
88
+ self._closed = False
89
+
90
+ @property
91
+ def closed(self) -> bool:
92
+ """Whether the sending end has closed, or the connection is gone."""
93
+ return self._closed
94
+
95
+ def deliver(self, item: Item) -> None:
96
+ """Hand over an arrived message."""
97
+ if not self._closed:
98
+ self._items.append(item)
99
+ self._wake()
100
+
101
+ def close(self) -> None:
102
+ """End the stream: once the delivered messages are read, `receive` ends."""
103
+ self._closed = True
104
+ self._wake()
105
+
106
+ def _wake(self) -> None:
107
+ if self._waiter is not None and not self._waiter.done():
108
+ self._waiter.set_result(None)
109
+
110
+ async def receive(self) -> Item:
111
+ while not self._items:
112
+ if self._closed:
113
+ raise anyio.EndOfStream
114
+ self._waiter = asyncio.get_running_loop().create_future()
115
+ await self._waiter
116
+ return self._items.popleft()
117
+
118
+ def __aiter__(self) -> Inbox:
119
+ return self
120
+
121
+ async def __anext__(self) -> Item:
122
+ try:
123
+ return await self.receive()
124
+ except anyio.EndOfStream:
125
+ raise StopAsyncIteration from None
126
+
127
+ async def aclose(self) -> None:
128
+ self.close()
129
+
130
+ async def __aenter__(self) -> Inbox:
131
+ return self
132
+
133
+ async def __aexit__(
134
+ self,
135
+ exc_type: type[BaseException] | None,
136
+ exc_val: BaseException | None,
137
+ exc_tb: TracebackType | None,
138
+ ) -> None:
139
+ self.close()
140
+
141
+
142
+ @dataclass
143
+ class _Requests:
144
+ """What one side's requests were: their message seq, tool or method, and intent, and
145
+ the deliveries of each that the other side has not answered yet, oldest first."""
146
+
147
+ sent: dict[object, tuple[int, str | None, str | None]] = field(default_factory=dict)
148
+ arrived: dict[object, deque[Envelope]] = field(default_factory=dict)
149
+
150
+ def answering(self, message_id: object) -> Envelope | None:
151
+ """The delivery that an answer with this JSON-RPC id answers: the oldest one."""
152
+ waiting = self.arrived.get(message_id)
153
+ if not waiting:
154
+ return None
155
+ delivery = waiting.popleft()
156
+ if not waiting:
157
+ del self.arrived[message_id]
158
+ return delivery
159
+
160
+
161
+ class Outbox:
162
+ """The sending end of one direction: each message goes through the transport."""
163
+
164
+ def __init__(
165
+ self,
166
+ link: Link,
167
+ transport: Transport,
168
+ *,
169
+ sender: str,
170
+ receiver: str,
171
+ peer: Inbox,
172
+ own: _Requests,
173
+ theirs: _Requests,
174
+ ) -> None:
175
+ self._link = link
176
+ self._transport = transport
177
+ self._sender = sender
178
+ self._receiver = receiver
179
+ self._peer = peer
180
+ self._own = own
181
+ self._theirs = theirs
182
+
183
+ async def send(self, item: SessionMessage, /) -> None:
184
+ body = item.message.model_dump(mode="json", by_alias=True, exclude_none=True)
185
+ method = body.get("method")
186
+ message_id = body.get("id")
187
+ if self._sender == self._link.agent and holds_nul(body):
188
+ self._refuse(method, message_id)
189
+ return
190
+ kind: Kind
191
+ correlation: int | None = None
192
+ api: str | None = None
193
+ intent_id: str | None = None
194
+ answer_class: str | None = None
195
+ if isinstance(method, str) and message_id is not None:
196
+ kind, api = "request", method
197
+ params = body.get("params")
198
+ if method == "tools/call" and isinstance(params, dict):
199
+ api = str(params.get("name"))
200
+ arguments = params.get("arguments")
201
+ if isinstance(arguments, dict) and isinstance(arguments.get("intent_id"), str):
202
+ intent_id = arguments["intent_id"]
203
+ elif isinstance(method, str):
204
+ kind, api = "notification", method
205
+ else:
206
+ kind = "answer"
207
+ correlation, api, intent_id = self._theirs.sent.get(message_id, (None, None, None))
208
+ answer_class = answer_label(body)
209
+ wanted = self._link.wanted(self._sender, kind)
210
+ if kind == "answer":
211
+ answered = self._theirs.answering(message_id)
212
+ if answered is not None:
213
+ correlation, api, intent_id = answered.seq, answered.api, answered.intent_id
214
+ if answered.copy_of is not None:
215
+ wanted = answer_to_copy
216
+
217
+ def arrive(envelope: Envelope) -> None:
218
+ delivered = item
219
+ if kind == "request":
220
+ # Deliver the transport's copied JSON, the same body that was logged.
221
+ # The sender's object can have a different mapping order or change in flight.
222
+ key = message_id
223
+ message = type(item.message).model_validate(envelope.body)
224
+ if envelope.copy_of is not None:
225
+ key = _copy_id(envelope.seq)
226
+ message = message.model_copy(update={"id": key})
227
+ delivered = SessionMessage(message, metadata=item.metadata)
228
+ self._own.arrived.setdefault(key, deque()).append(envelope)
229
+ if self._receiver == self._link.gateway:
230
+ metadata = ServerMessageMetadata(request_context=Delivery(envelope.seq))
231
+ delivered = SessionMessage(message, metadata=metadata)
232
+ self._peer.deliver(delivered)
233
+
234
+ envelope = self._transport.post(
235
+ hop="agent_gw",
236
+ kind=kind,
237
+ sender=self._sender,
238
+ receiver=self._receiver,
239
+ body=body,
240
+ on_arrival=arrive,
241
+ api=api,
242
+ correlation=correlation,
243
+ intent_id=intent_id,
244
+ answer_class=answer_class,
245
+ wanted=wanted,
246
+ )
247
+ if envelope is not None and kind == "request":
248
+ self._own.sent[message_id] = (envelope.seq, api, intent_id)
249
+
250
+ def _refuse(self, method: object, message_id: object) -> None:
251
+ """Refuse a message of the agent's that holds a NUL character, which the harness's
252
+ log cannot store (`fivefold.harness.log.holds_nul`): nothing is logged or sent. A
253
+ request gets the split gateway's answer, invalid params, at once, in the agent's
254
+ own inbox; a notification is dropped."""
255
+ if not isinstance(method, str) or not isinstance(message_id, int | str):
256
+ return
257
+ error = types.ErrorData(
258
+ code=types.INVALID_PARAMS, message=f"the message is refused: {NUL_REFUSED}"
259
+ )
260
+ refusal = types.JSONRPCError(jsonrpc="2.0", id=message_id, error=error)
261
+ self._link.client_read.deliver(SessionMessage(refusal))
262
+
263
+ async def aclose(self) -> None:
264
+ """End this direction, after the messages already on the wire arrive.
265
+
266
+ A component that is down closes nothing: its crash ends the connection.
267
+ """
268
+ self._link.close_after_flight(self._sender, self._peer)
269
+
270
+ async def __aenter__(self) -> Outbox:
271
+ return self
272
+
273
+ async def __aexit__(
274
+ self,
275
+ exc_type: type[BaseException] | None,
276
+ exc_val: BaseException | None,
277
+ exc_tb: TracebackType | None,
278
+ ) -> None:
279
+ await self.aclose()
280
+
281
+
282
+ def _copy_id(seq: int) -> str:
283
+ """The JSON-RPC id a copy carries into the receiver: its own, never the request's."""
284
+ return f"copy-{seq}"
285
+
286
+
287
+ def answer_label(body: dict[str, object]) -> str | None:
288
+ """A short label for an answer: `error`, or the tool result's `result` field."""
289
+ if "error" in body:
290
+ return "error"
291
+ result = body.get("result")
292
+ if not isinstance(result, dict):
293
+ return None
294
+ if result.get("isError"):
295
+ return "error"
296
+ structured = result.get("structuredContent")
297
+ if isinstance(structured, dict) and isinstance(structured.get("result"), str):
298
+ return str(structured["result"])
299
+ return None
300
+
301
+
302
+ class Link:
303
+ """One MCP connection from an agent worker to the gateway.
304
+
305
+ The agent's `ClientSession` uses (`client_read`, `client_write`); the gateway's
306
+ `Server.run` uses (`server_read`, `server_write`). Open a new link after a restart.
307
+ """
308
+
309
+ def __init__(
310
+ self, scheduler: Scheduler, transport: Transport, *, agent: str, gateway: str
311
+ ) -> None:
312
+ self._scheduler = scheduler
313
+ self._transport = transport
314
+ self.agent = agent
315
+ self.gateway = gateway
316
+ self._agent_incarnation = scheduler.incarnation(agent)
317
+ self._gateway_incarnation = scheduler.incarnation(gateway)
318
+ self._served = scheduler.is_up(gateway)
319
+ self.client_read = Inbox()
320
+ self.server_read = Inbox()
321
+ agent_requests, gateway_requests = _Requests(), _Requests()
322
+ self.client_write = Outbox(
323
+ self,
324
+ transport,
325
+ sender=agent,
326
+ receiver=gateway,
327
+ peer=self.server_read,
328
+ own=agent_requests,
329
+ theirs=gateway_requests,
330
+ )
331
+ self.server_write = Outbox(
332
+ self,
333
+ transport,
334
+ sender=gateway,
335
+ receiver=agent,
336
+ peer=self.client_read,
337
+ own=gateway_requests,
338
+ theirs=agent_requests,
339
+ )
340
+ scheduler.on_crash(self._crashed)
341
+
342
+ @property
343
+ def ended(self) -> bool:
344
+ """Whether the agent's end of the connection has ended, or is about to."""
345
+ return self.client_read.closed
346
+
347
+ def close(self) -> None:
348
+ """End the connection from the agent's side, once its messages have arrived."""
349
+ self.close_after_flight(self.agent, self.server_read)
350
+
351
+ def wanted(self, sender: str, kind: Kind) -> Callable[[], DropReason | None]:
352
+ """The check on arrival for a message `sender` sends on this connection."""
353
+ if kind == "answer":
354
+ return self.answer_wanted
355
+ if sender == self.agent:
356
+ return self.gateway_wanted
357
+ return _always_wanted
358
+
359
+ def answer_wanted(self) -> DropReason | None:
360
+ """Drop answers sent to an agent that has crashed since it opened this link."""
361
+ if self._scheduler.incarnation(self.agent) != self._agent_incarnation:
362
+ return "caller_crashed"
363
+ return None
364
+
365
+ def gateway_wanted(self) -> DropReason | None:
366
+ """Drop what the agent sends once the gateway's end is gone: the gateway was down
367
+ when the connection opened, or has crashed since, restarted or not."""
368
+ if (
369
+ not self._served
370
+ or self._scheduler.incarnation(self.gateway) != self._gateway_incarnation
371
+ ):
372
+ return "receiver_down"
373
+ return None
374
+
375
+ def close_after_flight(self, sender: str, inbox: Inbox) -> None:
376
+ """Close `inbox` once every message `sender` has sent can have arrived."""
377
+ if not self._scheduler.is_up(sender):
378
+ return
379
+ self._scheduler.call_later(
380
+ self._in_flight_ms(sender), inbox.close, label=f"{sender} closes its end"
381
+ )
382
+
383
+ def _in_flight_ms(self, sender: str) -> int:
384
+ """How long until every message `sender` has sent on this hop can have arrived:
385
+ `max_hop_latency_ms`, or longer while a copy that F4 makes is still due."""
386
+ wait = self._transport.max_hop_latency_ms + 1
387
+ due = self._transport.last_due_vt("agent_gw", sender)
388
+ if due is not None:
389
+ wait = max(wait, due - self._scheduler.now + 1)
390
+ return wait
391
+
392
+ def _crashed(self, component: str) -> None:
393
+ if component == self.agent:
394
+ self.client_read.close()
395
+ self._scheduler.call_later(
396
+ self._in_flight_ms(self.agent),
397
+ self.server_read.close,
398
+ label=f"{self.agent}'s connection ends",
399
+ )
400
+ elif component == self.gateway and self._served and not self.client_read.closed:
401
+ self._scheduler.call_later(
402
+ self._in_flight_ms(self.gateway),
403
+ self.client_read.close,
404
+ label=f"{self.agent}'s connection to the gateway ends",
405
+ )
406
+
407
+
408
+ def _always_wanted() -> DropReason | None:
409
+ return None