fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
"""The simulator's service in the split topology: its API over HTTP (0008, 0009).
|
|
2
|
+
|
|
3
|
+
Each call of the simulator's API (`fivefold.domain.api`) is a POST to `/api/{call}` with
|
|
4
|
+
the request model as a JSON body, and the caller's secret as a bearer token
|
|
5
|
+
(`credentials`). The answer is the answer model as JSON. Every answer the API defines,
|
|
6
|
+
refusals and rejections included, comes with status 200, except a caller refused for
|
|
7
|
+
holding no credential of the run, which gets 403. Requests that never reach the API get
|
|
8
|
+
other statuses, and are not logged:
|
|
9
|
+
|
|
10
|
+
| Status | When |
|
|
11
|
+
| --- | --- |
|
|
12
|
+
| 401 | no token, or not a secret of this run; the body is the call's forbidden answer |
|
|
13
|
+
| 404 | no such call |
|
|
14
|
+
| 422 | a body that is not the call's request model or holds a NUL; a wait over `max_wait_ms` |
|
|
15
|
+
| 503 | the run has stopped, or, for anyone but the gateway, its agents' phase is over |
|
|
16
|
+
|
|
17
|
+
A baseline's agent submits its final report the same way, to `/api/submit_final_report`:
|
|
18
|
+
the harness's report sink takes it, as in the inproc topology (0008). In G1 and G2 reports
|
|
19
|
+
go through the gateway, so this call is refused (403) to everyone.
|
|
20
|
+
|
|
21
|
+
The caller's identity comes from its token, never from what it sends. The simulator then
|
|
22
|
+
applies its own rule, which the run set up: in G1 and G2 only the gateway holds a
|
|
23
|
+
credential, so the agent's identity is refused, reads and writes alike (0008, trust
|
|
24
|
+
boundaries). A request from the gateway travels on the `gw_sim` hop, one from the agent on
|
|
25
|
+
`agent_sim`.
|
|
26
|
+
|
|
27
|
+
The service runs the components the inproc harness runs next to the simulator: the
|
|
28
|
+
scripted approver, F6 and F7 through the simulator's control interface, F5 and F10 at the
|
|
29
|
+
simulator's points (`fivefold.sim.endpoint`, `fivefold.harness.sim_points`), and the
|
|
30
|
+
faults of the network points it owns (`fivefold.harness.split.faults`). As the inproc
|
|
31
|
+
transport tells the simulator's points of each message it sends, the service tells them
|
|
32
|
+
of each answer it sends, so F10 logs one more row for each status answer it changed
|
|
33
|
+
(0014). Each request
|
|
34
|
+
runs as a task of the component `simulator`, so stopping the run cancels the requests
|
|
35
|
+
still in flight, such as a write F5 holds.
|
|
36
|
+
|
|
37
|
+
`/harness/drain` and `/harness/stop` take the harness's token. Drain refuses the agent
|
|
38
|
+
from then on: the agents' phase is over (0009). Stop aborts every write in flight past
|
|
39
|
+
its deadline, records the run's stop time (`harness.run_stop`), refuses everyone, and
|
|
40
|
+
writes no harness record after it.
|
|
41
|
+
`/health` says whether the service is serving, and how many of its tasks failed.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import asyncio
|
|
47
|
+
import json
|
|
48
|
+
from collections.abc import AsyncIterator, Mapping
|
|
49
|
+
from contextlib import asynccontextmanager
|
|
50
|
+
from dataclasses import dataclass, field
|
|
51
|
+
from typing import Any
|
|
52
|
+
|
|
53
|
+
from pydantic import ValidationError
|
|
54
|
+
from starlette.applications import Starlette
|
|
55
|
+
from starlette.requests import Request
|
|
56
|
+
from starlette.responses import JSONResponse, Response
|
|
57
|
+
from starlette.routing import Route
|
|
58
|
+
|
|
59
|
+
from fivefold.domain.api import (
|
|
60
|
+
REQUESTS,
|
|
61
|
+
ApiName,
|
|
62
|
+
Refusal,
|
|
63
|
+
RefusalReason,
|
|
64
|
+
WaitLimit,
|
|
65
|
+
WriteRejected,
|
|
66
|
+
WriteRejectReason,
|
|
67
|
+
)
|
|
68
|
+
from fivefold.domain.report import validation_messages
|
|
69
|
+
from fivefold.faults.plan import FaultInjector, Fired
|
|
70
|
+
from fivefold.harness.actors import ScriptedApprover
|
|
71
|
+
from fivefold.harness.fault_effects import SimulatorFaults, message_context
|
|
72
|
+
from fivefold.harness.log import NUL_REFUSED, DropReason, holds_nul
|
|
73
|
+
from fivefold.harness.reports import ReportSink, sink_endpoint
|
|
74
|
+
from fivefold.harness.sim_points import SimulatorPoints
|
|
75
|
+
from fivefold.harness.split.credentials import GATEWAY, HARNESS
|
|
76
|
+
from fivefold.harness.split.hops import SEQ_HEADER, WRITES, Hops, logged_message
|
|
77
|
+
from fivefold.harness.split.rundir import SplitRun
|
|
78
|
+
from fivefold.harness.split.service import OpenRun, open_run, unavailable, until_gone
|
|
79
|
+
from fivefold.harness.transport import Envelope, Handler, Hop, Reply
|
|
80
|
+
from fivefold.sim import Simulator
|
|
81
|
+
from fivefold.sim.endpoint import endpoint
|
|
82
|
+
|
|
83
|
+
SIMULATOR = "simulator"
|
|
84
|
+
REPORT_SINK = "report_sink"
|
|
85
|
+
REPORT_CALL = "submit_final_report"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class _Serving:
|
|
90
|
+
"""What the service holds while it serves its run."""
|
|
91
|
+
|
|
92
|
+
opened: OpenRun
|
|
93
|
+
simulator: Simulator
|
|
94
|
+
hops: Hops
|
|
95
|
+
points: SimulatorPoints
|
|
96
|
+
api: Handler
|
|
97
|
+
sink: Handler
|
|
98
|
+
wait_limit: WaitLimit
|
|
99
|
+
stopping: asyncio.Event = field(default_factory=asyncio.Event)
|
|
100
|
+
draining: bool = False
|
|
101
|
+
stopped_vt: int | None = None
|
|
102
|
+
|
|
103
|
+
def has_stopped(self) -> bool:
|
|
104
|
+
"""Whether the run has stopped. A request asks again after each wait, since the run
|
|
105
|
+
can stop meanwhile."""
|
|
106
|
+
return self.stopped_vt is not None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class SimulatorService:
|
|
110
|
+
"""The simulator's API for one split run, over HTTP."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, run: SplitRun, *, base_dsn: str) -> None:
|
|
113
|
+
self.run = run
|
|
114
|
+
self._base_dsn = base_dsn
|
|
115
|
+
self._callers = run.callers()
|
|
116
|
+
self._serving: _Serving | None = None
|
|
117
|
+
|
|
118
|
+
def app(self) -> Starlette:
|
|
119
|
+
"""The service's ASGI app."""
|
|
120
|
+
return Starlette(
|
|
121
|
+
routes=[
|
|
122
|
+
Route("/health", self._health, methods=["GET"]),
|
|
123
|
+
Route("/api/{call}", self._call, methods=["POST"]),
|
|
124
|
+
Route("/harness/{action}", self._control, methods=["POST"]),
|
|
125
|
+
],
|
|
126
|
+
lifespan=self._lifespan,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def serving(self) -> _Serving:
|
|
131
|
+
if self._serving is None:
|
|
132
|
+
raise RuntimeError("the simulator's service has not started")
|
|
133
|
+
return self._serving
|
|
134
|
+
|
|
135
|
+
# Starting and stopping ----------------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
@asynccontextmanager
|
|
138
|
+
async def _lifespan(self, app: Starlette) -> AsyncIterator[None]:
|
|
139
|
+
opened = open_run(self.run, self._base_dsn)
|
|
140
|
+
try:
|
|
141
|
+
self._serving = self._start(opened)
|
|
142
|
+
yield
|
|
143
|
+
finally:
|
|
144
|
+
if self._serving is not None:
|
|
145
|
+
self._serving.stopping.set()
|
|
146
|
+
self._serving.simulator.close()
|
|
147
|
+
opened.close()
|
|
148
|
+
|
|
149
|
+
def _start(self, opened: OpenRun) -> _Serving:
|
|
150
|
+
scenario, scheduler, log = opened.scenario, opened.scheduler, opened.log
|
|
151
|
+
params = scenario.params
|
|
152
|
+
simulator = Simulator(self.run.conninfo(self._base_dsn, "ff_sim"))
|
|
153
|
+
faults = FaultInjector(opened.plan)
|
|
154
|
+
approver = ScriptedApprover(scheduler, simulator.control, log, scenario.authority)
|
|
155
|
+
effects = SimulatorFaults(scheduler, simulator.control, log, scenario)
|
|
156
|
+
|
|
157
|
+
def effect(fired: Fired, envelope: Envelope) -> None:
|
|
158
|
+
effects.apply(fired, message_context(envelope.body))
|
|
159
|
+
|
|
160
|
+
points = SimulatorPoints(
|
|
161
|
+
scheduler,
|
|
162
|
+
faults,
|
|
163
|
+
log,
|
|
164
|
+
dispatch_timeout_ms=params.dispatch_timeout_ms,
|
|
165
|
+
status_lag_ms=params.status_lag_ms,
|
|
166
|
+
)
|
|
167
|
+
api = endpoint(
|
|
168
|
+
simulator,
|
|
169
|
+
scheduler,
|
|
170
|
+
poll_ms=params.approval_poll_interval_ms,
|
|
171
|
+
max_wait_ms=params.max_wait_ms,
|
|
172
|
+
served=[approver.served],
|
|
173
|
+
points=points,
|
|
174
|
+
)
|
|
175
|
+
intents = [intent.id for intent in scenario.intents]
|
|
176
|
+
sink = sink_endpoint(ReportSink(scheduler, log, intents))
|
|
177
|
+
hops = Hops(scheduler, log, faults, effect=effect)
|
|
178
|
+
limit = WaitLimit(params.max_wait_ms)
|
|
179
|
+
return _Serving(opened, simulator, hops, points, api, sink, limit)
|
|
180
|
+
|
|
181
|
+
def stop(self) -> int:
|
|
182
|
+
"""Stop the run: abort writes in flight past their deadline, record the stop time,
|
|
183
|
+
and end every task and timer. Returns the stop time; stopping again returns it."""
|
|
184
|
+
serving = self.serving
|
|
185
|
+
if serving.stopped_vt is None:
|
|
186
|
+
scheduler = serving.opened.scheduler
|
|
187
|
+
now = scheduler.now
|
|
188
|
+
serving.simulator.control.abort_overdue(vt=now)
|
|
189
|
+
serving.opened.log.run_stopped(now)
|
|
190
|
+
serving.stopped_vt = now
|
|
191
|
+
scheduler.stop()
|
|
192
|
+
serving.opened.stop_records()
|
|
193
|
+
serving.stopping.set()
|
|
194
|
+
return serving.stopped_vt
|
|
195
|
+
|
|
196
|
+
# Routes -------------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
async def _health(self, request: Request) -> Response:
|
|
199
|
+
serving = self.serving
|
|
200
|
+
status = "serving"
|
|
201
|
+
if serving.has_stopped():
|
|
202
|
+
status = "stopped"
|
|
203
|
+
elif serving.draining:
|
|
204
|
+
status = "draining"
|
|
205
|
+
failures = len(serving.opened.scheduler.failures)
|
|
206
|
+
body = {"run_id": self.run.run_id, "status": status, "failures": failures}
|
|
207
|
+
return JSONResponse(body)
|
|
208
|
+
|
|
209
|
+
async def _control(self, request: Request) -> Response:
|
|
210
|
+
if self._callers.identify(request.headers.get("authorization")) != HARNESS:
|
|
211
|
+
return JSONResponse({"error": "forbidden"}, status_code=401)
|
|
212
|
+
action = request.path_params["action"]
|
|
213
|
+
if action == "drain":
|
|
214
|
+
self.serving.draining = True
|
|
215
|
+
return JSONResponse({"status": "draining"})
|
|
216
|
+
if action == "stop":
|
|
217
|
+
return JSONResponse({"stopped_vt": self.stop()})
|
|
218
|
+
return JSONResponse({"error": "unknown_action", "detail": action}, status_code=404)
|
|
219
|
+
|
|
220
|
+
async def _call(self, request: Request) -> Response:
|
|
221
|
+
call = str(request.path_params["call"])
|
|
222
|
+
serving = self.serving
|
|
223
|
+
if serving.has_stopped():
|
|
224
|
+
return unavailable("the run has stopped")
|
|
225
|
+
caller = self._callers.identify(request.headers.get("authorization"))
|
|
226
|
+
if caller is None or caller == HARNESS:
|
|
227
|
+
return _forbidden(call, "no credential of this run", status=401)
|
|
228
|
+
if serving.draining and caller != GATEWAY:
|
|
229
|
+
return unavailable("the run's agents' phase is over")
|
|
230
|
+
try:
|
|
231
|
+
body = json.loads(await request.body())
|
|
232
|
+
except ValueError:
|
|
233
|
+
return _invalid("the body is not JSON")
|
|
234
|
+
if holds_nul(body):
|
|
235
|
+
return _invalid(f"the body is refused: {NUL_REFUSED}")
|
|
236
|
+
if serving.has_stopped():
|
|
237
|
+
return unavailable("the run has stopped")
|
|
238
|
+
receiver, handler = SIMULATOR, serving.api
|
|
239
|
+
if call == REPORT_CALL:
|
|
240
|
+
# Only B0's agent reports here; in G1 and G2 reports go through the gateway
|
|
241
|
+
# (0008), so no caller of the simulator may report.
|
|
242
|
+
if self.run.has_gateway or caller not in serving.simulator.config.credentials:
|
|
243
|
+
return _forbidden(
|
|
244
|
+
call, f"{caller} cannot report here in {self.run.config}", status=403
|
|
245
|
+
)
|
|
246
|
+
if not isinstance(body, dict):
|
|
247
|
+
return _invalid("a report is a JSON object")
|
|
248
|
+
receiver, handler = REPORT_SINK, serving.sink
|
|
249
|
+
else:
|
|
250
|
+
try:
|
|
251
|
+
api = ApiName(call)
|
|
252
|
+
except ValueError:
|
|
253
|
+
return JSONResponse({"error": "unknown_call", "detail": call}, status_code=404)
|
|
254
|
+
try:
|
|
255
|
+
REQUESTS[api].model_validate(body, context=serving.wait_limit)
|
|
256
|
+
except ValidationError as error:
|
|
257
|
+
return _invalid("; ".join(validation_messages(error)))
|
|
258
|
+
hop: Hop = "gw_sim" if caller == GATEWAY else "agent_sim"
|
|
259
|
+
arrived = self._arrived(request, hop, caller, receiver, call, body)
|
|
260
|
+
if arrived is None:
|
|
261
|
+
return _invalid(f"{SEQ_HEADER} names no request the gateway sent")
|
|
262
|
+
return await self._serve(request, arrived, handler)
|
|
263
|
+
|
|
264
|
+
def _arrived(
|
|
265
|
+
self, request: Request, hop: Hop, caller: str, receiver: str, call: str, body: object
|
|
266
|
+
) -> Envelope | None:
|
|
267
|
+
"""The arrived request, as logged: the gateway logged its own as it sent it; the
|
|
268
|
+
agent's is logged now. None for a seq that names no unanswered gateway request."""
|
|
269
|
+
serving = self.serving
|
|
270
|
+
header = request.headers.get(SEQ_HEADER)
|
|
271
|
+
if hop == "gw_sim" and header is not None:
|
|
272
|
+
try:
|
|
273
|
+
found = logged_message(serving.opened.conn, int(header))
|
|
274
|
+
except ValueError:
|
|
275
|
+
return None
|
|
276
|
+
if found is None:
|
|
277
|
+
return None
|
|
278
|
+
logged, has_fate = found
|
|
279
|
+
if (
|
|
280
|
+
has_fate
|
|
281
|
+
or logged.hop != hop
|
|
282
|
+
or logged.kind != "request"
|
|
283
|
+
or logged.sender != caller
|
|
284
|
+
or logged.receiver != receiver
|
|
285
|
+
or logged.api != call
|
|
286
|
+
):
|
|
287
|
+
return None
|
|
288
|
+
return logged
|
|
289
|
+
fields = body if isinstance(body, Mapping) else {}
|
|
290
|
+
return serving.hops.sent(
|
|
291
|
+
hop=hop,
|
|
292
|
+
kind="request",
|
|
293
|
+
sender=caller,
|
|
294
|
+
receiver=receiver,
|
|
295
|
+
api=call,
|
|
296
|
+
body=body,
|
|
297
|
+
intent_id=_text(fields.get("intent_id")),
|
|
298
|
+
idempotency_key=_text(fields.get("idempotency_key")),
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
async def _serve(self, request: Request, arrived: Envelope, handler: Handler) -> Response:
|
|
302
|
+
serving = self.serving
|
|
303
|
+
hops = serving.hops
|
|
304
|
+
point = f"net.{arrived.hop}.request"
|
|
305
|
+
outcome = hops.fire(point, arrived)
|
|
306
|
+
if outcome.dropped:
|
|
307
|
+
hops.dropped(arrived, "fault")
|
|
308
|
+
await until_gone(request, serving.stopping)
|
|
309
|
+
return unavailable("the request was lost")
|
|
310
|
+
hops.delivered(arrived)
|
|
311
|
+
for fired in outcome.copies:
|
|
312
|
+
self._deliver_copy_later(fired, point, arrived, handler)
|
|
313
|
+
reply = await self._handled(arrived, handler)
|
|
314
|
+
if reply is None or serving.has_stopped():
|
|
315
|
+
return unavailable("the run stopped while the request was handled")
|
|
316
|
+
answer = hops.sent(
|
|
317
|
+
hop=arrived.hop,
|
|
318
|
+
kind="answer",
|
|
319
|
+
sender=arrived.receiver,
|
|
320
|
+
receiver=arrived.sender,
|
|
321
|
+
api=arrived.api,
|
|
322
|
+
body=reply.body,
|
|
323
|
+
intent_id=arrived.intent_id,
|
|
324
|
+
idempotency_key=arrived.idempotency_key,
|
|
325
|
+
correlation=arrived.seq,
|
|
326
|
+
answer_class=reply.answer_class,
|
|
327
|
+
)
|
|
328
|
+
serving.points.answer_sent(answer, "sent")
|
|
329
|
+
status = 403 if _refused_forbidden(reply.body) else 200
|
|
330
|
+
if arrived.hop == "gw_sim":
|
|
331
|
+
# The gateway logs the answer's fate as it arrives, or does not arrive.
|
|
332
|
+
return JSONResponse(
|
|
333
|
+
reply.body, status_code=status, headers={SEQ_HEADER: str(answer.seq)}
|
|
334
|
+
)
|
|
335
|
+
if hops.fire("net.agent_sim.answer", answer).dropped:
|
|
336
|
+
hops.dropped(answer, "fault")
|
|
337
|
+
await until_gone(request, serving.stopping)
|
|
338
|
+
return unavailable("the answer was lost")
|
|
339
|
+
gone = await request.is_disconnected()
|
|
340
|
+
if serving.has_stopped():
|
|
341
|
+
return unavailable("the run stopped while the request was handled")
|
|
342
|
+
if gone:
|
|
343
|
+
hops.dropped(answer, "caller_timed_out")
|
|
344
|
+
return unavailable("the caller has gone")
|
|
345
|
+
hops.delivered(answer)
|
|
346
|
+
return JSONResponse(reply.body, status_code=status)
|
|
347
|
+
|
|
348
|
+
async def _handled(self, arrived: Envelope, handler: Handler) -> Reply | None:
|
|
349
|
+
"""`handler`'s reply to `arrived`, run as a task of the simulator; None if the run
|
|
350
|
+
stopped first."""
|
|
351
|
+
scheduler = self.serving.opened.scheduler
|
|
352
|
+
if scheduler.stopped:
|
|
353
|
+
return None
|
|
354
|
+
|
|
355
|
+
async def handled() -> Reply:
|
|
356
|
+
return await handler(arrived)
|
|
357
|
+
|
|
358
|
+
task = scheduler.spawn(SIMULATOR, handled(), name=f"{arrived.api} {arrived.seq}")
|
|
359
|
+
await asyncio.wait({task})
|
|
360
|
+
if task.cancelled():
|
|
361
|
+
return None
|
|
362
|
+
return task.result()
|
|
363
|
+
|
|
364
|
+
# F4's copies ----------------------------------------------------------------------------
|
|
365
|
+
|
|
366
|
+
def _deliver_copy_later(
|
|
367
|
+
self, fired: Fired, point: str, original: Envelope, handler: Handler
|
|
368
|
+
) -> None:
|
|
369
|
+
"""F4: log the copy now, and deliver it `copy_delay_ms` after the request (0006)."""
|
|
370
|
+
serving = self.serving
|
|
371
|
+
scheduler = serving.opened.scheduler
|
|
372
|
+
copy = serving.hops.copy(fired, point, original)
|
|
373
|
+
|
|
374
|
+
def arrive() -> None:
|
|
375
|
+
scheduler.spawn(SIMULATOR, self._copy_arrives(copy, handler), name=f"copy {copy.seq}")
|
|
376
|
+
|
|
377
|
+
scheduler.call_later(
|
|
378
|
+
fired.planned.param("copy_delay_ms"), arrive, label=f"deliver copy {copy.seq}"
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
async def _copy_arrives(self, copy: Envelope, handler: Handler) -> None:
|
|
382
|
+
serving = self.serving
|
|
383
|
+
hops = serving.hops
|
|
384
|
+
hops.delivered(copy)
|
|
385
|
+
reply = await handler(copy)
|
|
386
|
+
answer = hops.sent(
|
|
387
|
+
hop=copy.hop,
|
|
388
|
+
kind="answer",
|
|
389
|
+
sender=copy.receiver,
|
|
390
|
+
receiver=copy.sender,
|
|
391
|
+
api=copy.api,
|
|
392
|
+
body=reply.body,
|
|
393
|
+
intent_id=copy.intent_id,
|
|
394
|
+
idempotency_key=copy.idempotency_key,
|
|
395
|
+
correlation=copy.seq,
|
|
396
|
+
answer_class=reply.answer_class,
|
|
397
|
+
)
|
|
398
|
+
serving.points.answer_sent(answer, "sent")
|
|
399
|
+
reason: DropReason = "answer_to_copy"
|
|
400
|
+
hops.dropped(answer, reason)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _text(value: object) -> str | None:
|
|
404
|
+
return value if isinstance(value, str) else None
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _refused_forbidden(body: object) -> bool:
|
|
408
|
+
"""Whether an answer refuses its caller for holding no credential of the run."""
|
|
409
|
+
return isinstance(body, Mapping) and body.get("reason") == "forbidden"
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _forbidden(call: str, detail: str, *, status: int) -> JSONResponse:
|
|
413
|
+
"""The call's own answer to a caller with no credential: a write is rejected, every
|
|
414
|
+
other call refused (`fivefold.domain.api`)."""
|
|
415
|
+
answer: Any
|
|
416
|
+
if call in WRITES:
|
|
417
|
+
answer = WriteRejected(reason=WriteRejectReason.FORBIDDEN, detail=detail)
|
|
418
|
+
elif call == REPORT_CALL:
|
|
419
|
+
return JSONResponse(
|
|
420
|
+
{"result": "refused", "reason": "forbidden", "details": {"message": detail}},
|
|
421
|
+
status_code=status,
|
|
422
|
+
headers={"www-authenticate": "Bearer"},
|
|
423
|
+
)
|
|
424
|
+
else:
|
|
425
|
+
answer = Refusal(reason=RefusalReason.FORBIDDEN, detail=detail)
|
|
426
|
+
return JSONResponse(
|
|
427
|
+
answer.model_dump(mode="json"), status_code=status, headers={"www-authenticate": "Bearer"}
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _invalid(detail: str) -> JSONResponse:
|
|
432
|
+
return JSONResponse({"error": "invalid_request", "detail": detail}, status_code=422)
|