fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"""The scripted agent over the network: a stand-in for your own agent (0008, 0009).
|
|
2
|
+
|
|
3
|
+
In the split topology the agent is the user's own. This is the shared LangGraph agent of
|
|
4
|
+
record 0008 with the scripted model, reaching a split run's services as any agent would:
|
|
5
|
+
in G1 and G2 the gateway's MCP endpoint over streamable HTTP, and in B0 the simulator's
|
|
6
|
+
HTTP API with the agent's credential. Use it to check a split setup before bringing your
|
|
7
|
+
own agent; the tests run the split topology end to end with it.
|
|
8
|
+
|
|
9
|
+
Its tools are the inproc agent's (`fivefold.agents.tools`), with the same timeouts: T plus
|
|
10
|
+
each tool's declared wait (0008). It runs on a `HostScheduler`, so the timeouts pass in
|
|
11
|
+
wall time. A model call takes no time: `think_ms` is virtual time the inproc scheduler
|
|
12
|
+
spends, and no clock here moves for it. It keeps its checkpoints in memory.
|
|
13
|
+
|
|
14
|
+
As in the inproc topology, a call whose connection to the gateway closes raises
|
|
15
|
+
`ConnectionError`, and the next call opens a new connection; a call with no answer within
|
|
16
|
+
its timeout raises `TransportTimeout`; and a server error raises `ServerError`. LangGraph's
|
|
17
|
+
default retry policy retries all three (0008). A call the gateway refuses because the
|
|
18
|
+
run's agents' phase is over raises `RunOver`, which it does not retry: the run has nothing
|
|
19
|
+
more for the agent.
|
|
20
|
+
|
|
21
|
+
python -m fivefold.agents.remote --scenario PATH --config G2 --contract C2 \\
|
|
22
|
+
--gateway-url http://gateway:8000/mcp
|
|
23
|
+
python -m fivefold.agents.remote --scenario PATH --config B0 --contract C2 \\
|
|
24
|
+
--simulator-url http://127.0.0.1:8100 --token-file split-run/agent-1.token
|
|
25
|
+
|
|
26
|
+
In G1 and G2 run it where your own agent would run: in a container on the Docker network
|
|
27
|
+
`fivefold-agent`, as the Compose service `agent` does, which reaches the gateway at
|
|
28
|
+
`http://gateway:8000/mcp` and nothing else (0008).
|
|
29
|
+
|
|
30
|
+
MCP SDK calls used (mcp 2.3.0): `mcp.client.streamable_http.streamable_http_client(url)`,
|
|
31
|
+
an async context manager that yields the read and write streams
|
|
32
|
+
(`mcp/client/streamable_http.py`); `ClientSession(read, write)` as an async context
|
|
33
|
+
manager, with `initialize`, `list_tools` and `call_tool`; `MCPError.code`; and
|
|
34
|
+
`mcp_types.CONNECTION_CLOSED` and `INTERNAL_ERROR`, the codes a call gets when its
|
|
35
|
+
connection closed and when the server answered with an error status. Checked in
|
|
36
|
+
langgraph-checkpoint 4.3.2: `langgraph.checkpoint.memory.InMemorySaver`. Checked in httpx
|
|
37
|
+
0.28.1: `AsyncClient(base_url=, timeout=)`, `post(url, json=, headers=)` and `HTTPError`.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import argparse
|
|
43
|
+
import asyncio
|
|
44
|
+
import sys
|
|
45
|
+
from collections.abc import Awaitable, Sequence
|
|
46
|
+
from dataclasses import dataclass
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
from typing import Any, Literal
|
|
49
|
+
|
|
50
|
+
import httpx
|
|
51
|
+
import mcp_types as types
|
|
52
|
+
from langgraph.checkpoint.memory import InMemorySaver
|
|
53
|
+
from mcp.client.session import ClientSession
|
|
54
|
+
from mcp.client.streamable_http import streamable_http_client
|
|
55
|
+
from mcp.shared.exceptions import MCPError
|
|
56
|
+
|
|
57
|
+
from fivefold.agents.connection import within
|
|
58
|
+
from fivefold.agents.policy import Mode, scripted_policy
|
|
59
|
+
from fivefold.agents.scripted import ScriptedModel
|
|
60
|
+
from fivefold.agents.tools import ServerError, Timeouts, baseline_tools, gateway_tools
|
|
61
|
+
from fivefold.agents.worker import WorkerEnd, WorkerSpec, build_graph, run_worker
|
|
62
|
+
from fivefold.domain.contracts import Capability, Contract
|
|
63
|
+
from fivefold.domain.scenario import Scenario, load_scenario_file
|
|
64
|
+
from fivefold.harness.clock import HostClock
|
|
65
|
+
from fivefold.harness.log import JsonValue
|
|
66
|
+
from fivefold.harness.scheduler import HostScheduler, Scheduler, SchedulerTimeout
|
|
67
|
+
from fivefold.harness.split import SERVICE_STOPPED
|
|
68
|
+
from fivefold.harness.transport import Hop, Reply, TransportTimeout
|
|
69
|
+
|
|
70
|
+
WORKER = "agent-1"
|
|
71
|
+
"""The split run's one agent worker (0009)."""
|
|
72
|
+
|
|
73
|
+
Config = Literal["B0", "G1", "G2"]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class RunOver(RuntimeError):
|
|
77
|
+
"""The gateway answered that the run's agents' phase is over, or that the run has
|
|
78
|
+
stopped (`fivefold.harness.split.SERVICE_STOPPED`): nothing the agent sends counts any
|
|
79
|
+
more. A `RuntimeError`, which LangGraph's default retry policy never retries
|
|
80
|
+
(langgraph 1.2.22, `langgraph/_internal/_retry.py`), so the worker ends at once."""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# G1 and G2: the gateway's MCP endpoint over streamable HTTP ------------------------------------
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class _Session:
|
|
88
|
+
session: ClientSession
|
|
89
|
+
done: asyncio.Event
|
|
90
|
+
ended: asyncio.Event
|
|
91
|
+
holder: asyncio.Task[None]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class HttpGatewayConnection:
|
|
95
|
+
"""An agent worker's MCP connection to the gateway over streamable HTTP: one session at
|
|
96
|
+
a time, and a new one after a connection closes."""
|
|
97
|
+
|
|
98
|
+
def __init__(self, scheduler: Scheduler, url: str, *, timeout_ms: int) -> None:
|
|
99
|
+
"""`url` is the gateway's MCP endpoint; `timeout_ms` bounds each handshake request."""
|
|
100
|
+
self._scheduler = scheduler
|
|
101
|
+
self._url = url
|
|
102
|
+
self._timeout_ms = timeout_ms
|
|
103
|
+
self._current: _Session | None = None
|
|
104
|
+
self._opening = asyncio.Lock()
|
|
105
|
+
|
|
106
|
+
async def list_tools(self) -> types.ListToolsResult:
|
|
107
|
+
"""The gateway's tools, from `tools/list`."""
|
|
108
|
+
session = await self._session()
|
|
109
|
+
return await self._ask(session.list_tools(), self._timeout_ms, "tools/list")
|
|
110
|
+
|
|
111
|
+
async def call_tool(
|
|
112
|
+
self, name: str, arguments: dict[str, Any], timeout_ms: int
|
|
113
|
+
) -> types.CallToolResult:
|
|
114
|
+
"""Call a tool, waiting at most `timeout_ms` for its result."""
|
|
115
|
+
session = await self._session()
|
|
116
|
+
return await self._ask(session.call_tool(name, arguments), timeout_ms, name)
|
|
117
|
+
|
|
118
|
+
async def _ask[T](self, call: Awaitable[T], timeout_ms: int, what: str) -> T:
|
|
119
|
+
try:
|
|
120
|
+
return await within(self._scheduler, call, timeout_ms, what)
|
|
121
|
+
except MCPError as error:
|
|
122
|
+
if error.code == SERVICE_STOPPED:
|
|
123
|
+
raise RunOver(f"{what}: {error.message}") from error
|
|
124
|
+
if error.code == types.CONNECTION_CLOSED:
|
|
125
|
+
self.close()
|
|
126
|
+
raise ConnectionError(f"{what}: the connection to the gateway closed") from error
|
|
127
|
+
if error.code == types.INTERNAL_ERROR:
|
|
128
|
+
raise ServerError(f"{what}: {error.message}") from error
|
|
129
|
+
raise
|
|
130
|
+
|
|
131
|
+
def close(self) -> None:
|
|
132
|
+
"""End the current session, if there is one."""
|
|
133
|
+
current, self._current = self._current, None
|
|
134
|
+
if current is not None:
|
|
135
|
+
current.done.set()
|
|
136
|
+
|
|
137
|
+
async def _session(self) -> ClientSession:
|
|
138
|
+
async with self._opening:
|
|
139
|
+
current = self._current
|
|
140
|
+
if current is not None and not current.ended.is_set():
|
|
141
|
+
return current.session
|
|
142
|
+
self.close()
|
|
143
|
+
loop = asyncio.get_running_loop()
|
|
144
|
+
opened: asyncio.Future[ClientSession] = loop.create_future()
|
|
145
|
+
done, ended = asyncio.Event(), asyncio.Event()
|
|
146
|
+
holder = loop.create_task(self._hold(opened, done, ended), name="gateway session")
|
|
147
|
+
session = await opened
|
|
148
|
+
fresh = _Session(session, done, ended, holder)
|
|
149
|
+
self._current = fresh
|
|
150
|
+
try:
|
|
151
|
+
await within(self._scheduler, session.initialize(), self._timeout_ms, "initialize")
|
|
152
|
+
except BaseException as error:
|
|
153
|
+
if self._current is fresh:
|
|
154
|
+
self.close()
|
|
155
|
+
if isinstance(error, MCPError) and error.code == SERVICE_STOPPED:
|
|
156
|
+
raise RunOver(f"initialize: {error.message}") from error
|
|
157
|
+
raise
|
|
158
|
+
return session
|
|
159
|
+
|
|
160
|
+
async def _hold(
|
|
161
|
+
self, opened: asyncio.Future[ClientSession], done: asyncio.Event, ended: asyncio.Event
|
|
162
|
+
) -> None:
|
|
163
|
+
"""Keep one session open until it is no longer needed, or its connection fails."""
|
|
164
|
+
try:
|
|
165
|
+
async with (
|
|
166
|
+
streamable_http_client(self._url) as (read, write),
|
|
167
|
+
ClientSession(read, write) as session,
|
|
168
|
+
):
|
|
169
|
+
opened.set_result(session)
|
|
170
|
+
await done.wait()
|
|
171
|
+
except Exception as error:
|
|
172
|
+
if not opened.done():
|
|
173
|
+
opened.set_exception(ConnectionError(f"cannot reach the gateway: {error}"))
|
|
174
|
+
finally:
|
|
175
|
+
ended.set()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
# B0: the simulator's HTTP API ----------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class HttpApi:
|
|
182
|
+
"""The simulator's API over HTTP, with the agent's credential: B0's tools' transport
|
|
183
|
+
(`fivefold.harness.transport.Calls`). The report goes to the same service."""
|
|
184
|
+
|
|
185
|
+
def __init__(self, scheduler: Scheduler, client: httpx.AsyncClient, *, secret: str) -> None:
|
|
186
|
+
self._scheduler = scheduler
|
|
187
|
+
self._client = client
|
|
188
|
+
self._secret = secret
|
|
189
|
+
|
|
190
|
+
async def call(
|
|
191
|
+
self,
|
|
192
|
+
*,
|
|
193
|
+
hop: Hop,
|
|
194
|
+
sender: str,
|
|
195
|
+
receiver: str,
|
|
196
|
+
api: str,
|
|
197
|
+
body: JsonValue,
|
|
198
|
+
timeout_ms: int,
|
|
199
|
+
intent_id: str | None = None,
|
|
200
|
+
idempotency_key: str | None = None,
|
|
201
|
+
sent_vt: int | None = None,
|
|
202
|
+
) -> Reply:
|
|
203
|
+
"""Send one call; wait at most `timeout_ms` for its answer.
|
|
204
|
+
|
|
205
|
+
Raises `TransportTimeout` with no answer in time, `ConnectionError` when the
|
|
206
|
+
connection fails, and `ServerError` for a 5xx status. `sent_vt` changes nothing:
|
|
207
|
+
the simulator logs the agent's calls as they arrive, and the agent's tools record
|
|
208
|
+
no send time.
|
|
209
|
+
"""
|
|
210
|
+
headers = {"authorization": f"Bearer {self._secret}"}
|
|
211
|
+
try:
|
|
212
|
+
response = await self._scheduler.wait_for(
|
|
213
|
+
self._client.post(f"/api/{api}", json=body, headers=headers), timeout_ms
|
|
214
|
+
)
|
|
215
|
+
except SchedulerTimeout:
|
|
216
|
+
raise TransportTimeout(f"{api}: no answer within {timeout_ms} ms") from None
|
|
217
|
+
except httpx.HTTPError as error:
|
|
218
|
+
raise ConnectionError(
|
|
219
|
+
f"{api}: the connection to the simulator failed: {error}"
|
|
220
|
+
) from None
|
|
221
|
+
if response.status_code >= 500:
|
|
222
|
+
raise ServerError(f"{api}: the simulator answered {response.status_code}")
|
|
223
|
+
return Reply(response.json())
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# The agent -----------------------------------------------------------------------------------
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass(frozen=True)
|
|
230
|
+
class RemoteAgent:
|
|
231
|
+
"""The scripted agent of one split run, and where it reaches the run's services."""
|
|
232
|
+
|
|
233
|
+
scenario: Scenario
|
|
234
|
+
config: Config
|
|
235
|
+
contract: Contract
|
|
236
|
+
gateway_url: str | None = None
|
|
237
|
+
"""The gateway's MCP endpoint, in G1 and G2."""
|
|
238
|
+
simulator_url: str | None = None
|
|
239
|
+
"""The simulator's service, in B0."""
|
|
240
|
+
secret: str | None = None
|
|
241
|
+
"""The agent's credential, in B0."""
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
async def run_remote_agent(agent: RemoteAgent) -> tuple[WorkerEnd, str]:
|
|
245
|
+
"""Run the scripted agent to its end; return how it ended (`run_worker`)."""
|
|
246
|
+
scenario = agent.scenario
|
|
247
|
+
scheduler = HostScheduler(HostClock(HostClock.host_ms()))
|
|
248
|
+
params = scenario.params
|
|
249
|
+
timeouts = Timeouts(
|
|
250
|
+
t_ms=params.dispatch_timeout_ms,
|
|
251
|
+
tool_wait_ms=params.tool_wait_ms,
|
|
252
|
+
max_wait_ms=params.max_wait_ms,
|
|
253
|
+
)
|
|
254
|
+
gateway = agent.config != "B0"
|
|
255
|
+
mode = Mode.GATEWAY if gateway else Mode.BASELINE
|
|
256
|
+
|
|
257
|
+
def think() -> Any:
|
|
258
|
+
return scheduler.sleep(0)
|
|
259
|
+
|
|
260
|
+
model = ScriptedModel(
|
|
261
|
+
lambda: scripted_policy(
|
|
262
|
+
scenario.intents, mode=mode, rank=0, max_wait_ms=params.max_wait_ms
|
|
263
|
+
),
|
|
264
|
+
think,
|
|
265
|
+
)
|
|
266
|
+
budget = scenario.budget
|
|
267
|
+
spec = WorkerSpec(WORKER, scenario.task, budget.max_agent_steps, budget.max_tokens)
|
|
268
|
+
if gateway:
|
|
269
|
+
if agent.gateway_url is None:
|
|
270
|
+
raise ValueError(f"{agent.config} reaches the gateway: give its MCP endpoint")
|
|
271
|
+
connection = HttpGatewayConnection(
|
|
272
|
+
scheduler, agent.gateway_url, timeout_ms=params.dispatch_timeout_ms
|
|
273
|
+
)
|
|
274
|
+
try:
|
|
275
|
+
tools = gateway_tools(connection, await connection.list_tools(), timeouts)
|
|
276
|
+
graph = build_graph(
|
|
277
|
+
model,
|
|
278
|
+
tools,
|
|
279
|
+
InMemorySaver(),
|
|
280
|
+
max_steps=spec.max_steps,
|
|
281
|
+
max_tokens=spec.max_tokens,
|
|
282
|
+
)
|
|
283
|
+
return await run_worker(graph, spec, resume=False)
|
|
284
|
+
finally:
|
|
285
|
+
connection.close()
|
|
286
|
+
if agent.simulator_url is None or agent.secret is None:
|
|
287
|
+
raise ValueError("B0 reaches the simulator: give its address and the agent's credential")
|
|
288
|
+
async with httpx.AsyncClient(base_url=agent.simulator_url, timeout=None) as client:
|
|
289
|
+
api = HttpApi(scheduler, client, secret=agent.secret)
|
|
290
|
+
tools = baseline_tools(
|
|
291
|
+
api, scheduler, timeouts, worker=WORKER, status=agent.contract.has(Capability.STATUS)
|
|
292
|
+
)
|
|
293
|
+
graph = build_graph(
|
|
294
|
+
model, tools, InMemorySaver(), max_steps=spec.max_steps, max_tokens=spec.max_tokens
|
|
295
|
+
)
|
|
296
|
+
return await run_worker(graph, spec, resume=False)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
300
|
+
"""Command-line entry point. Returns the process exit status."""
|
|
301
|
+
parser = argparse.ArgumentParser(
|
|
302
|
+
prog="python -m fivefold.agents.remote",
|
|
303
|
+
description="Run the scripted agent against a split run's services.",
|
|
304
|
+
)
|
|
305
|
+
parser.add_argument("--scenario", type=Path, required=True, help="the run's scenario file")
|
|
306
|
+
parser.add_argument("--config", choices=["B0", "G1", "G2"], required=True)
|
|
307
|
+
parser.add_argument("--contract", choices=[str(level) for level in Contract], required=True)
|
|
308
|
+
parser.add_argument("--gateway-url", help="the gateway's MCP endpoint (G1, G2)")
|
|
309
|
+
parser.add_argument("--simulator-url", help="the simulator's service (B0)")
|
|
310
|
+
parser.add_argument("--token-file", type=Path, help="the agent's credential (B0)")
|
|
311
|
+
args = parser.parse_args(argv)
|
|
312
|
+
secret = None
|
|
313
|
+
if args.token_file is not None:
|
|
314
|
+
secret = args.token_file.read_text(encoding="utf-8").strip()
|
|
315
|
+
agent = RemoteAgent(
|
|
316
|
+
scenario=load_scenario_file(args.scenario),
|
|
317
|
+
config=args.config,
|
|
318
|
+
contract=Contract(args.contract),
|
|
319
|
+
gateway_url=args.gateway_url,
|
|
320
|
+
simulator_url=args.simulator_url,
|
|
321
|
+
secret=secret,
|
|
322
|
+
)
|
|
323
|
+
try:
|
|
324
|
+
end, detail = asyncio.run(run_remote_agent(agent))
|
|
325
|
+
except ValueError as error:
|
|
326
|
+
print(f"agent: {error}", file=sys.stderr)
|
|
327
|
+
return 1
|
|
328
|
+
print(f"{WORKER}: {end} ({detail})")
|
|
329
|
+
return 0 if end is WorkerEnd.FINISHED else 1
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
if __name__ == "__main__":
|
|
333
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""The replay adapter (0009): serves a cassette's responses in order.
|
|
2
|
+
|
|
3
|
+
All of a run's replay models share one `CassettePlayer`, which hands out the cassette's
|
|
4
|
+
calls in the order the recorded run made them. Each model call first takes `think_ms` of
|
|
5
|
+
virtual time, as with every adapter (0009). It then hashes the request as the recorder did
|
|
6
|
+
(`fivefold.recorder.cassette`). If the next recorded call has that hash and was made by
|
|
7
|
+
the same agent worker, the call answers with the recorded response.
|
|
8
|
+
|
|
9
|
+
Otherwise the replay has failed. The call raises `ReplayError`, naming the call's number
|
|
10
|
+
and both hashes, and so does every later call. The same happens when the run asks for more
|
|
11
|
+
calls than the cassette holds. `CassettePlayer.finish`, once the run's agents are done,
|
|
12
|
+
raises the first failure, or a `ReplayError` if the run left recorded calls unasked: the
|
|
13
|
+
run fails (0009). The adapter makes no network call and spends no tokens.
|
|
14
|
+
|
|
15
|
+
Checked in langchain-core 1.8.0: `BaseChatModel` subclasses implement `_generate`,
|
|
16
|
+
`_agenerate` and `_llm_type` (`language_models/chat_models.py`).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
22
|
+
from typing import Any, NoReturn
|
|
23
|
+
|
|
24
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
25
|
+
from langchain_core.messages import AIMessage, BaseMessage
|
|
26
|
+
from langchain_core.outputs import ChatGeneration, ChatResult
|
|
27
|
+
from pydantic import JsonValue, PrivateAttr
|
|
28
|
+
|
|
29
|
+
from fivefold.recorder.cassette import (
|
|
30
|
+
Cassette,
|
|
31
|
+
Tool,
|
|
32
|
+
plain_object,
|
|
33
|
+
request_hash,
|
|
34
|
+
request_params,
|
|
35
|
+
response_message,
|
|
36
|
+
tool_schemas,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ReplayError(RuntimeError):
|
|
41
|
+
"""The run did not ask the model what the cassette recorded, so the run fails (0009).
|
|
42
|
+
|
|
43
|
+
`call` is the number of the model call where the replay failed, counted from 1 across
|
|
44
|
+
the run's workers. `recorded` and `asked` are the recorded request's hash and the
|
|
45
|
+
hash of the request the run made, when there are two to compare.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self, message: str, *, call: int, recorded: str | None = None, asked: str | None = None
|
|
50
|
+
) -> None:
|
|
51
|
+
super().__init__(message)
|
|
52
|
+
self.call = call
|
|
53
|
+
self.recorded = recorded
|
|
54
|
+
self.asked = asked
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class CassettePlayer:
|
|
58
|
+
"""Serves one cassette's calls, in order, to every agent worker of one run."""
|
|
59
|
+
|
|
60
|
+
def __init__(self, cassette: Cassette) -> None:
|
|
61
|
+
self.cassette = cassette
|
|
62
|
+
self._served = 0
|
|
63
|
+
self.failure: ReplayError | None = None
|
|
64
|
+
"""The first failure, if the replay failed."""
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def served(self) -> int:
|
|
68
|
+
"""How many calls the player has answered."""
|
|
69
|
+
return self._served
|
|
70
|
+
|
|
71
|
+
def answer(self, worker: str, request: str) -> AIMessage:
|
|
72
|
+
"""The response to the next recorded call, if `worker` asked it as recorded."""
|
|
73
|
+
if self.failure is not None:
|
|
74
|
+
raise self.failure
|
|
75
|
+
number = self._served + 1
|
|
76
|
+
calls = self.cassette.calls
|
|
77
|
+
if number > len(calls):
|
|
78
|
+
self._fail(
|
|
79
|
+
ReplayError(
|
|
80
|
+
f"model call {number} ({worker}): the cassette holds only {len(calls)}"
|
|
81
|
+
f" calls; the request's hash is {request}",
|
|
82
|
+
call=number,
|
|
83
|
+
asked=request,
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
recorded = calls[number - 1]
|
|
87
|
+
if recorded.request != request or recorded.worker != worker:
|
|
88
|
+
self._fail(
|
|
89
|
+
ReplayError(
|
|
90
|
+
f"model call {number}: {worker} asked a request with hash {request},"
|
|
91
|
+
f" but the cassette recorded {recorded.worker} asking one with hash"
|
|
92
|
+
f" {recorded.request}",
|
|
93
|
+
call=number,
|
|
94
|
+
recorded=recorded.request,
|
|
95
|
+
asked=request,
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
self._served = number
|
|
99
|
+
return response_message(recorded.response)
|
|
100
|
+
|
|
101
|
+
def finish(self) -> None:
|
|
102
|
+
"""Raise unless the run asked for every recorded call, each as it was recorded."""
|
|
103
|
+
if self.failure is not None:
|
|
104
|
+
raise self.failure
|
|
105
|
+
total = len(self.cassette.calls)
|
|
106
|
+
if self._served < total:
|
|
107
|
+
raise ReplayError(
|
|
108
|
+
f"the run made {self._served} model calls, but the cassette holds {total}:"
|
|
109
|
+
f" calls {self._served + 1} to {total} were never asked",
|
|
110
|
+
call=self._served + 1,
|
|
111
|
+
recorded=self.cassette.calls[self._served].request,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def _fail(self, error: ReplayError) -> NoReturn:
|
|
115
|
+
self.failure = error
|
|
116
|
+
raise error
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ReplayModel(BaseChatModel):
|
|
120
|
+
"""A chat model whose every reply is a cassette's next recorded response (0009)."""
|
|
121
|
+
|
|
122
|
+
_player: CassettePlayer = PrivateAttr()
|
|
123
|
+
_worker: str = PrivateAttr()
|
|
124
|
+
_think: Callable[[], Awaitable[None]] = PrivateAttr()
|
|
125
|
+
_tools: list[JsonValue] = PrivateAttr()
|
|
126
|
+
_with_tools: dict[str, JsonValue] = PrivateAttr()
|
|
127
|
+
|
|
128
|
+
def __init__(
|
|
129
|
+
self, player: CassettePlayer, *, worker: str, think: Callable[[], Awaitable[None]]
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Replay `player`'s cassette to the agent worker `worker`."""
|
|
132
|
+
super().__init__()
|
|
133
|
+
self._player = player
|
|
134
|
+
self._worker = worker
|
|
135
|
+
self._think = think
|
|
136
|
+
self._tools = []
|
|
137
|
+
self._with_tools = {}
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def _llm_type(self) -> str:
|
|
141
|
+
return "fivefold-replay"
|
|
142
|
+
|
|
143
|
+
def _generate(
|
|
144
|
+
self,
|
|
145
|
+
messages: list[BaseMessage],
|
|
146
|
+
stop: list[str] | None = None,
|
|
147
|
+
run_manager: Any = None,
|
|
148
|
+
**kwargs: Any,
|
|
149
|
+
) -> ChatResult:
|
|
150
|
+
raise NotImplementedError("the replay model runs on the scheduler's loop only")
|
|
151
|
+
|
|
152
|
+
async def _agenerate(
|
|
153
|
+
self,
|
|
154
|
+
messages: list[BaseMessage],
|
|
155
|
+
stop: list[str] | None = None,
|
|
156
|
+
run_manager: Any = None,
|
|
157
|
+
**kwargs: Any,
|
|
158
|
+
) -> ChatResult:
|
|
159
|
+
await self._think()
|
|
160
|
+
params = request_params(self._player.cassette.model, self._with_tools, stop, kwargs)
|
|
161
|
+
request = request_hash(messages, tools=self._tools, params=params)
|
|
162
|
+
reply = self._player.answer(self._worker, request)
|
|
163
|
+
return ChatResult(generations=[ChatGeneration(message=reply)])
|
|
164
|
+
|
|
165
|
+
def bind_tools(self, tools: Sequence[Tool], **kwargs: Any) -> ReplayModel:
|
|
166
|
+
"""Hash `tools` with every request, as the recording wrapper did."""
|
|
167
|
+
bound = ReplayModel(self._player, worker=self._worker, think=self._think)
|
|
168
|
+
bound._tools = tool_schemas(tools)
|
|
169
|
+
bound._with_tools = plain_object(kwargs, "what was bound with the tools")
|
|
170
|
+
return bound
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""The scripted model adapter (0009): the scripted policy behind a chat model interface.
|
|
2
|
+
|
|
3
|
+
Each model call replays the policy over the tool results in the conversation and replies
|
|
4
|
+
with the policy's next tool call, or with a closing message once the policy is done. The
|
|
5
|
+
policy holds no state of its own, so a worker resumed from its checkpoint makes the same
|
|
6
|
+
calls. Tool call ids are `call-0001`, `call-0002` and so on, so they are the same in
|
|
7
|
+
every run with the same results.
|
|
8
|
+
|
|
9
|
+
Every call takes `think_ms` of virtual time first (0009). The adapter spends no tokens.
|
|
10
|
+
|
|
11
|
+
Checked in langchain-core 1.8.0: `BaseChatModel` subclasses implement `_generate` and
|
|
12
|
+
`_llm_type`; `_agenerate` is the async path that `ainvoke` takes; `bind_tools` is the
|
|
13
|
+
hook a graph calls to attach tools, which a scripted model has no use for.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
23
|
+
from langchain_core.messages import AIMessage, BaseMessage, ToolMessage
|
|
24
|
+
from langchain_core.outputs import ChatGeneration, ChatResult
|
|
25
|
+
from pydantic import PrivateAttr
|
|
26
|
+
|
|
27
|
+
from fivefold.agents.policy import Policy, Result, ToolCall
|
|
28
|
+
|
|
29
|
+
DONE_TEXT = "Done."
|
|
30
|
+
"""The closing message once the policy has nothing left to call."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ScriptError(RuntimeError):
|
|
34
|
+
"""The conversation does not match the calls the policy made."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def call_id(number: int) -> str:
|
|
38
|
+
"""The id of the policy's `number`-th tool call."""
|
|
39
|
+
return f"call-{number:04d}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def next_call(policy: Policy, messages: Sequence[BaseMessage]) -> ToolCall | None:
|
|
43
|
+
"""Replay `policy` over the tool results in `messages`; return its next call."""
|
|
44
|
+
results = [message for message in messages if isinstance(message, ToolMessage)]
|
|
45
|
+
try:
|
|
46
|
+
call = next(policy)
|
|
47
|
+
for number, message in enumerate(results, start=1):
|
|
48
|
+
if message.tool_call_id != call_id(number):
|
|
49
|
+
raise ScriptError(f"result {number} answers {message.tool_call_id}")
|
|
50
|
+
if message.name is not None and message.name != call.name:
|
|
51
|
+
raise ScriptError(f"result {number} came from {message.name}, not {call.name}")
|
|
52
|
+
call = policy.send(_parse(message))
|
|
53
|
+
except StopIteration:
|
|
54
|
+
return None
|
|
55
|
+
return call
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _parse(message: ToolMessage) -> Result:
|
|
59
|
+
"""A tool result as the policy reads it: the tool's JSON, or the error text."""
|
|
60
|
+
content = message.content
|
|
61
|
+
text = content if isinstance(content, str) else json.dumps(content)
|
|
62
|
+
try:
|
|
63
|
+
parsed = json.loads(text)
|
|
64
|
+
except json.JSONDecodeError:
|
|
65
|
+
return {"error": text}
|
|
66
|
+
return parsed if isinstance(parsed, dict) else {"value": parsed}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ScriptedModel(BaseChatModel):
|
|
70
|
+
"""A chat model whose every reply is the scripted policy's next tool call."""
|
|
71
|
+
|
|
72
|
+
_make_policy: Callable[[], Policy] = PrivateAttr()
|
|
73
|
+
_think: Callable[[], Awaitable[None]] = PrivateAttr()
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self, make_policy: Callable[[], Policy], think: Callable[[], Awaitable[None]]
|
|
77
|
+
) -> None:
|
|
78
|
+
super().__init__()
|
|
79
|
+
self._make_policy = make_policy
|
|
80
|
+
self._think = think
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def _llm_type(self) -> str:
|
|
84
|
+
return "fivefold-scripted"
|
|
85
|
+
|
|
86
|
+
def _generate(
|
|
87
|
+
self,
|
|
88
|
+
messages: list[BaseMessage],
|
|
89
|
+
stop: list[str] | None = None,
|
|
90
|
+
run_manager: Any = None,
|
|
91
|
+
**kwargs: Any,
|
|
92
|
+
) -> ChatResult:
|
|
93
|
+
raise NotImplementedError("the scripted model runs on the scheduler's loop only")
|
|
94
|
+
|
|
95
|
+
async def _agenerate(
|
|
96
|
+
self,
|
|
97
|
+
messages: list[BaseMessage],
|
|
98
|
+
stop: list[str] | None = None,
|
|
99
|
+
run_manager: Any = None,
|
|
100
|
+
**kwargs: Any,
|
|
101
|
+
) -> ChatResult:
|
|
102
|
+
await self._think()
|
|
103
|
+
call = next_call(self._make_policy(), messages)
|
|
104
|
+
if call is None:
|
|
105
|
+
message = AIMessage(content=DONE_TEXT)
|
|
106
|
+
else:
|
|
107
|
+
number = sum(1 for m in messages if isinstance(m, ToolMessage)) + 1
|
|
108
|
+
message = AIMessage(
|
|
109
|
+
content="",
|
|
110
|
+
tool_calls=[
|
|
111
|
+
{
|
|
112
|
+
"name": call.name,
|
|
113
|
+
"args": dict(call.args),
|
|
114
|
+
"id": call_id(number),
|
|
115
|
+
"type": "tool_call",
|
|
116
|
+
}
|
|
117
|
+
],
|
|
118
|
+
)
|
|
119
|
+
return ChatResult(generations=[ChatGeneration(message=message)])
|
|
120
|
+
|
|
121
|
+
def bind_tools(self, tools: Any, **kwargs: Any) -> ScriptedModel:
|
|
122
|
+
return self
|