fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,333 @@
1
+ """The scripted agent over the network: a stand-in for your own agent (0008, 0009).
2
+
3
+ In the split topology the agent is the user's own. This is the shared LangGraph agent of
4
+ record 0008 with the scripted model, reaching a split run's services as any agent would:
5
+ in G1 and G2 the gateway's MCP endpoint over streamable HTTP, and in B0 the simulator's
6
+ HTTP API with the agent's credential. Use it to check a split setup before bringing your
7
+ own agent; the tests run the split topology end to end with it.
8
+
9
+ Its tools are the inproc agent's (`fivefold.agents.tools`), with the same timeouts: T plus
10
+ each tool's declared wait (0008). It runs on a `HostScheduler`, so the timeouts pass in
11
+ wall time. A model call takes no time: `think_ms` is virtual time the inproc scheduler
12
+ spends, and no clock here moves for it. It keeps its checkpoints in memory.
13
+
14
+ As in the inproc topology, a call whose connection to the gateway closes raises
15
+ `ConnectionError`, and the next call opens a new connection; a call with no answer within
16
+ its timeout raises `TransportTimeout`; and a server error raises `ServerError`. LangGraph's
17
+ default retry policy retries all three (0008). A call the gateway refuses because the
18
+ run's agents' phase is over raises `RunOver`, which it does not retry: the run has nothing
19
+ more for the agent.
20
+
21
+ python -m fivefold.agents.remote --scenario PATH --config G2 --contract C2 \\
22
+ --gateway-url http://gateway:8000/mcp
23
+ python -m fivefold.agents.remote --scenario PATH --config B0 --contract C2 \\
24
+ --simulator-url http://127.0.0.1:8100 --token-file split-run/agent-1.token
25
+
26
+ In G1 and G2 run it where your own agent would run: in a container on the Docker network
27
+ `fivefold-agent`, as the Compose service `agent` does, which reaches the gateway at
28
+ `http://gateway:8000/mcp` and nothing else (0008).
29
+
30
+ MCP SDK calls used (mcp 2.3.0): `mcp.client.streamable_http.streamable_http_client(url)`,
31
+ an async context manager that yields the read and write streams
32
+ (`mcp/client/streamable_http.py`); `ClientSession(read, write)` as an async context
33
+ manager, with `initialize`, `list_tools` and `call_tool`; `MCPError.code`; and
34
+ `mcp_types.CONNECTION_CLOSED` and `INTERNAL_ERROR`, the codes a call gets when its
35
+ connection closed and when the server answered with an error status. Checked in
36
+ langgraph-checkpoint 4.3.2: `langgraph.checkpoint.memory.InMemorySaver`. Checked in httpx
37
+ 0.28.1: `AsyncClient(base_url=, timeout=)`, `post(url, json=, headers=)` and `HTTPError`.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import argparse
43
+ import asyncio
44
+ import sys
45
+ from collections.abc import Awaitable, Sequence
46
+ from dataclasses import dataclass
47
+ from pathlib import Path
48
+ from typing import Any, Literal
49
+
50
+ import httpx
51
+ import mcp_types as types
52
+ from langgraph.checkpoint.memory import InMemorySaver
53
+ from mcp.client.session import ClientSession
54
+ from mcp.client.streamable_http import streamable_http_client
55
+ from mcp.shared.exceptions import MCPError
56
+
57
+ from fivefold.agents.connection import within
58
+ from fivefold.agents.policy import Mode, scripted_policy
59
+ from fivefold.agents.scripted import ScriptedModel
60
+ from fivefold.agents.tools import ServerError, Timeouts, baseline_tools, gateway_tools
61
+ from fivefold.agents.worker import WorkerEnd, WorkerSpec, build_graph, run_worker
62
+ from fivefold.domain.contracts import Capability, Contract
63
+ from fivefold.domain.scenario import Scenario, load_scenario_file
64
+ from fivefold.harness.clock import HostClock
65
+ from fivefold.harness.log import JsonValue
66
+ from fivefold.harness.scheduler import HostScheduler, Scheduler, SchedulerTimeout
67
+ from fivefold.harness.split import SERVICE_STOPPED
68
+ from fivefold.harness.transport import Hop, Reply, TransportTimeout
69
+
70
+ WORKER = "agent-1"
71
+ """The split run's one agent worker (0009)."""
72
+
73
+ Config = Literal["B0", "G1", "G2"]
74
+
75
+
76
+ class RunOver(RuntimeError):
77
+ """The gateway answered that the run's agents' phase is over, or that the run has
78
+ stopped (`fivefold.harness.split.SERVICE_STOPPED`): nothing the agent sends counts any
79
+ more. A `RuntimeError`, which LangGraph's default retry policy never retries
80
+ (langgraph 1.2.22, `langgraph/_internal/_retry.py`), so the worker ends at once."""
81
+
82
+
83
+ # G1 and G2: the gateway's MCP endpoint over streamable HTTP ------------------------------------
84
+
85
+
86
+ @dataclass
87
+ class _Session:
88
+ session: ClientSession
89
+ done: asyncio.Event
90
+ ended: asyncio.Event
91
+ holder: asyncio.Task[None]
92
+
93
+
94
+ class HttpGatewayConnection:
95
+ """An agent worker's MCP connection to the gateway over streamable HTTP: one session at
96
+ a time, and a new one after a connection closes."""
97
+
98
+ def __init__(self, scheduler: Scheduler, url: str, *, timeout_ms: int) -> None:
99
+ """`url` is the gateway's MCP endpoint; `timeout_ms` bounds each handshake request."""
100
+ self._scheduler = scheduler
101
+ self._url = url
102
+ self._timeout_ms = timeout_ms
103
+ self._current: _Session | None = None
104
+ self._opening = asyncio.Lock()
105
+
106
+ async def list_tools(self) -> types.ListToolsResult:
107
+ """The gateway's tools, from `tools/list`."""
108
+ session = await self._session()
109
+ return await self._ask(session.list_tools(), self._timeout_ms, "tools/list")
110
+
111
+ async def call_tool(
112
+ self, name: str, arguments: dict[str, Any], timeout_ms: int
113
+ ) -> types.CallToolResult:
114
+ """Call a tool, waiting at most `timeout_ms` for its result."""
115
+ session = await self._session()
116
+ return await self._ask(session.call_tool(name, arguments), timeout_ms, name)
117
+
118
+ async def _ask[T](self, call: Awaitable[T], timeout_ms: int, what: str) -> T:
119
+ try:
120
+ return await within(self._scheduler, call, timeout_ms, what)
121
+ except MCPError as error:
122
+ if error.code == SERVICE_STOPPED:
123
+ raise RunOver(f"{what}: {error.message}") from error
124
+ if error.code == types.CONNECTION_CLOSED:
125
+ self.close()
126
+ raise ConnectionError(f"{what}: the connection to the gateway closed") from error
127
+ if error.code == types.INTERNAL_ERROR:
128
+ raise ServerError(f"{what}: {error.message}") from error
129
+ raise
130
+
131
+ def close(self) -> None:
132
+ """End the current session, if there is one."""
133
+ current, self._current = self._current, None
134
+ if current is not None:
135
+ current.done.set()
136
+
137
+ async def _session(self) -> ClientSession:
138
+ async with self._opening:
139
+ current = self._current
140
+ if current is not None and not current.ended.is_set():
141
+ return current.session
142
+ self.close()
143
+ loop = asyncio.get_running_loop()
144
+ opened: asyncio.Future[ClientSession] = loop.create_future()
145
+ done, ended = asyncio.Event(), asyncio.Event()
146
+ holder = loop.create_task(self._hold(opened, done, ended), name="gateway session")
147
+ session = await opened
148
+ fresh = _Session(session, done, ended, holder)
149
+ self._current = fresh
150
+ try:
151
+ await within(self._scheduler, session.initialize(), self._timeout_ms, "initialize")
152
+ except BaseException as error:
153
+ if self._current is fresh:
154
+ self.close()
155
+ if isinstance(error, MCPError) and error.code == SERVICE_STOPPED:
156
+ raise RunOver(f"initialize: {error.message}") from error
157
+ raise
158
+ return session
159
+
160
+ async def _hold(
161
+ self, opened: asyncio.Future[ClientSession], done: asyncio.Event, ended: asyncio.Event
162
+ ) -> None:
163
+ """Keep one session open until it is no longer needed, or its connection fails."""
164
+ try:
165
+ async with (
166
+ streamable_http_client(self._url) as (read, write),
167
+ ClientSession(read, write) as session,
168
+ ):
169
+ opened.set_result(session)
170
+ await done.wait()
171
+ except Exception as error:
172
+ if not opened.done():
173
+ opened.set_exception(ConnectionError(f"cannot reach the gateway: {error}"))
174
+ finally:
175
+ ended.set()
176
+
177
+
178
+ # B0: the simulator's HTTP API ----------------------------------------------------------------
179
+
180
+
181
+ class HttpApi:
182
+ """The simulator's API over HTTP, with the agent's credential: B0's tools' transport
183
+ (`fivefold.harness.transport.Calls`). The report goes to the same service."""
184
+
185
+ def __init__(self, scheduler: Scheduler, client: httpx.AsyncClient, *, secret: str) -> None:
186
+ self._scheduler = scheduler
187
+ self._client = client
188
+ self._secret = secret
189
+
190
+ async def call(
191
+ self,
192
+ *,
193
+ hop: Hop,
194
+ sender: str,
195
+ receiver: str,
196
+ api: str,
197
+ body: JsonValue,
198
+ timeout_ms: int,
199
+ intent_id: str | None = None,
200
+ idempotency_key: str | None = None,
201
+ sent_vt: int | None = None,
202
+ ) -> Reply:
203
+ """Send one call; wait at most `timeout_ms` for its answer.
204
+
205
+ Raises `TransportTimeout` with no answer in time, `ConnectionError` when the
206
+ connection fails, and `ServerError` for a 5xx status. `sent_vt` changes nothing:
207
+ the simulator logs the agent's calls as they arrive, and the agent's tools record
208
+ no send time.
209
+ """
210
+ headers = {"authorization": f"Bearer {self._secret}"}
211
+ try:
212
+ response = await self._scheduler.wait_for(
213
+ self._client.post(f"/api/{api}", json=body, headers=headers), timeout_ms
214
+ )
215
+ except SchedulerTimeout:
216
+ raise TransportTimeout(f"{api}: no answer within {timeout_ms} ms") from None
217
+ except httpx.HTTPError as error:
218
+ raise ConnectionError(
219
+ f"{api}: the connection to the simulator failed: {error}"
220
+ ) from None
221
+ if response.status_code >= 500:
222
+ raise ServerError(f"{api}: the simulator answered {response.status_code}")
223
+ return Reply(response.json())
224
+
225
+
226
+ # The agent -----------------------------------------------------------------------------------
227
+
228
+
229
+ @dataclass(frozen=True)
230
+ class RemoteAgent:
231
+ """The scripted agent of one split run, and where it reaches the run's services."""
232
+
233
+ scenario: Scenario
234
+ config: Config
235
+ contract: Contract
236
+ gateway_url: str | None = None
237
+ """The gateway's MCP endpoint, in G1 and G2."""
238
+ simulator_url: str | None = None
239
+ """The simulator's service, in B0."""
240
+ secret: str | None = None
241
+ """The agent's credential, in B0."""
242
+
243
+
244
+ async def run_remote_agent(agent: RemoteAgent) -> tuple[WorkerEnd, str]:
245
+ """Run the scripted agent to its end; return how it ended (`run_worker`)."""
246
+ scenario = agent.scenario
247
+ scheduler = HostScheduler(HostClock(HostClock.host_ms()))
248
+ params = scenario.params
249
+ timeouts = Timeouts(
250
+ t_ms=params.dispatch_timeout_ms,
251
+ tool_wait_ms=params.tool_wait_ms,
252
+ max_wait_ms=params.max_wait_ms,
253
+ )
254
+ gateway = agent.config != "B0"
255
+ mode = Mode.GATEWAY if gateway else Mode.BASELINE
256
+
257
+ def think() -> Any:
258
+ return scheduler.sleep(0)
259
+
260
+ model = ScriptedModel(
261
+ lambda: scripted_policy(
262
+ scenario.intents, mode=mode, rank=0, max_wait_ms=params.max_wait_ms
263
+ ),
264
+ think,
265
+ )
266
+ budget = scenario.budget
267
+ spec = WorkerSpec(WORKER, scenario.task, budget.max_agent_steps, budget.max_tokens)
268
+ if gateway:
269
+ if agent.gateway_url is None:
270
+ raise ValueError(f"{agent.config} reaches the gateway: give its MCP endpoint")
271
+ connection = HttpGatewayConnection(
272
+ scheduler, agent.gateway_url, timeout_ms=params.dispatch_timeout_ms
273
+ )
274
+ try:
275
+ tools = gateway_tools(connection, await connection.list_tools(), timeouts)
276
+ graph = build_graph(
277
+ model,
278
+ tools,
279
+ InMemorySaver(),
280
+ max_steps=spec.max_steps,
281
+ max_tokens=spec.max_tokens,
282
+ )
283
+ return await run_worker(graph, spec, resume=False)
284
+ finally:
285
+ connection.close()
286
+ if agent.simulator_url is None or agent.secret is None:
287
+ raise ValueError("B0 reaches the simulator: give its address and the agent's credential")
288
+ async with httpx.AsyncClient(base_url=agent.simulator_url, timeout=None) as client:
289
+ api = HttpApi(scheduler, client, secret=agent.secret)
290
+ tools = baseline_tools(
291
+ api, scheduler, timeouts, worker=WORKER, status=agent.contract.has(Capability.STATUS)
292
+ )
293
+ graph = build_graph(
294
+ model, tools, InMemorySaver(), max_steps=spec.max_steps, max_tokens=spec.max_tokens
295
+ )
296
+ return await run_worker(graph, spec, resume=False)
297
+
298
+
299
+ def main(argv: Sequence[str] | None = None) -> int:
300
+ """Command-line entry point. Returns the process exit status."""
301
+ parser = argparse.ArgumentParser(
302
+ prog="python -m fivefold.agents.remote",
303
+ description="Run the scripted agent against a split run's services.",
304
+ )
305
+ parser.add_argument("--scenario", type=Path, required=True, help="the run's scenario file")
306
+ parser.add_argument("--config", choices=["B0", "G1", "G2"], required=True)
307
+ parser.add_argument("--contract", choices=[str(level) for level in Contract], required=True)
308
+ parser.add_argument("--gateway-url", help="the gateway's MCP endpoint (G1, G2)")
309
+ parser.add_argument("--simulator-url", help="the simulator's service (B0)")
310
+ parser.add_argument("--token-file", type=Path, help="the agent's credential (B0)")
311
+ args = parser.parse_args(argv)
312
+ secret = None
313
+ if args.token_file is not None:
314
+ secret = args.token_file.read_text(encoding="utf-8").strip()
315
+ agent = RemoteAgent(
316
+ scenario=load_scenario_file(args.scenario),
317
+ config=args.config,
318
+ contract=Contract(args.contract),
319
+ gateway_url=args.gateway_url,
320
+ simulator_url=args.simulator_url,
321
+ secret=secret,
322
+ )
323
+ try:
324
+ end, detail = asyncio.run(run_remote_agent(agent))
325
+ except ValueError as error:
326
+ print(f"agent: {error}", file=sys.stderr)
327
+ return 1
328
+ print(f"{WORKER}: {end} ({detail})")
329
+ return 0 if end is WorkerEnd.FINISHED else 1
330
+
331
+
332
+ if __name__ == "__main__":
333
+ raise SystemExit(main())
@@ -0,0 +1,170 @@
1
+ """The replay adapter (0009): serves a cassette's responses in order.
2
+
3
+ All of a run's replay models share one `CassettePlayer`, which hands out the cassette's
4
+ calls in the order the recorded run made them. Each model call first takes `think_ms` of
5
+ virtual time, as with every adapter (0009). It then hashes the request as the recorder did
6
+ (`fivefold.recorder.cassette`). If the next recorded call has that hash and was made by
7
+ the same agent worker, the call answers with the recorded response.
8
+
9
+ Otherwise the replay has failed. The call raises `ReplayError`, naming the call's number
10
+ and both hashes, and so does every later call. The same happens when the run asks for more
11
+ calls than the cassette holds. `CassettePlayer.finish`, once the run's agents are done,
12
+ raises the first failure, or a `ReplayError` if the run left recorded calls unasked: the
13
+ run fails (0009). The adapter makes no network call and spends no tokens.
14
+
15
+ Checked in langchain-core 1.8.0: `BaseChatModel` subclasses implement `_generate`,
16
+ `_agenerate` and `_llm_type` (`language_models/chat_models.py`).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Awaitable, Callable, Sequence
22
+ from typing import Any, NoReturn
23
+
24
+ from langchain_core.language_models.chat_models import BaseChatModel
25
+ from langchain_core.messages import AIMessage, BaseMessage
26
+ from langchain_core.outputs import ChatGeneration, ChatResult
27
+ from pydantic import JsonValue, PrivateAttr
28
+
29
+ from fivefold.recorder.cassette import (
30
+ Cassette,
31
+ Tool,
32
+ plain_object,
33
+ request_hash,
34
+ request_params,
35
+ response_message,
36
+ tool_schemas,
37
+ )
38
+
39
+
40
+ class ReplayError(RuntimeError):
41
+ """The run did not ask the model what the cassette recorded, so the run fails (0009).
42
+
43
+ `call` is the number of the model call where the replay failed, counted from 1 across
44
+ the run's workers. `recorded` and `asked` are the recorded request's hash and the
45
+ hash of the request the run made, when there are two to compare.
46
+ """
47
+
48
+ def __init__(
49
+ self, message: str, *, call: int, recorded: str | None = None, asked: str | None = None
50
+ ) -> None:
51
+ super().__init__(message)
52
+ self.call = call
53
+ self.recorded = recorded
54
+ self.asked = asked
55
+
56
+
57
+ class CassettePlayer:
58
+ """Serves one cassette's calls, in order, to every agent worker of one run."""
59
+
60
+ def __init__(self, cassette: Cassette) -> None:
61
+ self.cassette = cassette
62
+ self._served = 0
63
+ self.failure: ReplayError | None = None
64
+ """The first failure, if the replay failed."""
65
+
66
+ @property
67
+ def served(self) -> int:
68
+ """How many calls the player has answered."""
69
+ return self._served
70
+
71
+ def answer(self, worker: str, request: str) -> AIMessage:
72
+ """The response to the next recorded call, if `worker` asked it as recorded."""
73
+ if self.failure is not None:
74
+ raise self.failure
75
+ number = self._served + 1
76
+ calls = self.cassette.calls
77
+ if number > len(calls):
78
+ self._fail(
79
+ ReplayError(
80
+ f"model call {number} ({worker}): the cassette holds only {len(calls)}"
81
+ f" calls; the request's hash is {request}",
82
+ call=number,
83
+ asked=request,
84
+ )
85
+ )
86
+ recorded = calls[number - 1]
87
+ if recorded.request != request or recorded.worker != worker:
88
+ self._fail(
89
+ ReplayError(
90
+ f"model call {number}: {worker} asked a request with hash {request},"
91
+ f" but the cassette recorded {recorded.worker} asking one with hash"
92
+ f" {recorded.request}",
93
+ call=number,
94
+ recorded=recorded.request,
95
+ asked=request,
96
+ )
97
+ )
98
+ self._served = number
99
+ return response_message(recorded.response)
100
+
101
+ def finish(self) -> None:
102
+ """Raise unless the run asked for every recorded call, each as it was recorded."""
103
+ if self.failure is not None:
104
+ raise self.failure
105
+ total = len(self.cassette.calls)
106
+ if self._served < total:
107
+ raise ReplayError(
108
+ f"the run made {self._served} model calls, but the cassette holds {total}:"
109
+ f" calls {self._served + 1} to {total} were never asked",
110
+ call=self._served + 1,
111
+ recorded=self.cassette.calls[self._served].request,
112
+ )
113
+
114
+ def _fail(self, error: ReplayError) -> NoReturn:
115
+ self.failure = error
116
+ raise error
117
+
118
+
119
+ class ReplayModel(BaseChatModel):
120
+ """A chat model whose every reply is a cassette's next recorded response (0009)."""
121
+
122
+ _player: CassettePlayer = PrivateAttr()
123
+ _worker: str = PrivateAttr()
124
+ _think: Callable[[], Awaitable[None]] = PrivateAttr()
125
+ _tools: list[JsonValue] = PrivateAttr()
126
+ _with_tools: dict[str, JsonValue] = PrivateAttr()
127
+
128
+ def __init__(
129
+ self, player: CassettePlayer, *, worker: str, think: Callable[[], Awaitable[None]]
130
+ ) -> None:
131
+ """Replay `player`'s cassette to the agent worker `worker`."""
132
+ super().__init__()
133
+ self._player = player
134
+ self._worker = worker
135
+ self._think = think
136
+ self._tools = []
137
+ self._with_tools = {}
138
+
139
+ @property
140
+ def _llm_type(self) -> str:
141
+ return "fivefold-replay"
142
+
143
+ def _generate(
144
+ self,
145
+ messages: list[BaseMessage],
146
+ stop: list[str] | None = None,
147
+ run_manager: Any = None,
148
+ **kwargs: Any,
149
+ ) -> ChatResult:
150
+ raise NotImplementedError("the replay model runs on the scheduler's loop only")
151
+
152
+ async def _agenerate(
153
+ self,
154
+ messages: list[BaseMessage],
155
+ stop: list[str] | None = None,
156
+ run_manager: Any = None,
157
+ **kwargs: Any,
158
+ ) -> ChatResult:
159
+ await self._think()
160
+ params = request_params(self._player.cassette.model, self._with_tools, stop, kwargs)
161
+ request = request_hash(messages, tools=self._tools, params=params)
162
+ reply = self._player.answer(self._worker, request)
163
+ return ChatResult(generations=[ChatGeneration(message=reply)])
164
+
165
+ def bind_tools(self, tools: Sequence[Tool], **kwargs: Any) -> ReplayModel:
166
+ """Hash `tools` with every request, as the recording wrapper did."""
167
+ bound = ReplayModel(self._player, worker=self._worker, think=self._think)
168
+ bound._tools = tool_schemas(tools)
169
+ bound._with_tools = plain_object(kwargs, "what was bound with the tools")
170
+ return bound
@@ -0,0 +1,122 @@
1
+ """The scripted model adapter (0009): the scripted policy behind a chat model interface.
2
+
3
+ Each model call replays the policy over the tool results in the conversation and replies
4
+ with the policy's next tool call, or with a closing message once the policy is done. The
5
+ policy holds no state of its own, so a worker resumed from its checkpoint makes the same
6
+ calls. Tool call ids are `call-0001`, `call-0002` and so on, so they are the same in
7
+ every run with the same results.
8
+
9
+ Every call takes `think_ms` of virtual time first (0009). The adapter spends no tokens.
10
+
11
+ Checked in langchain-core 1.8.0: `BaseChatModel` subclasses implement `_generate` and
12
+ `_llm_type`; `_agenerate` is the async path that `ainvoke` takes; `bind_tools` is the
13
+ hook a graph calls to attach tools, which a scripted model has no use for.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from collections.abc import Awaitable, Callable, Sequence
20
+ from typing import Any
21
+
22
+ from langchain_core.language_models.chat_models import BaseChatModel
23
+ from langchain_core.messages import AIMessage, BaseMessage, ToolMessage
24
+ from langchain_core.outputs import ChatGeneration, ChatResult
25
+ from pydantic import PrivateAttr
26
+
27
+ from fivefold.agents.policy import Policy, Result, ToolCall
28
+
29
+ DONE_TEXT = "Done."
30
+ """The closing message once the policy has nothing left to call."""
31
+
32
+
33
+ class ScriptError(RuntimeError):
34
+ """The conversation does not match the calls the policy made."""
35
+
36
+
37
+ def call_id(number: int) -> str:
38
+ """The id of the policy's `number`-th tool call."""
39
+ return f"call-{number:04d}"
40
+
41
+
42
+ def next_call(policy: Policy, messages: Sequence[BaseMessage]) -> ToolCall | None:
43
+ """Replay `policy` over the tool results in `messages`; return its next call."""
44
+ results = [message for message in messages if isinstance(message, ToolMessage)]
45
+ try:
46
+ call = next(policy)
47
+ for number, message in enumerate(results, start=1):
48
+ if message.tool_call_id != call_id(number):
49
+ raise ScriptError(f"result {number} answers {message.tool_call_id}")
50
+ if message.name is not None and message.name != call.name:
51
+ raise ScriptError(f"result {number} came from {message.name}, not {call.name}")
52
+ call = policy.send(_parse(message))
53
+ except StopIteration:
54
+ return None
55
+ return call
56
+
57
+
58
+ def _parse(message: ToolMessage) -> Result:
59
+ """A tool result as the policy reads it: the tool's JSON, or the error text."""
60
+ content = message.content
61
+ text = content if isinstance(content, str) else json.dumps(content)
62
+ try:
63
+ parsed = json.loads(text)
64
+ except json.JSONDecodeError:
65
+ return {"error": text}
66
+ return parsed if isinstance(parsed, dict) else {"value": parsed}
67
+
68
+
69
+ class ScriptedModel(BaseChatModel):
70
+ """A chat model whose every reply is the scripted policy's next tool call."""
71
+
72
+ _make_policy: Callable[[], Policy] = PrivateAttr()
73
+ _think: Callable[[], Awaitable[None]] = PrivateAttr()
74
+
75
+ def __init__(
76
+ self, make_policy: Callable[[], Policy], think: Callable[[], Awaitable[None]]
77
+ ) -> None:
78
+ super().__init__()
79
+ self._make_policy = make_policy
80
+ self._think = think
81
+
82
+ @property
83
+ def _llm_type(self) -> str:
84
+ return "fivefold-scripted"
85
+
86
+ def _generate(
87
+ self,
88
+ messages: list[BaseMessage],
89
+ stop: list[str] | None = None,
90
+ run_manager: Any = None,
91
+ **kwargs: Any,
92
+ ) -> ChatResult:
93
+ raise NotImplementedError("the scripted model runs on the scheduler's loop only")
94
+
95
+ async def _agenerate(
96
+ self,
97
+ messages: list[BaseMessage],
98
+ stop: list[str] | None = None,
99
+ run_manager: Any = None,
100
+ **kwargs: Any,
101
+ ) -> ChatResult:
102
+ await self._think()
103
+ call = next_call(self._make_policy(), messages)
104
+ if call is None:
105
+ message = AIMessage(content=DONE_TEXT)
106
+ else:
107
+ number = sum(1 for m in messages if isinstance(m, ToolMessage)) + 1
108
+ message = AIMessage(
109
+ content="",
110
+ tool_calls=[
111
+ {
112
+ "name": call.name,
113
+ "args": dict(call.args),
114
+ "id": call_id(number),
115
+ "type": "tool_call",
116
+ }
117
+ ],
118
+ )
119
+ return ChatResult(generations=[ChatGeneration(message=message)])
120
+
121
+ def bind_tools(self, tools: Any, **kwargs: Any) -> ScriptedModel:
122
+ return self