fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
fivefold/harness/run.py
ADDED
|
@@ -0,0 +1,881 @@
|
|
|
1
|
+
"""One run in the inproc topology: every component on one scheduler (0009).
|
|
2
|
+
|
|
3
|
+
A run is a scenario, a configuration, a contract level, a seed and a model (0009). This
|
|
4
|
+
build runs B0, G1 and G2 with the scripted model, recorded into a cassette or not, with a
|
|
5
|
+
cassette's replay, with a live model, always recorded, or as an experimental branch of a
|
|
6
|
+
recorded run, which replays the parent's first K model calls and then switches to the
|
|
7
|
+
scripted model or a live one, also always recorded (`fivefold.harness.models`). A branch
|
|
8
|
+
runs its parent's run: the same scenario, configuration, contract level, seed, fault plan
|
|
9
|
+
and workers. It is a new run all the same, with its own run id, the parent's with
|
|
10
|
+
`.branch-K` added, and `harness.run` names its parent and its branch point.
|
|
11
|
+
|
|
12
|
+
The run's fault plan is fixed before anything runs (`run_plan`): the run's own plan, or
|
|
13
|
+
else the scenario's, is a list of faults, checked against the run, or a random plan,
|
|
14
|
+
from which the run's `faults` generator draws such a list (0006, Plans;
|
|
15
|
+
`fivefold.faults.plan`). Either way, the list, with the parameters drawn from the seed,
|
|
16
|
+
is what fires and what `harness.run` records, so playback, branches and the report see
|
|
17
|
+
the faults the run had.
|
|
18
|
+
|
|
19
|
+
`InprocRun.execute`:
|
|
20
|
+
|
|
21
|
+
1. records the run, in its run database and in the run registry (0012,
|
|
22
|
+
`fivefold.harness.registry`), sets up the simulator for the contract with the
|
|
23
|
+
configuration's credentials (only the gateway in G1 and G2, the agent workers in B0;
|
|
24
|
+
0008), and serves the simulator's API and, in B0, the report sink on the transport;
|
|
25
|
+
2. in G1 and G2, starts the gateway: its store, its dispatcher after startup recovery, and
|
|
26
|
+
its tools, one MCP server per agent connection. G1's store has action identity off,
|
|
27
|
+
which makes the gateway authorization only (0008);
|
|
28
|
+
3. starts each agent worker with its own checkpoint thread, and the scripted approver,
|
|
29
|
+
which decides queued approval requests by the scenario's rules (0002, rule 7). F9
|
|
30
|
+
starts one more worker on the same task when it fires; the gateway expects a report
|
|
31
|
+
from every worker the plan can start (0010);
|
|
32
|
+
4. fires the plan's faults: the transport fires those on its hops, the dispatcher those
|
|
33
|
+
at its points (F1, F6 and F7 early), and this module those at the agent's points (F1's
|
|
34
|
+
B0 equivalent, F11). A crashed gateway restarts through startup recovery (0003), and
|
|
35
|
+
each agent worker then opens a new MCP connection to it; a crashed worker restarts
|
|
36
|
+
from its last checkpoint. A component that a crash stopped reaches no further fault
|
|
37
|
+
point, at the gateway's points and the agent's alike: when two crash faults meet at
|
|
38
|
+
one event, only the first fires, and the component restarts after its delay. The
|
|
39
|
+
simulator's endpoint fires F5 and F10 at the simulator's points
|
|
40
|
+
(`fivefold.harness.sim_points`). Each fault's record names the task intent it
|
|
41
|
+
touched, which recovery latency starts from (0014);
|
|
42
|
+
5. runs until the agents' phase ends: every worker finished, used up its step budget or
|
|
43
|
+
failed;
|
|
44
|
+
6. in G2, closes the run if the last report did not (0003); then keeps everything but
|
|
45
|
+
the agents running for the drain window, `2 x (D + max_hop_latency) + L` (0009),
|
|
46
|
+
records the stop time, and a recording's cassette digest, and stops every component.
|
|
47
|
+
A replay that did not serve exactly its cassette's calls then fails the run with
|
|
48
|
+
`ReplayError` (0009);
|
|
49
|
+
7. runs the evaluator, which scores the run from the simulator's and the harness's
|
|
50
|
+
records and writes its result to the `eval` schema (0004, 0009).
|
|
51
|
+
|
|
52
|
+
When the drain ends, before the stop is recorded, the harness has the simulator abort
|
|
53
|
+
every write still in flight past its deadline, unless a fault waived the deadline (F5
|
|
54
|
+
beyond the deadline): such a write never commits, and the ledger then says so (0005).
|
|
55
|
+
|
|
56
|
+
In G modes, when the scenario has a scripted resolver (0009), the gateway's dispatcher
|
|
57
|
+
tells it when an action is left to a human, and it attests what the simulator's ledger
|
|
58
|
+
shows through the gateway's admin path (`fivefold.harness.actors`, 0003).
|
|
59
|
+
|
|
60
|
+
Each component connects to the run database as its own role: `ff_sim`, `ff_gw`,
|
|
61
|
+
`ff_harness`, `ff_agent` and `ff_eval` (0008). The registry's row is written as
|
|
62
|
+
`ff_registry`, in the registry's own database on the same server.
|
|
63
|
+
|
|
64
|
+
Every model call of every worker goes into the harness's model-call log once the model
|
|
65
|
+
answers, with its virtual times, the run's model and the reply's token usage (0004). Each
|
|
66
|
+
worker has the scenario's step and token budgets (0009, `fivefold.agents.worker`).
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
from __future__ import annotations
|
|
70
|
+
|
|
71
|
+
import asyncio
|
|
72
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
73
|
+
from dataclasses import dataclass, field
|
|
74
|
+
from typing import Any, Literal
|
|
75
|
+
|
|
76
|
+
import psycopg
|
|
77
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
78
|
+
from langchain_core.messages import AIMessage
|
|
79
|
+
from psycopg.conninfo import make_conninfo
|
|
80
|
+
|
|
81
|
+
from fivefold.agents.checkpoint import InlinePostgresSaver, connect
|
|
82
|
+
from fivefold.agents.connection import GatewayConnection
|
|
83
|
+
from fivefold.agents.live import Think, refusal
|
|
84
|
+
from fivefold.agents.policy import Mode, scripted_policy
|
|
85
|
+
from fivefold.agents.scripted import ScriptedModel
|
|
86
|
+
from fivefold.agents.tools import Timeouts, baseline_tools, gateway_tools
|
|
87
|
+
from fivefold.agents.worker import CallLog, WorkerEnd, WorkerSpec, build_graph, run_worker
|
|
88
|
+
from fivefold.domain.contracts import Capability, Contract
|
|
89
|
+
from fivefold.domain.scenario import Scenario, scenario_digest
|
|
90
|
+
from fivefold.evaluator import RunResult as Evaluation
|
|
91
|
+
from fivefold.evaluator import evaluate_run
|
|
92
|
+
from fivefold.faults.catalog import Action
|
|
93
|
+
from fivefold.faults.plan import (
|
|
94
|
+
FaultInjector,
|
|
95
|
+
Fired,
|
|
96
|
+
PlannedFault,
|
|
97
|
+
check_configuration,
|
|
98
|
+
check_run,
|
|
99
|
+
draw_plan,
|
|
100
|
+
parse_plan,
|
|
101
|
+
parse_random_plan,
|
|
102
|
+
seed_parameters,
|
|
103
|
+
)
|
|
104
|
+
from fivefold.gateway.admin import GatewayAdmin
|
|
105
|
+
from fivefold.gateway.dispatcher import Dispatcher, Timing
|
|
106
|
+
from fivefold.gateway.freshness import RuleFreshness
|
|
107
|
+
from fivefold.gateway.pg_store import PgStore
|
|
108
|
+
from fivefold.gateway.server import serve
|
|
109
|
+
from fivefold.gateway.sim_client import SimClient
|
|
110
|
+
from fivefold.gateway.state_machine import GatewaySettings
|
|
111
|
+
from fivefold.gateway.tools import GatewayTools, ToolTiming
|
|
112
|
+
from fivefold.harness.actors import PgEffectsLedger, ScriptedApprover, ScriptedResolver
|
|
113
|
+
from fivefold.harness.clock import settle, wall_ms
|
|
114
|
+
from fivefold.harness.fault_effects import SimulatorFaults, message_context
|
|
115
|
+
from fivefold.harness.log import FaultRecord, ModelCallRecord, PgHarnessLog, RunRecord
|
|
116
|
+
from fivefold.harness.mcp_link import Link
|
|
117
|
+
from fivefold.harness.models import (
|
|
118
|
+
SCRIPTED,
|
|
119
|
+
Branch,
|
|
120
|
+
ModelSpec,
|
|
121
|
+
Parent,
|
|
122
|
+
RunModels,
|
|
123
|
+
branch_run_id,
|
|
124
|
+
parent_of,
|
|
125
|
+
)
|
|
126
|
+
from fivefold.harness.registry import RegistryEntry, database_of, register
|
|
127
|
+
from fivefold.harness.reports import ReportSink, sink_endpoint
|
|
128
|
+
from fivefold.harness.rng import Generators
|
|
129
|
+
from fivefold.harness.scheduler import Scheduler
|
|
130
|
+
from fivefold.harness.sim_points import SimulatorPoints
|
|
131
|
+
from fivefold.harness.transport import Envelope, Outgoing, Stage, Transport
|
|
132
|
+
from fivefold.recorder.cassette import RunHeader, plain_object
|
|
133
|
+
from fivefold.sim import SimConfig, Simulator
|
|
134
|
+
from fivefold.sim.endpoint import endpoint
|
|
135
|
+
|
|
136
|
+
Config = Literal["B0", "G1", "G2"]
|
|
137
|
+
|
|
138
|
+
GATEWAY_CONFIGS = ("G1", "G2")
|
|
139
|
+
"""The configurations that write through the gateway (0008). G1 is authorization only."""
|
|
140
|
+
|
|
141
|
+
CONFIGS: tuple[Config, ...] = ("B0", "G1", "G2")
|
|
142
|
+
"""Every configuration a run can have (0008)."""
|
|
143
|
+
|
|
144
|
+
GATEWAY = "gateway"
|
|
145
|
+
SIMULATOR = "simulator"
|
|
146
|
+
REPORT_SINK = "report_sink"
|
|
147
|
+
|
|
148
|
+
POSSIBLY_SENT_POINT = "gw.dispatch.after_record"
|
|
149
|
+
"""A gateway crash here leaves a request that may or may not have left (0006, crashes)."""
|
|
150
|
+
|
|
151
|
+
VOIDING_POINT = "gw.dispatch.after_answer"
|
|
152
|
+
"""A gateway crash here voids the answer it just received (0006, crashes)."""
|
|
153
|
+
|
|
154
|
+
GATEWAY_TOOL_NAMES = frozenset(
|
|
155
|
+
{
|
|
156
|
+
"get_quote",
|
|
157
|
+
"list_suppliers",
|
|
158
|
+
"get_budget",
|
|
159
|
+
"get_inventory",
|
|
160
|
+
"submit_purchase_order",
|
|
161
|
+
"reserve_inventory",
|
|
162
|
+
"get_action_status",
|
|
163
|
+
"submit_final_report",
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
BASELINE_TOOL_NAMES = frozenset(
|
|
167
|
+
{
|
|
168
|
+
"get_quote",
|
|
169
|
+
"list_suppliers",
|
|
170
|
+
"get_budget",
|
|
171
|
+
"get_inventory",
|
|
172
|
+
"request_approval",
|
|
173
|
+
"get_approval_status",
|
|
174
|
+
"submit_purchase_order",
|
|
175
|
+
"reserve_inventory",
|
|
176
|
+
"get_order_status",
|
|
177
|
+
"get_reservation_status",
|
|
178
|
+
"submit_final_report",
|
|
179
|
+
}
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
@dataclass(frozen=True)
|
|
184
|
+
class RunSpec:
|
|
185
|
+
"""What to run: `plan` defaults to the scenario's fault plan. A plan is a list of
|
|
186
|
+
faults, or a random plan, `{random: {from: [...], max: N}}`, which the run draws from
|
|
187
|
+
its seed (`run_plan`)."""
|
|
188
|
+
|
|
189
|
+
scenario: Scenario
|
|
190
|
+
config: Config
|
|
191
|
+
contract: Contract
|
|
192
|
+
seed: int
|
|
193
|
+
plan: Sequence[Mapping[str, Any]] | Mapping[str, Any] | None = None
|
|
194
|
+
workers: int = 1
|
|
195
|
+
model: ModelSpec = SCRIPTED
|
|
196
|
+
"""The model adapter: scripted, scripted with recording, replay, live, or a branch's
|
|
197
|
+
(0009)."""
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def parent(self) -> Parent | None:
|
|
201
|
+
"""For an experimental branch, or a replay of one, the run it branched from and
|
|
202
|
+
its branch point (0009); None for any other run."""
|
|
203
|
+
return parent_of(self.model)
|
|
204
|
+
|
|
205
|
+
@property
|
|
206
|
+
def run_id(self) -> str:
|
|
207
|
+
"""A readable id that names the run's choices. A branch's is its parent's run id
|
|
208
|
+
with `.branch-K` added, K being its branch point (`branch_run_id`)."""
|
|
209
|
+
parent = self.parent
|
|
210
|
+
if parent is not None:
|
|
211
|
+
return branch_run_id(parent)
|
|
212
|
+
return f"{self.scenario.id}.{self.config}.{self.contract}.{self.seed}"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@dataclass
|
|
216
|
+
class RunResult:
|
|
217
|
+
"""How a run ended: each worker's end, the virtual time the run stopped, and the
|
|
218
|
+
evaluator's result."""
|
|
219
|
+
|
|
220
|
+
run_id: str
|
|
221
|
+
ends: dict[str, tuple[WorkerEnd, str]] = field(default_factory=dict)
|
|
222
|
+
stopped_vt: int = 0
|
|
223
|
+
evaluation: Evaluation | None = None
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def drain_window_ms(scenario: Scenario) -> int:
|
|
227
|
+
"""`2 x (D + max_hop_latency) + L` (0009)."""
|
|
228
|
+
params = scenario.params
|
|
229
|
+
return 2 * (params.commit_deadline_ms + params.max_hop_latency_ms) + params.status_lag_ms
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def run_plan(spec: RunSpec) -> tuple[PlannedFault, ...]:
|
|
233
|
+
"""The run's fault plan, as it fires and as `harness.run` records it (0006, Plans).
|
|
234
|
+
|
|
235
|
+
The run's own plan, or else the scenario's, is either a list of faults, which must
|
|
236
|
+
fit the run's configuration, timing and contract level, or a random plan, from which
|
|
237
|
+
the run's `faults` generator draws such a list (`fivefold.faults.plan.draw_plan`).
|
|
238
|
+
Then the same generator draws each seeded parameter the list leaves unset (0009). So
|
|
239
|
+
a seed always gives the same plan, and a list with no random part draws what it
|
|
240
|
+
always did. Raises `ValueError` for a plan the run cannot carry out.
|
|
241
|
+
"""
|
|
242
|
+
faults = spec.plan if spec.plan is not None else spec.scenario.faults
|
|
243
|
+
draws = Generators(spec.seed).get("faults")
|
|
244
|
+
if isinstance(faults, Mapping):
|
|
245
|
+
plan = draw_plan(
|
|
246
|
+
parse_random_plan(faults),
|
|
247
|
+
draws,
|
|
248
|
+
scenario=spec.scenario,
|
|
249
|
+
config=spec.config,
|
|
250
|
+
contract=spec.contract,
|
|
251
|
+
)
|
|
252
|
+
else:
|
|
253
|
+
plan = parse_plan(faults)
|
|
254
|
+
check_configuration(plan, gateway=spec.config != "B0")
|
|
255
|
+
check_run(plan, params=spec.scenario.params, contract=spec.contract)
|
|
256
|
+
return seed_parameters(plan, draws)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class InprocRun:
|
|
260
|
+
"""One run of a scenario on one scheduler, in a migrated run database."""
|
|
261
|
+
|
|
262
|
+
def __init__(self, spec: RunSpec, dsn: str) -> None:
|
|
263
|
+
"""`dsn` names the run database; each component connects with its own role."""
|
|
264
|
+
if spec.config not in ("B0", *GATEWAY_CONFIGS):
|
|
265
|
+
raise ValueError(f"this build runs B0, G1 and G2, not {spec.config}")
|
|
266
|
+
self.spec = spec
|
|
267
|
+
self._plan = run_plan(spec)
|
|
268
|
+
self._dsn = dsn
|
|
269
|
+
self.scheduler = Scheduler()
|
|
270
|
+
self.result = RunResult(spec.run_id)
|
|
271
|
+
self._workers = list(_worker_names(spec.workers))
|
|
272
|
+
self._ranks: dict[str, int] = {}
|
|
273
|
+
self._agent_workers = spec.workers + sum(
|
|
274
|
+
1 for entry in self._plan if entry.kind.action is Action.START_WORKER
|
|
275
|
+
)
|
|
276
|
+
gateway = spec.config in GATEWAY_CONFIGS
|
|
277
|
+
self._tool_names = GATEWAY_TOOL_NAMES if gateway else BASELINE_TOOL_NAMES
|
|
278
|
+
self._closing: list[Any] = []
|
|
279
|
+
self._last_write_answers: dict[tuple[str, str], int] = {}
|
|
280
|
+
self._params: dict[str, object] = {
|
|
281
|
+
"params": spec.scenario.params.model_dump(mode="json"),
|
|
282
|
+
"plan": [entry.model_dump(mode="json") for entry in self._plan],
|
|
283
|
+
"workers": spec.workers,
|
|
284
|
+
"agent_workers": self._agent_workers,
|
|
285
|
+
}
|
|
286
|
+
"""The values the run uses, as `harness.run.params` records them (0009)."""
|
|
287
|
+
if isinstance(spec.model, Branch):
|
|
288
|
+
_check_branch(spec, self._params, spec.model.parent.run)
|
|
289
|
+
self.models = RunModels(spec.model)
|
|
290
|
+
self._model_calls = 0
|
|
291
|
+
|
|
292
|
+
def _conninfo(self, role: str) -> str:
|
|
293
|
+
return make_conninfo(self._dsn, user=role)
|
|
294
|
+
|
|
295
|
+
async def execute(self) -> RunResult:
|
|
296
|
+
"""Run to the end of the drain window; return how each worker ended."""
|
|
297
|
+
spec, scenario = self.spec, self.spec.scenario
|
|
298
|
+
params = scenario.params
|
|
299
|
+
harness_conn = psycopg.connect(self._conninfo("ff_harness"), autocommit=True)
|
|
300
|
+
self._closing.append(harness_conn)
|
|
301
|
+
self.log = PgHarnessLog(harness_conn)
|
|
302
|
+
plan = self._plan
|
|
303
|
+
parent = spec.parent
|
|
304
|
+
self.log.run_recorded(
|
|
305
|
+
RunRecord(
|
|
306
|
+
run_id=spec.run_id,
|
|
307
|
+
scenario_id=scenario.id,
|
|
308
|
+
scenario_digest=scenario_digest(scenario),
|
|
309
|
+
config=spec.config,
|
|
310
|
+
contract=str(spec.contract), # type: ignore[arg-type]
|
|
311
|
+
seed=spec.seed,
|
|
312
|
+
model_adapter=self.models.adapter,
|
|
313
|
+
model=self.models.name,
|
|
314
|
+
cassette_digest=self.models.cassette_digest,
|
|
315
|
+
params=self._params,
|
|
316
|
+
parent_run_id=None if parent is None else parent.run_id,
|
|
317
|
+
branch_point=None if parent is None else parent.branch_point,
|
|
318
|
+
)
|
|
319
|
+
)
|
|
320
|
+
register(
|
|
321
|
+
self._dsn,
|
|
322
|
+
RegistryEntry(
|
|
323
|
+
database=database_of(self._dsn),
|
|
324
|
+
run_id=spec.run_id,
|
|
325
|
+
scenario_id=scenario.id,
|
|
326
|
+
config=spec.config,
|
|
327
|
+
contract=str(spec.contract),
|
|
328
|
+
seed=spec.seed,
|
|
329
|
+
),
|
|
330
|
+
)
|
|
331
|
+
self.faults = FaultInjector(plan)
|
|
332
|
+
generators = Generators(spec.seed)
|
|
333
|
+
self.transport = Transport(
|
|
334
|
+
self.scheduler,
|
|
335
|
+
self.log,
|
|
336
|
+
generators.get("latency"),
|
|
337
|
+
max_hop_latency_ms=params.max_hop_latency_ms,
|
|
338
|
+
faults=self.faults,
|
|
339
|
+
)
|
|
340
|
+
self.transport.observe(self._agent_points)
|
|
341
|
+
self.transport.observe(self._gateway_answers)
|
|
342
|
+
self.transport.before_send(self._before_tool_call)
|
|
343
|
+
gateway = spec.config in GATEWAY_CONFIGS
|
|
344
|
+
credentials = (GATEWAY,) if gateway else _worker_names(self._agent_workers)
|
|
345
|
+
config = SimConfig.preset(
|
|
346
|
+
spec.contract, credentials=credentials, params=params.contract_params()
|
|
347
|
+
)
|
|
348
|
+
self.simulator = Simulator.setup(self._conninfo("ff_sim"), scenario, config, vt=0)
|
|
349
|
+
self._closing.append(self.simulator)
|
|
350
|
+
self.approver = ScriptedApprover(
|
|
351
|
+
self.scheduler, self.simulator.control, self.log, scenario.authority
|
|
352
|
+
)
|
|
353
|
+
self.resolver = self._resolver(PgEffectsLedger(harness_conn))
|
|
354
|
+
self.effects = SimulatorFaults(self.scheduler, self.simulator.control, self.log, scenario)
|
|
355
|
+
self.transport.apply_faults_with(
|
|
356
|
+
lambda fired, envelope: self.effects.apply(fired, message_context(envelope.body))
|
|
357
|
+
)
|
|
358
|
+
self.transport.serve(
|
|
359
|
+
SIMULATOR,
|
|
360
|
+
endpoint(
|
|
361
|
+
self.simulator,
|
|
362
|
+
self.scheduler,
|
|
363
|
+
poll_ms=params.approval_poll_interval_ms,
|
|
364
|
+
max_wait_ms=params.max_wait_ms,
|
|
365
|
+
served=[self.approver.served],
|
|
366
|
+
points=self._simulator_points(),
|
|
367
|
+
),
|
|
368
|
+
)
|
|
369
|
+
intents = {intent.id: str(intent.operation) for intent in scenario.intents}
|
|
370
|
+
self._intents = intents
|
|
371
|
+
self.sink = ReportSink(self.scheduler, self.log, intents)
|
|
372
|
+
if spec.config == "B0":
|
|
373
|
+
self.transport.serve(REPORT_SINK, sink_endpoint(self.sink))
|
|
374
|
+
checkpoint_conn = connect(self._conninfo("ff_agent"))
|
|
375
|
+
self._closing.append(checkpoint_conn)
|
|
376
|
+
self.checkpointer = InlinePostgresSaver(checkpoint_conn)
|
|
377
|
+
self.checkpointer.setup()
|
|
378
|
+
self.timeouts = Timeouts(
|
|
379
|
+
t_ms=params.dispatch_timeout_ms,
|
|
380
|
+
tool_wait_ms=params.tool_wait_ms,
|
|
381
|
+
max_wait_ms=params.max_wait_ms,
|
|
382
|
+
)
|
|
383
|
+
if gateway:
|
|
384
|
+
self._start_gateway()
|
|
385
|
+
for worker in self._workers:
|
|
386
|
+
self._start_worker(worker, resume=False)
|
|
387
|
+
await self.scheduler.run(until=self._agents_done)
|
|
388
|
+
self._close_run()
|
|
389
|
+
drain_end = self.scheduler.now + drain_window_ms(scenario)
|
|
390
|
+
await self.scheduler.run(deadline_ms=drain_end)
|
|
391
|
+
# The drain lasts its whole window (0009). The scheduler returns as soon as nothing
|
|
392
|
+
# is queued and leaves the clock there, so a quiet drain moves it to the end here.
|
|
393
|
+
self.scheduler.clock.advance_to(max(self.scheduler.now, drain_end))
|
|
394
|
+
self.simulator.control.abort_overdue(vt=self.scheduler.now)
|
|
395
|
+
self.result.stopped_vt = self.scheduler.now
|
|
396
|
+
self.log.run_stopped(self.scheduler.now)
|
|
397
|
+
self._models_stopped()
|
|
398
|
+
await self._stop()
|
|
399
|
+
self.models.check()
|
|
400
|
+
self.result.evaluation = evaluate_run(self._conninfo("ff_eval"), scenario)
|
|
401
|
+
return self.result
|
|
402
|
+
|
|
403
|
+
def _agents_done(self) -> bool:
|
|
404
|
+
return len(self.result.ends) == len(self._workers)
|
|
405
|
+
|
|
406
|
+
def _simulator_points(self) -> SimulatorPoints:
|
|
407
|
+
"""F5 and F10, at the simulator's fault points (0006). They also watch the status
|
|
408
|
+
answers go out, to log each one F10 changed (0014)."""
|
|
409
|
+
params = self.spec.scenario.params
|
|
410
|
+
points = SimulatorPoints(
|
|
411
|
+
self.scheduler,
|
|
412
|
+
self.faults,
|
|
413
|
+
self.log,
|
|
414
|
+
dispatch_timeout_ms=params.dispatch_timeout_ms,
|
|
415
|
+
status_lag_ms=params.status_lag_ms,
|
|
416
|
+
)
|
|
417
|
+
self.transport.observe(points.answer_sent)
|
|
418
|
+
return points
|
|
419
|
+
|
|
420
|
+
# The gateway --------------------------------------------------------------------------
|
|
421
|
+
|
|
422
|
+
def _start_gateway(self) -> None:
|
|
423
|
+
"""Start the gateway: its store, startup recovery and its dispatcher (0003).
|
|
424
|
+
|
|
425
|
+
Also how a crashed gateway restarts: everything it had in memory is new.
|
|
426
|
+
"""
|
|
427
|
+
scenario, params = self.spec.scenario, self.spec.scenario.params
|
|
428
|
+
intents = self._intents
|
|
429
|
+
settings = GatewaySettings(
|
|
430
|
+
approval_wait_ms=scenario.authority.approval_wait_ms,
|
|
431
|
+
params=params.contract_params(),
|
|
432
|
+
dispatch_timeout_ms=params.dispatch_timeout_ms,
|
|
433
|
+
max_attempts=params.max_attempts,
|
|
434
|
+
max_probes=params.max_probes,
|
|
435
|
+
agent_workers=self._agent_workers,
|
|
436
|
+
)
|
|
437
|
+
self.store = PgStore(
|
|
438
|
+
self._conninfo("ff_gw"),
|
|
439
|
+
task_id=scenario.task_id,
|
|
440
|
+
intents=intents,
|
|
441
|
+
contract=self.spec.contract,
|
|
442
|
+
settings=settings,
|
|
443
|
+
identity=self.spec.config == "G2",
|
|
444
|
+
)
|
|
445
|
+
self._closing.append(self.store)
|
|
446
|
+
client = SimClient(self.transport, timeout_ms=params.dispatch_timeout_ms)
|
|
447
|
+
self.dispatcher = Dispatcher(
|
|
448
|
+
self.scheduler,
|
|
449
|
+
self.store,
|
|
450
|
+
client,
|
|
451
|
+
RuleFreshness(self.store),
|
|
452
|
+
component=GATEWAY,
|
|
453
|
+
timing=Timing(
|
|
454
|
+
approval_poll_ms=params.approval_poll_interval_ms,
|
|
455
|
+
probe_interval_ms=params.status_probe_interval_ms,
|
|
456
|
+
),
|
|
457
|
+
faults=self.faults,
|
|
458
|
+
on_fault=self._gateway_fault,
|
|
459
|
+
on_escalated=self._escalated,
|
|
460
|
+
)
|
|
461
|
+
self.tools = GatewayTools(
|
|
462
|
+
self.scheduler,
|
|
463
|
+
self.store,
|
|
464
|
+
self.dispatcher,
|
|
465
|
+
client,
|
|
466
|
+
self.sink,
|
|
467
|
+
task_id=scenario.task_id,
|
|
468
|
+
intents=intents,
|
|
469
|
+
timing=ToolTiming(tool_wait_ms=params.tool_wait_ms, max_wait_ms=params.max_wait_ms),
|
|
470
|
+
)
|
|
471
|
+
self.dispatcher.start()
|
|
472
|
+
|
|
473
|
+
def _close_run(self) -> None:
|
|
474
|
+
"""In G2, once the agents' phase is over, close the run if no report did (0003).
|
|
475
|
+
|
|
476
|
+
A gateway that is down then closes it when it restarts.
|
|
477
|
+
"""
|
|
478
|
+
if self.spec.config != "G2" or not self.scheduler.is_up(GATEWAY):
|
|
479
|
+
return
|
|
480
|
+
if not self.store.snapshot().run_closed:
|
|
481
|
+
self.store.close_run(self.scheduler.now)
|
|
482
|
+
|
|
483
|
+
def _gateway_fault(self, fired: Fired, point: str, context: Mapping[str, object]) -> None:
|
|
484
|
+
"""A fault at a dispatch point: record it, then apply it: F6 and F7 early through
|
|
485
|
+
the simulator, F1 and F3 by crashing the gateway.
|
|
486
|
+
|
|
487
|
+
What the crash changes for the evidence goes in the fault's record (0006, crashes;
|
|
488
|
+
0004, property 5). A crash after the attempt is recorded leaves a request that may
|
|
489
|
+
have left: the record names it. A crash after the answer arrived voids that answer,
|
|
490
|
+
which the gateway never recorded.
|
|
491
|
+
"""
|
|
492
|
+
action = fired.planned.kind.action
|
|
493
|
+
params: dict[str, object] = dict(fired.planned.params)
|
|
494
|
+
voided: tuple[int, ...] = ()
|
|
495
|
+
if action is Action.KILL and point == POSSIBLY_SENT_POINT:
|
|
496
|
+
params["request"] = {
|
|
497
|
+
"intent_id": context["intent_id"],
|
|
498
|
+
"api": context["api"],
|
|
499
|
+
"idempotency_key": context["idempotency_key"],
|
|
500
|
+
}
|
|
501
|
+
if action is Action.KILL and point == VOIDING_POINT:
|
|
502
|
+
answer = self._last_write_answers.get((str(context["api"]), str(context["intent_id"])))
|
|
503
|
+
if answer is None:
|
|
504
|
+
raise RuntimeError(f"{fired.fault} at {point}: no answer reached the gateway")
|
|
505
|
+
voided = (answer,)
|
|
506
|
+
self.log.fault_fired(
|
|
507
|
+
FaultRecord(
|
|
508
|
+
seq=self.log.next_seq(),
|
|
509
|
+
fault=str(fired.fault),
|
|
510
|
+
variant=fired.planned.variant,
|
|
511
|
+
point=point,
|
|
512
|
+
occurrence=fired.occurrence,
|
|
513
|
+
action=str(action),
|
|
514
|
+
vt=self.scheduler.now,
|
|
515
|
+
component=GATEWAY,
|
|
516
|
+
voided=voided,
|
|
517
|
+
params=params,
|
|
518
|
+
intent_id=_text(context.get("intent_id")),
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
if action is Action.KILL:
|
|
522
|
+
self._crash_gateway(fired.planned.param("restart_delay_ms"))
|
|
523
|
+
else:
|
|
524
|
+
self.effects.apply(fired, context)
|
|
525
|
+
|
|
526
|
+
def _crash_gateway(self, restart_after_ms: int) -> None:
|
|
527
|
+
"""Stop the gateway now and start it again later (0006, crashes).
|
|
528
|
+
|
|
529
|
+
Its memory goes, and so does its database connection, with any open
|
|
530
|
+
transaction and the run's lock; its tasks are cancelled. The restart builds a new
|
|
531
|
+
gateway, which recovers from its records before it serves anything (0003).
|
|
532
|
+
"""
|
|
533
|
+
self.store.close()
|
|
534
|
+
self.scheduler.crash(
|
|
535
|
+
GATEWAY, restart_after_ms=restart_after_ms, restart=self._restart_gateway
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
def _restart_gateway(self) -> None:
|
|
539
|
+
self._start_gateway()
|
|
540
|
+
if self._agents_done():
|
|
541
|
+
self._close_run()
|
|
542
|
+
|
|
543
|
+
def _connect(self, worker: str) -> Link:
|
|
544
|
+
"""Open a new MCP connection from `worker` to the gateway.
|
|
545
|
+
|
|
546
|
+
A gateway that is up serves it, with the tools of its current incarnation; one
|
|
547
|
+
that is down never does.
|
|
548
|
+
"""
|
|
549
|
+
link = Link(self.scheduler, self.transport, agent=worker, gateway=GATEWAY)
|
|
550
|
+
if self.scheduler.is_up(GATEWAY):
|
|
551
|
+
self.scheduler.spawn(GATEWAY, serve(self.tools, link), name=f"mcp server for {worker}")
|
|
552
|
+
return link
|
|
553
|
+
|
|
554
|
+
# The scripted resolver (0003, 0009) -----------------------------------------------------
|
|
555
|
+
|
|
556
|
+
def _resolver(self, ledger: PgEffectsLedger) -> ScriptedResolver | None:
|
|
557
|
+
"""The scripted resolver, in G modes when the scenario has one (0009).
|
|
558
|
+
|
|
559
|
+
It reads the simulator's ledger out of band through `ledger`, as `ff_harness`
|
|
560
|
+
(0008). B0 has no gateway, so nothing there escalates.
|
|
561
|
+
"""
|
|
562
|
+
rule = self.spec.scenario.authority.resolver
|
|
563
|
+
if self.spec.config == "B0" or rule == "none":
|
|
564
|
+
return None
|
|
565
|
+
settle_ms = self.spec.scenario.params.contract_params().settle_ms
|
|
566
|
+
return ScriptedResolver(
|
|
567
|
+
self.scheduler, self._admin, ledger, self.log, rule, settle_ms=settle_ms
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
def _admin(self) -> GatewayAdmin | None:
|
|
571
|
+
"""The admin path of the gateway's current incarnation, or None while it is down."""
|
|
572
|
+
if not self.scheduler.is_up(GATEWAY):
|
|
573
|
+
return None
|
|
574
|
+
return GatewayAdmin(self.scheduler, self.store, self.dispatcher)
|
|
575
|
+
|
|
576
|
+
def _escalated(self, action_id: str) -> None:
|
|
577
|
+
"""The gateway left an escalated action to a human: tell the resolver, if any."""
|
|
578
|
+
if self.resolver is not None:
|
|
579
|
+
self.resolver.escalated(action_id)
|
|
580
|
+
|
|
581
|
+
# Agent workers ------------------------------------------------------------------------
|
|
582
|
+
|
|
583
|
+
def _start_worker(self, worker: str, *, resume: bool) -> None:
|
|
584
|
+
label = "resumes" if resume else "starts"
|
|
585
|
+
self.scheduler.spawn(worker, self._worker(worker, resume=resume), name=f"{worker} {label}")
|
|
586
|
+
|
|
587
|
+
async def _worker(self, worker: str, *, resume: bool) -> None:
|
|
588
|
+
scenario = self.spec.scenario
|
|
589
|
+
model = self._chat_model(worker)
|
|
590
|
+
budget = scenario.budget
|
|
591
|
+
spec = WorkerSpec(worker, scenario.task, budget.max_agent_steps, budget.max_tokens)
|
|
592
|
+
on_call = self._call_log(worker)
|
|
593
|
+
|
|
594
|
+
def on_step(step: int) -> None:
|
|
595
|
+
self._agent_step(worker, step)
|
|
596
|
+
|
|
597
|
+
if self.spec.config == "B0":
|
|
598
|
+
tools = baseline_tools(
|
|
599
|
+
self.transport,
|
|
600
|
+
self.scheduler,
|
|
601
|
+
self.timeouts,
|
|
602
|
+
worker=worker,
|
|
603
|
+
status=self.spec.contract.has(Capability.STATUS),
|
|
604
|
+
)
|
|
605
|
+
graph = build_graph(
|
|
606
|
+
model,
|
|
607
|
+
tools,
|
|
608
|
+
self.checkpointer,
|
|
609
|
+
max_steps=spec.max_steps,
|
|
610
|
+
max_tokens=spec.max_tokens,
|
|
611
|
+
on_step=on_step,
|
|
612
|
+
on_call=on_call,
|
|
613
|
+
)
|
|
614
|
+
self.result.ends[worker] = await run_worker(graph, spec, resume=resume)
|
|
615
|
+
return
|
|
616
|
+
connection = GatewayConnection(
|
|
617
|
+
self.scheduler,
|
|
618
|
+
lambda: self._connect(worker),
|
|
619
|
+
worker=worker,
|
|
620
|
+
timeout_ms=scenario.params.dispatch_timeout_ms,
|
|
621
|
+
)
|
|
622
|
+
try:
|
|
623
|
+
listed = await connection.list_tools()
|
|
624
|
+
tools = gateway_tools(connection, listed, self.timeouts)
|
|
625
|
+
graph = build_graph(
|
|
626
|
+
model,
|
|
627
|
+
tools,
|
|
628
|
+
self.checkpointer,
|
|
629
|
+
max_steps=spec.max_steps,
|
|
630
|
+
max_tokens=spec.max_tokens,
|
|
631
|
+
on_step=on_step,
|
|
632
|
+
on_call=on_call,
|
|
633
|
+
)
|
|
634
|
+
self.result.ends[worker] = await run_worker(graph, spec, resume=resume)
|
|
635
|
+
except asyncio.CancelledError:
|
|
636
|
+
raise
|
|
637
|
+
except Exception as error:
|
|
638
|
+
# A worker that cannot connect, or fails otherwise, ends without a report (0008).
|
|
639
|
+
self.result.ends[worker] = (WorkerEnd.ERROR, _describe(error))
|
|
640
|
+
finally:
|
|
641
|
+
# After a crash the link ends the connection itself, once in-flight requests
|
|
642
|
+
# have reached the gateway; this closes it only when the worker finishes.
|
|
643
|
+
connection.close()
|
|
644
|
+
|
|
645
|
+
# Fault points of the agent (0006) -----------------------------------------------------
|
|
646
|
+
|
|
647
|
+
def _agent_step(self, worker: str, step: int) -> None:
|
|
648
|
+
"""`agent.step`: F9 starts another worker on the same task (0006).
|
|
649
|
+
|
|
650
|
+
The new worker runs the same scripted policy from the start, with its own
|
|
651
|
+
checkpoint thread and step budget; in the `different_args` variant it chooses the
|
|
652
|
+
second-cheapest acceptable option (0008).
|
|
653
|
+
"""
|
|
654
|
+
for fired in self.faults.at("agent.step", {"worker": worker, "step": step}):
|
|
655
|
+
if fired.planned.kind.action is not Action.START_WORKER:
|
|
656
|
+
continue
|
|
657
|
+
started = f"agent-{len(self._workers) + 1}"
|
|
658
|
+
self.log.fault_fired(
|
|
659
|
+
FaultRecord(
|
|
660
|
+
seq=self.log.next_seq(),
|
|
661
|
+
fault=str(fired.fault),
|
|
662
|
+
variant=fired.planned.variant,
|
|
663
|
+
point="agent.step",
|
|
664
|
+
occurrence=fired.occurrence,
|
|
665
|
+
action=str(Action.START_WORKER),
|
|
666
|
+
vt=self.scheduler.now,
|
|
667
|
+
component=started,
|
|
668
|
+
params={**fired.planned.params, "worker": worker, "step": step},
|
|
669
|
+
intent_id=None,
|
|
670
|
+
)
|
|
671
|
+
)
|
|
672
|
+
self._workers.append(started)
|
|
673
|
+
self._ranks[started] = 1 if fired.planned.variant == "different_args" else 0
|
|
674
|
+
self._start_worker(started, resume=False)
|
|
675
|
+
|
|
676
|
+
def _before_tool_call(self, outgoing: Outgoing) -> None:
|
|
677
|
+
"""F1's B0 equivalent: kill a worker just before it sends a tool call (0006).
|
|
678
|
+
|
|
679
|
+
The call is never sent. The worker resumes from its last checkpoint, which comes
|
|
680
|
+
before the tool node, so it sends the call once it resumes.
|
|
681
|
+
"""
|
|
682
|
+
worker = outgoing.sender
|
|
683
|
+
if outgoing.kind != "request" or worker not in self._workers:
|
|
684
|
+
return
|
|
685
|
+
if outgoing.api not in self._tool_names:
|
|
686
|
+
return
|
|
687
|
+
fields = {"api": outgoing.api, "tool": outgoing.api, "intent_id": outgoing.intent_id}
|
|
688
|
+
for fired in self.faults.at("agent.before_tool_call", fields):
|
|
689
|
+
if fired.planned.kind.action is not Action.KILL:
|
|
690
|
+
continue
|
|
691
|
+
self.log.fault_fired(
|
|
692
|
+
FaultRecord(
|
|
693
|
+
seq=self.log.next_seq(),
|
|
694
|
+
fault=str(fired.fault),
|
|
695
|
+
variant=fired.planned.variant,
|
|
696
|
+
point="agent.before_tool_call",
|
|
697
|
+
occurrence=fired.occurrence,
|
|
698
|
+
action=str(Action.KILL),
|
|
699
|
+
vt=self.scheduler.now,
|
|
700
|
+
component=worker,
|
|
701
|
+
params=dict(fired.planned.params),
|
|
702
|
+
intent_id=outgoing.intent_id,
|
|
703
|
+
)
|
|
704
|
+
)
|
|
705
|
+
self._kill(worker, fired.planned.param("restart_delay_ms"))
|
|
706
|
+
return # The worker is down, so it reaches no further fault (`_agent_points`).
|
|
707
|
+
|
|
708
|
+
def _gateway_answers(self, envelope: Envelope, stage: Stage) -> None:
|
|
709
|
+
"""Note each write answer delivered to the gateway, by API and intent, so that a
|
|
710
|
+
crash right after it (F3) can void it."""
|
|
711
|
+
if (
|
|
712
|
+
stage == "delivered"
|
|
713
|
+
and envelope.hop == "gw_sim"
|
|
714
|
+
and envelope.kind == "answer"
|
|
715
|
+
and envelope.receiver == GATEWAY
|
|
716
|
+
and envelope.api is not None
|
|
717
|
+
and envelope.intent_id is not None
|
|
718
|
+
):
|
|
719
|
+
self._last_write_answers[(envelope.api, envelope.intent_id)] = envelope.seq
|
|
720
|
+
|
|
721
|
+
def _agent_points(self, envelope: Envelope, stage: Stage) -> None:
|
|
722
|
+
"""F11, and F3's B0 equivalent: kill a worker after it sends a tool call, or as a
|
|
723
|
+
tool result reaches it."""
|
|
724
|
+
if stage == "sent" and envelope.kind == "request":
|
|
725
|
+
worker, point = envelope.sender, "agent.after_tool_call"
|
|
726
|
+
elif stage == "delivered" and envelope.kind == "answer":
|
|
727
|
+
worker, point = envelope.receiver, "agent.after_tool_result"
|
|
728
|
+
else:
|
|
729
|
+
return
|
|
730
|
+
if worker not in self._workers or envelope.api not in self._tool_names:
|
|
731
|
+
return
|
|
732
|
+
fields = {"api": envelope.api, "tool": envelope.api, "intent_id": envelope.intent_id}
|
|
733
|
+
for fired in self.faults.at(point, fields):
|
|
734
|
+
if fired.planned.kind.action is not Action.KILL:
|
|
735
|
+
continue
|
|
736
|
+
delay = fired.planned.param("restart_delay_ms")
|
|
737
|
+
voided = (envelope.seq,) if point == "agent.after_tool_result" else ()
|
|
738
|
+
self.log.fault_fired(
|
|
739
|
+
FaultRecord(
|
|
740
|
+
seq=self.log.next_seq(),
|
|
741
|
+
fault=str(fired.fault),
|
|
742
|
+
variant=fired.planned.variant,
|
|
743
|
+
point=point,
|
|
744
|
+
occurrence=fired.occurrence,
|
|
745
|
+
action=str(Action.KILL),
|
|
746
|
+
vt=self.scheduler.now,
|
|
747
|
+
target_seq=envelope.seq,
|
|
748
|
+
component=worker,
|
|
749
|
+
voided=voided,
|
|
750
|
+
params=dict(fired.planned.params),
|
|
751
|
+
intent_id=envelope.intent_id,
|
|
752
|
+
)
|
|
753
|
+
)
|
|
754
|
+
# The kill is immediate. After a tool call, the request is already on the
|
|
755
|
+
# wire; after a result, the worker must not see it: the transport hands a
|
|
756
|
+
# delivery to its receiver only after the observers ran, and only if the
|
|
757
|
+
# receiver is still up (0006, voided results).
|
|
758
|
+
self._kill(worker, delay)
|
|
759
|
+
# A worker that is down reaches no fault point, as the gateway's dispatcher
|
|
760
|
+
# stops at the first crash: a second kill at this event does not fire, and
|
|
761
|
+
# the worker restarts once, after this kill's delay.
|
|
762
|
+
return
|
|
763
|
+
|
|
764
|
+
def _kill(self, worker: str, restart_after_ms: int) -> None:
|
|
765
|
+
self.scheduler.crash(
|
|
766
|
+
worker,
|
|
767
|
+
restart_after_ms=restart_after_ms,
|
|
768
|
+
restart=lambda: self._start_worker(worker, resume=True),
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
# The model adapter (0009) -------------------------------------------------------------
|
|
772
|
+
|
|
773
|
+
def _chat_model(self, worker: str) -> BaseChatModel:
|
|
774
|
+
"""The chat model of `worker`, built each time it starts or resumes, by the run's
|
|
775
|
+
model adapter (`fivefold.harness.models`)."""
|
|
776
|
+
scenario = self.spec.scenario
|
|
777
|
+
mode = Mode.GATEWAY if self.spec.config in GATEWAY_CONFIGS else Mode.BASELINE
|
|
778
|
+
rank = self._ranks.get(worker, 0)
|
|
779
|
+
|
|
780
|
+
def think() -> Awaitable[None]:
|
|
781
|
+
return self.scheduler.sleep(scenario.params.think_ms)
|
|
782
|
+
|
|
783
|
+
def scripted(its_think: Think) -> BaseChatModel:
|
|
784
|
+
return ScriptedModel(
|
|
785
|
+
lambda: scripted_policy(
|
|
786
|
+
scenario.intents, mode=mode, rank=rank, max_wait_ms=scenario.params.max_wait_ms
|
|
787
|
+
),
|
|
788
|
+
its_think,
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
return self.models.chat_model(
|
|
792
|
+
worker, scripted=scripted, think=think, hold=self.scheduler.hold
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
def _call_log(self, worker: str) -> CallLog:
|
|
796
|
+
"""Log each model call of `worker` once it is answered (0004): its number in the
|
|
797
|
+
run, its virtual times, the run's model, and the reply's token usage."""
|
|
798
|
+
|
|
799
|
+
def asked() -> Callable[[AIMessage], None]:
|
|
800
|
+
asked_vt, started = self.scheduler.now, wall_ms()
|
|
801
|
+
|
|
802
|
+
def answered(reply: AIMessage) -> None:
|
|
803
|
+
self._model_calls += 1
|
|
804
|
+
usage = reply.usage_metadata
|
|
805
|
+
self.log.model_called(
|
|
806
|
+
ModelCallRecord(
|
|
807
|
+
call_no=self._model_calls,
|
|
808
|
+
worker=worker,
|
|
809
|
+
asked_vt=asked_vt,
|
|
810
|
+
answered_vt=self.scheduler.now,
|
|
811
|
+
model=self.models.name,
|
|
812
|
+
input_tokens=None if usage is None else usage["input_tokens"],
|
|
813
|
+
output_tokens=None if usage is None else usage["output_tokens"],
|
|
814
|
+
refused=refusal(reply),
|
|
815
|
+
wall_ms=wall_ms() - started,
|
|
816
|
+
)
|
|
817
|
+
)
|
|
818
|
+
|
|
819
|
+
return answered
|
|
820
|
+
|
|
821
|
+
return asked
|
|
822
|
+
|
|
823
|
+
def _models_stopped(self) -> None:
|
|
824
|
+
"""The run has stopped: a recording's cassette is complete, and `harness.run`
|
|
825
|
+
records its digest."""
|
|
826
|
+
if self.log.run is None:
|
|
827
|
+
raise RuntimeError("the run was never recorded")
|
|
828
|
+
digest = self.models.stopped(self.log.run)
|
|
829
|
+
if digest is not None:
|
|
830
|
+
self.log.cassette_recorded(digest)
|
|
831
|
+
|
|
832
|
+
# Stopping -------------------------------------------------------------------------------
|
|
833
|
+
|
|
834
|
+
async def _stop(self) -> None:
|
|
835
|
+
"""Stop every component, let their tasks unwind, and close their connections."""
|
|
836
|
+
for component in (*self._workers, GATEWAY, SIMULATOR, REPORT_SINK):
|
|
837
|
+
if self.scheduler.is_up(component):
|
|
838
|
+
self.scheduler.crash(component)
|
|
839
|
+
await settle()
|
|
840
|
+
for resource in reversed(self._closing):
|
|
841
|
+
resource.close()
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _worker_names(count: int) -> tuple[str, ...]:
|
|
845
|
+
"""The names of a run's agent workers: `agent-1` to `agent-{count}`."""
|
|
846
|
+
return tuple(f"agent-{number}" for number in range(1, count + 1))
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _text(value: object) -> str | None:
|
|
850
|
+
return value if isinstance(value, str) else None
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _describe(error: BaseException) -> str:
|
|
854
|
+
"""The error that ended a worker; the first one inside an exception group, as the
|
|
855
|
+
MCP session's task group wraps them."""
|
|
856
|
+
while isinstance(error, BaseExceptionGroup) and error.exceptions:
|
|
857
|
+
error = error.exceptions[0]
|
|
858
|
+
return f"{type(error).__name__}: {error}"
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
def _check_branch(spec: RunSpec, params: Mapping[str, object], parent: RunHeader) -> None:
|
|
862
|
+
"""Refuse a branch that would not run its parent's run (0009): the same scenario, with
|
|
863
|
+
the same digest, configuration, contract level, seed and topology, and the same
|
|
864
|
+
values, which hold the fault plan with the parameters drawn from the seed, and the
|
|
865
|
+
workers. Otherwise the branch would name a parent it does not continue."""
|
|
866
|
+
scenario = spec.scenario
|
|
867
|
+
mine: dict[str, object] = {
|
|
868
|
+
"scenario_id": scenario.id,
|
|
869
|
+
"scenario_digest": scenario_digest(scenario),
|
|
870
|
+
"config": spec.config,
|
|
871
|
+
"contract": str(spec.contract),
|
|
872
|
+
"seed": spec.seed,
|
|
873
|
+
"topology": "inproc",
|
|
874
|
+
"params": plain_object(params, "the run's params"),
|
|
875
|
+
}
|
|
876
|
+
differ = [name for name, value in mine.items() if getattr(parent, name) != value]
|
|
877
|
+
if differ:
|
|
878
|
+
raise ValueError(
|
|
879
|
+
f"a branch runs its parent's run, but it differs from {parent.run_id} in"
|
|
880
|
+
f" {', '.join(differ)}"
|
|
881
|
+
)
|