fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""The run's virtual clock, and the only wall-clock access in the harness (0009).
|
|
2
|
+
|
|
3
|
+
Every component reads time only from the run's `VirtualClock`: grant expiry, timeouts,
|
|
4
|
+
`action_event.vt`, `attempt.sent_vt` and `effect.committed_vt` are virtual milliseconds.
|
|
5
|
+
This module and `rng.py` are the only places allowed to use wall-clock time or system
|
|
6
|
+
randomness; the lint configuration in pyproject.toml enforces it (0007). Wall-clock time
|
|
7
|
+
is read here for metrics only, never for decisions.
|
|
8
|
+
|
|
9
|
+
`settle` lets every other task of the event loop run until each one waits on something
|
|
10
|
+
the scheduler controls. It reads two internals of CPython 3.12's asyncio event loop, the
|
|
11
|
+
ready queue (`_ready`) and the timer heap (`_scheduled`), checked in
|
|
12
|
+
`Lib/asyncio/base_events.py` of Python 3.12.3. The project pins Python 3.12
|
|
13
|
+
(pyproject.toml), and `test_scheduler.py` fails loudly if either internal disappears.
|
|
14
|
+
|
|
15
|
+
A task that waits on the real world, as a live model call waits on its provider, does so
|
|
16
|
+
inside a real wait (`RealWaits.hold`). Network I/O registers nothing that `settle` can
|
|
17
|
+
see: an HTTP client with no timeout leaves no timer in the heap. So while a real wait is
|
|
18
|
+
in flight, `settle` waits for it to end, in wall time, and does not return. Virtual time
|
|
19
|
+
stands still meanwhile, and the call takes no virtual time of its own (0009).
|
|
20
|
+
|
|
21
|
+
In the split topology, virtual time follows the host's monotonic clock (0009).
|
|
22
|
+
`HostClock` reads it, counting milliseconds from the run's origin, and its timers are the
|
|
23
|
+
event loop's, in wall time. Every process of one split run reads the same origin, so on
|
|
24
|
+
one host they all read the same virtual time.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import asyncio
|
|
30
|
+
import time
|
|
31
|
+
from collections import deque
|
|
32
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
33
|
+
from typing import TypeVar, cast
|
|
34
|
+
|
|
35
|
+
T = TypeVar("T")
|
|
36
|
+
|
|
37
|
+
TIMER_SLACK_S = 0.000_5
|
|
38
|
+
"""How much later than due a host timer fires, in seconds, so that the clock reads at least
|
|
39
|
+
its due time when it fires, whatever the rounding to whole milliseconds."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class VirtualClock:
|
|
43
|
+
"""Virtual milliseconds since the run started. Only the scheduler moves it."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, start_ms: int = 0) -> None:
|
|
46
|
+
if start_ms < 0:
|
|
47
|
+
raise ValueError("virtual time starts at or after 0")
|
|
48
|
+
self._now = start_ms
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def now(self) -> int:
|
|
52
|
+
"""The current virtual time, in milliseconds."""
|
|
53
|
+
return self._now
|
|
54
|
+
|
|
55
|
+
def advance_to(self, at_ms: int) -> None:
|
|
56
|
+
"""Move the clock forward to `at_ms`. Virtual time never runs backwards."""
|
|
57
|
+
if at_ms < self._now:
|
|
58
|
+
raise ValueError(f"virtual time cannot go back from {self._now} to {at_ms}")
|
|
59
|
+
self._now = at_ms
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ClockBehind(RuntimeError):
|
|
63
|
+
"""The host's clock reads earlier than the run's origin: the process is on another host."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class HostClock(VirtualClock):
|
|
67
|
+
"""The split topology's clock: virtual time follows the host's monotonic clock (0009).
|
|
68
|
+
|
|
69
|
+
Virtual milliseconds are the host's monotonic milliseconds since `origin_ms`, so the
|
|
70
|
+
processes of a run on one host, which share its origin, read the same time. The host
|
|
71
|
+
moves it; nothing else can. It has no events of its own: `call_at` hands a callback to
|
|
72
|
+
the running event loop, which calls it in wall time.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self, origin_ms: int) -> None:
|
|
76
|
+
super().__init__(0)
|
|
77
|
+
self._origin = origin_ms
|
|
78
|
+
|
|
79
|
+
@staticmethod
|
|
80
|
+
def host_ms() -> int:
|
|
81
|
+
"""The host's monotonic clock, in whole milliseconds."""
|
|
82
|
+
return time.monotonic_ns() // 1_000_000
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def origin_ms(self) -> int:
|
|
86
|
+
"""The host's monotonic milliseconds at virtual time 0."""
|
|
87
|
+
return self._origin
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def now(self) -> int:
|
|
91
|
+
"""Milliseconds since the origin. Raises `ClockBehind` before the origin."""
|
|
92
|
+
elapsed = self.host_ms() - self._origin
|
|
93
|
+
if elapsed < 0:
|
|
94
|
+
raise ClockBehind(
|
|
95
|
+
f"the host's clock reads {-elapsed} ms before the run's origin: every process"
|
|
96
|
+
" of a split run must run on one host"
|
|
97
|
+
)
|
|
98
|
+
return elapsed
|
|
99
|
+
|
|
100
|
+
def advance_to(self, at_ms: int) -> None:
|
|
101
|
+
"""Refused: only the host moves this clock."""
|
|
102
|
+
raise RuntimeError("virtual time follows the host's clock here; nothing can move it")
|
|
103
|
+
|
|
104
|
+
def call_at(self, at_ms: int, callback: Callable[[], None]) -> asyncio.TimerHandle:
|
|
105
|
+
"""Call `callback` on the running event loop once virtual time reaches `at_ms`."""
|
|
106
|
+
due_ns = (self._origin + at_ms) * 1_000_000
|
|
107
|
+
delay_s = max(0.0, (due_ns - time.monotonic_ns()) / 1e9) + TIMER_SLACK_S
|
|
108
|
+
return asyncio.get_running_loop().call_later(delay_s, callback)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def wall_ms() -> float:
|
|
112
|
+
"""Wall-clock milliseconds from a monotonic clock, for metrics only (0009)."""
|
|
113
|
+
return time.monotonic() * 1000.0
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class LoopInternalsMissing(RuntimeError):
|
|
117
|
+
"""The running event loop lacks the CPython internals `settle` reads."""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _ready_queue(loop: asyncio.AbstractEventLoop) -> deque[asyncio.Handle]:
|
|
121
|
+
ready = getattr(loop, "_ready", None)
|
|
122
|
+
if not isinstance(ready, deque):
|
|
123
|
+
raise LoopInternalsMissing("the event loop has no _ready deque (CPython 3.12 asyncio)")
|
|
124
|
+
return cast(deque[asyncio.Handle], ready)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _timer_heap(loop: asyncio.AbstractEventLoop) -> Sequence[asyncio.TimerHandle]:
|
|
128
|
+
scheduled = getattr(loop, "_scheduled", None)
|
|
129
|
+
if not isinstance(scheduled, list):
|
|
130
|
+
raise LoopInternalsMissing("the event loop has no _scheduled list (CPython 3.12 asyncio)")
|
|
131
|
+
return cast(list[asyncio.TimerHandle], scheduled)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
async def yield_to_loop() -> None:
|
|
135
|
+
"""Let every callback that is ready now run once, then continue."""
|
|
136
|
+
await asyncio.sleep(0)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def within[T](awaitable: Awaitable[T], limit_s: float) -> T:
|
|
140
|
+
"""Await `awaitable`, a wait on the real world such as a live model call, for at most
|
|
141
|
+
`limit_s` wall-clock seconds. A wait still in flight then is cancelled and raises
|
|
142
|
+
`TimeoutError`, so a stalled provider cannot hold the run forever. The limit only
|
|
143
|
+
bounds a real wait; nothing in the run is decided by wall-clock time (0009)."""
|
|
144
|
+
return await asyncio.wait_for(awaitable, timeout=limit_s)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class RealWaits:
|
|
148
|
+
"""The real waits in flight, such as live model calls awaiting their provider (0009).
|
|
149
|
+
|
|
150
|
+
While one is in flight, virtual time stands still: `settle` does not return, so the
|
|
151
|
+
scheduler fires no event, and the run waits for it in wall time. What a real wait
|
|
152
|
+
awaits must never wait on the scheduler, which cannot move until the wait ends.
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
def __init__(self) -> None:
|
|
156
|
+
self._count = 0
|
|
157
|
+
self._waiters: list[asyncio.Future[None]] = []
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def active(self) -> bool:
|
|
161
|
+
"""Whether a real wait is in flight."""
|
|
162
|
+
return self._count > 0
|
|
163
|
+
|
|
164
|
+
async def hold(self, awaitable: Awaitable[T]) -> T:
|
|
165
|
+
"""Await `awaitable`, a real wait, with virtual time held still until it ends."""
|
|
166
|
+
self._count += 1
|
|
167
|
+
try:
|
|
168
|
+
return await awaitable
|
|
169
|
+
finally:
|
|
170
|
+
self._count -= 1
|
|
171
|
+
if self._count == 0:
|
|
172
|
+
waiters, self._waiters = self._waiters, []
|
|
173
|
+
for waiter in waiters:
|
|
174
|
+
if not waiter.done():
|
|
175
|
+
waiter.set_result(None)
|
|
176
|
+
|
|
177
|
+
async def ended(self) -> None:
|
|
178
|
+
"""Return once no real wait is in flight."""
|
|
179
|
+
if self._count == 0:
|
|
180
|
+
return
|
|
181
|
+
waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future()
|
|
182
|
+
self._waiters.append(waiter)
|
|
183
|
+
await waiter
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
async def settle(real: RealWaits | None = None) -> None:
|
|
187
|
+
"""Return once every other task waits on something the scheduler controls.
|
|
188
|
+
|
|
189
|
+
Ready callbacks run first. A task in one of `real`'s real waits, such as a live model
|
|
190
|
+
call awaiting its provider, holds the run until the wait ends. A task that sleeps on a
|
|
191
|
+
wall-clock timer, such as LangGraph's retry backoff (0008), holds it until its timer
|
|
192
|
+
fires. Virtual time must not move during either: they change wall-clock metrics only
|
|
193
|
+
(0009). While a real wait is in flight, `settle` waits for that wait alone, so a timer
|
|
194
|
+
the real wait itself set, such as an HTTP client's timeout, never keeps the run
|
|
195
|
+
asleep after the answer arrives.
|
|
196
|
+
"""
|
|
197
|
+
loop = asyncio.get_running_loop()
|
|
198
|
+
ready = _ready_queue(loop)
|
|
199
|
+
while True:
|
|
200
|
+
await yield_to_loop()
|
|
201
|
+
if ready:
|
|
202
|
+
continue
|
|
203
|
+
if real is not None and real.active:
|
|
204
|
+
await real.ended()
|
|
205
|
+
continue
|
|
206
|
+
pending = [timer.when() for timer in _timer_heap(loop) if not timer.cancelled()]
|
|
207
|
+
if not pending:
|
|
208
|
+
return
|
|
209
|
+
await asyncio.sleep(max(0.0, min(pending) - loop.time()))
|
fivefold/harness/demo.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""The demos of record 0006, run from a seed: `python -m fivefold.harness.demo` (`make demo`).
|
|
2
|
+
|
|
3
|
+
Each demo runs B0 and G2 on the same scenario and seed and shows them side by side
|
|
4
|
+
(0006, Demos):
|
|
5
|
+
|
|
6
|
+
1. "Order placed, reply lost" (dev-001, C2): F2 drops the purchase order's committed
|
|
7
|
+
answer, and F11 kills the agent right after it sends that call, resuming it from its
|
|
8
|
+
checkpoint 10,000 virtual ms later.
|
|
9
|
+
2. "Approval goes stale" (dev-002, C2): a human approves the order, then, before the
|
|
10
|
+
gateway sends it, F7 raises the quote's price (2a) or F6 revokes the approval (2b).
|
|
11
|
+
B0 meets the fault as the approval reaches the agent. dev-002 is dev-001 with the
|
|
12
|
+
auto-approval threshold below the order's amount, so the approval is a human's.
|
|
13
|
+
3. "Outcome unknown" (dev-001, C0): F2 drops the order's committed answer, and no
|
|
14
|
+
resolver attests what happened.
|
|
15
|
+
|
|
16
|
+
Every run gets its own database, as role `ff_*` connections need (0008): a fresh copy of
|
|
17
|
+
the migrated template database (0009, `fivefold.harness.migrations.create_run_database`),
|
|
18
|
+
which replaces the database of the demo's last run. Each run writes its row in the run
|
|
19
|
+
registry, so the viewer lists it (0012). The admin connection comes from
|
|
20
|
+
`FIVEFOLD_ADMIN_DSN`, as for the tests. A demo needs no API key: the model is scripted
|
|
21
|
+
(0009).
|
|
22
|
+
|
|
23
|
+
The demo reads the gateway's records only to show what the gateway did and told the
|
|
24
|
+
agent; the verdict comes from the evaluator, which never reads them to decide (0004).
|
|
25
|
+
|
|
26
|
+
`--out DIR` saves each run's result JSON in `DIR`, one file per demo, configuration and
|
|
27
|
+
seed, for `python -m fivefold.evaluator report`.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import argparse
|
|
33
|
+
import asyncio
|
|
34
|
+
import hashlib
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
import sys
|
|
38
|
+
from collections.abc import Callable, Sequence
|
|
39
|
+
from dataclasses import dataclass, replace
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from typing import Any
|
|
42
|
+
|
|
43
|
+
import psycopg
|
|
44
|
+
|
|
45
|
+
from fivefold.domain.contracts import Contract
|
|
46
|
+
from fivefold.domain.scenario import Scenario, load_scenario_file
|
|
47
|
+
from fivefold.evaluator import RunResult as Evaluation
|
|
48
|
+
from fivefold.evaluator import result_json
|
|
49
|
+
from fivefold.harness.migrations import (
|
|
50
|
+
DEFAULT_ADMIN_DSN,
|
|
51
|
+
bootstrap_server,
|
|
52
|
+
create_run_database,
|
|
53
|
+
)
|
|
54
|
+
from fivefold.harness.run import GATEWAY_CONFIGS, Config, InprocRun, RunSpec
|
|
55
|
+
from fivefold.paths import DB_DIR
|
|
56
|
+
from fivefold.paths import DEV_001 as DEV_001
|
|
57
|
+
from fivefold.paths import DEV_002 as DEV_002
|
|
58
|
+
|
|
59
|
+
ORDER = "submit_purchase_order"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _lost_reply(config: Config) -> dict[str, Any]:
|
|
63
|
+
"""F2 on the purchase order's committed answer, on the stack's hop to the simulator."""
|
|
64
|
+
hop = "gw_sim" if config in GATEWAY_CONFIGS else "agent_sim"
|
|
65
|
+
return {
|
|
66
|
+
"fault": "F2",
|
|
67
|
+
"at": {
|
|
68
|
+
"point": f"net.{hop}.answer",
|
|
69
|
+
"match": {"api": ORDER, "answer": "committed"},
|
|
70
|
+
"occurrence": 1,
|
|
71
|
+
},
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _before_the_send(config: Config) -> tuple[str, dict[str, str]]:
|
|
76
|
+
"""Where demo 2's faults fire: at the gateway's dispatch check in G1 and G2, and in B0 as the
|
|
77
|
+
approval answer reaches the agent (0006, F6 and F7 early)."""
|
|
78
|
+
if config in GATEWAY_CONFIGS:
|
|
79
|
+
return "gw.dispatch.before_check", {}
|
|
80
|
+
return "net.agent_sim.answer", {"api": "get_approval_status"}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def demo_1_plan(config: Config) -> list[dict[str, Any]]:
|
|
84
|
+
"""Demo 1's faults (0006): F2 on the order's answer, F11 after the order call."""
|
|
85
|
+
return [
|
|
86
|
+
_lost_reply(config),
|
|
87
|
+
{
|
|
88
|
+
"fault": "F11",
|
|
89
|
+
"at": {
|
|
90
|
+
"point": "agent.after_tool_call",
|
|
91
|
+
"match": {"tool": ORDER},
|
|
92
|
+
"occurrence": 1,
|
|
93
|
+
},
|
|
94
|
+
"params": {"restart_delay_ms": 10_000},
|
|
95
|
+
},
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def demo_2a_plan(config: Config) -> list[dict[str, Any]]:
|
|
100
|
+
"""Demo 2a's fault (0006): F7 early, `price_up`."""
|
|
101
|
+
point, match = _before_the_send(config)
|
|
102
|
+
return [
|
|
103
|
+
{
|
|
104
|
+
"fault": "F7",
|
|
105
|
+
"variant": "price_up",
|
|
106
|
+
"at": {"point": point, "match": match, "occurrence": 1},
|
|
107
|
+
}
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def demo_2b_plan(config: Config) -> list[dict[str, Any]]:
|
|
112
|
+
"""Demo 2b's fault (0006): F6 early."""
|
|
113
|
+
point, match = _before_the_send(config)
|
|
114
|
+
return [
|
|
115
|
+
{
|
|
116
|
+
"fault": "F6",
|
|
117
|
+
"variant": "early",
|
|
118
|
+
"at": {"point": point, "match": match, "occurrence": 1},
|
|
119
|
+
}
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def demo_3_plan(config: Config) -> list[dict[str, Any]]:
|
|
124
|
+
"""Demo 3's fault (0006): F2, with no resolver."""
|
|
125
|
+
return [_lost_reply(config)]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True)
|
|
129
|
+
class Demo:
|
|
130
|
+
"""One demo of record 0006: what it shows, and how each configuration runs it."""
|
|
131
|
+
|
|
132
|
+
key: str
|
|
133
|
+
title: str
|
|
134
|
+
scenario: Path
|
|
135
|
+
contract: Contract
|
|
136
|
+
faults: str
|
|
137
|
+
plan: Callable[[Config], list[dict[str, Any]]]
|
|
138
|
+
|
|
139
|
+
def spec(self, config: Config, seed: int, scenario: Scenario | None = None) -> RunSpec:
|
|
140
|
+
"""The demo's run in one configuration."""
|
|
141
|
+
loaded = scenario if scenario is not None else load_scenario_file(self.scenario)
|
|
142
|
+
return RunSpec(loaded, config, self.contract, seed, plan=self.plan(config))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
DEMOS: tuple[Demo, ...] = (
|
|
146
|
+
Demo(
|
|
147
|
+
"1",
|
|
148
|
+
"Order placed, reply lost",
|
|
149
|
+
DEV_001,
|
|
150
|
+
Contract.C2,
|
|
151
|
+
"F2 drops the order's committed answer; F11 kills the agent after the order call.",
|
|
152
|
+
demo_1_plan,
|
|
153
|
+
),
|
|
154
|
+
Demo(
|
|
155
|
+
"2a",
|
|
156
|
+
"Approval goes stale: quote",
|
|
157
|
+
DEV_002,
|
|
158
|
+
Contract.C2,
|
|
159
|
+
"A human approves the order; then F7 raises the quote's price before it is sent.",
|
|
160
|
+
demo_2a_plan,
|
|
161
|
+
),
|
|
162
|
+
Demo(
|
|
163
|
+
"2b",
|
|
164
|
+
"Approval goes stale: revoked",
|
|
165
|
+
DEV_002,
|
|
166
|
+
Contract.C2,
|
|
167
|
+
"A human approves the order; then F6 revokes the approval before it is sent.",
|
|
168
|
+
demo_2b_plan,
|
|
169
|
+
),
|
|
170
|
+
Demo(
|
|
171
|
+
"3",
|
|
172
|
+
"Outcome unknown",
|
|
173
|
+
DEV_001,
|
|
174
|
+
Contract.C0,
|
|
175
|
+
"F2 drops the order's committed answer; C0 offers no lookup, and no resolver attests.",
|
|
176
|
+
demo_3_plan,
|
|
177
|
+
),
|
|
178
|
+
)
|
|
179
|
+
"""The demos of record 0006, in order."""
|
|
180
|
+
|
|
181
|
+
DEMO_BY_KEY = {demo.key: demo for demo in DEMOS}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def demo_1_spec(config: Config, seed: int, scenario: Scenario | None = None) -> RunSpec:
|
|
185
|
+
"""Demo 1 in one configuration."""
|
|
186
|
+
return DEMO_BY_KEY["1"].spec(config, seed, scenario)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True)
|
|
190
|
+
class Summary:
|
|
191
|
+
"""What a run did, read back from its database."""
|
|
192
|
+
|
|
193
|
+
config: str
|
|
194
|
+
effects: list[tuple[str, str]]
|
|
195
|
+
budget: dict[str, int]
|
|
196
|
+
claims: list[dict[str, Any]]
|
|
197
|
+
faults: list[tuple[str, str, int]]
|
|
198
|
+
messages: int
|
|
199
|
+
digest: str
|
|
200
|
+
ended: str
|
|
201
|
+
verdict: str = ""
|
|
202
|
+
gateway: tuple[str, ...] = ()
|
|
203
|
+
"""What the gateway did with each action, and what it told the agent to reconcile."""
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def summarize(dsn: str, config: str, ended: str) -> Summary:
|
|
207
|
+
"""Read a finished run's ledger, report, faults and message log."""
|
|
208
|
+
with psycopg.connect(dsn) as conn:
|
|
209
|
+
effects = [
|
|
210
|
+
(str(effect_id), str(approval_id))
|
|
211
|
+
for effect_id, approval_id in conn.execute(
|
|
212
|
+
"SELECT effect_id, approval_id FROM sim.effect ORDER BY commit_seq"
|
|
213
|
+
)
|
|
214
|
+
]
|
|
215
|
+
budget_row = conn.execute(
|
|
216
|
+
"SELECT value FROM sim.fact_version WHERE key = 'budget/CC-OPS'"
|
|
217
|
+
" ORDER BY version DESC LIMIT 1"
|
|
218
|
+
).fetchone()
|
|
219
|
+
report = conn.execute(
|
|
220
|
+
"SELECT report FROM harness.final_report WHERE valid ORDER BY seq DESC LIMIT 1"
|
|
221
|
+
).fetchone()
|
|
222
|
+
faults = [
|
|
223
|
+
(str(fault), str(point), int(vt))
|
|
224
|
+
for fault, point, vt in conn.execute(
|
|
225
|
+
"SELECT fault, point, vt FROM harness.fault_log ORDER BY seq"
|
|
226
|
+
)
|
|
227
|
+
]
|
|
228
|
+
log = conn.execute(
|
|
229
|
+
"SELECT seq, hop, kind, sender, receiver, api, correlation, intent_id,"
|
|
230
|
+
" idempotency_key, answer_class, body, sent_vt, fate, drop_reason,"
|
|
231
|
+
" delivered_seq, delivered_vt FROM harness.message_log ORDER BY seq"
|
|
232
|
+
).fetchall()
|
|
233
|
+
gateway = _gateway_lines(conn) if config != "B0" else ()
|
|
234
|
+
text = json.dumps([list(row) for row in log], sort_keys=True, default=str)
|
|
235
|
+
return Summary(
|
|
236
|
+
config=config,
|
|
237
|
+
effects=effects,
|
|
238
|
+
budget=dict(budget_row[0]) if budget_row else {},
|
|
239
|
+
claims=list(report[0]["claims"]) if report else [],
|
|
240
|
+
faults=faults,
|
|
241
|
+
messages=len(log),
|
|
242
|
+
digest=hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
|
243
|
+
ended=ended,
|
|
244
|
+
gateway=gateway,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _gateway_lines(conn: psycopg.Connection[Any]) -> tuple[str, ...]:
|
|
249
|
+
"""One line per action: its state, why, the stale facts it named, and its approval."""
|
|
250
|
+
lines: list[str] = []
|
|
251
|
+
actions = conn.execute(
|
|
252
|
+
"SELECT a.action_id, a.state, g.approval_id FROM gw.action a"
|
|
253
|
+
' LEFT JOIN gw."grant" g USING (action_id) ORDER BY a.action_id'
|
|
254
|
+
).fetchall()
|
|
255
|
+
for action_id, state, approval_id in actions:
|
|
256
|
+
event = conn.execute(
|
|
257
|
+
"SELECT reason, details FROM gw.action_event WHERE action_id = %s"
|
|
258
|
+
" AND to_state IS NOT NULL ORDER BY seq DESC LIMIT 1",
|
|
259
|
+
(action_id,),
|
|
260
|
+
).fetchone()
|
|
261
|
+
reason, details = (str(event[0]), event[1]) if event else ("", None)
|
|
262
|
+
line = f"{action_id} {state} ({reason}), approval {approval_id}"
|
|
263
|
+
stale = details.get("stale") if isinstance(details, dict) else None
|
|
264
|
+
for fact in stale or ():
|
|
265
|
+
changed = ", ".join(fact.get("changed_fields") or ())
|
|
266
|
+
line += (
|
|
267
|
+
f"; stale: {fact['key']} version {fact['cited_version']} to"
|
|
268
|
+
f" {fact['current_version']} ({changed})"
|
|
269
|
+
)
|
|
270
|
+
lines.append(line)
|
|
271
|
+
told = conn.execute(
|
|
272
|
+
"SELECT result->'reconcile' FROM gw.tool_call"
|
|
273
|
+
" WHERE result->'reconcile' IS NOT NULL AND result->'reconcile' <> 'null'::jsonb"
|
|
274
|
+
" ORDER BY call_no DESC LIMIT 1"
|
|
275
|
+
).fetchone()
|
|
276
|
+
if told is not None:
|
|
277
|
+
reconcile = told[0]
|
|
278
|
+
lines.append(
|
|
279
|
+
f"told the agent to reconcile: {reconcile['how']} ({reconcile['escalation_id']}):"
|
|
280
|
+
f" {reconcile['what']}"
|
|
281
|
+
)
|
|
282
|
+
return tuple(lines)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def save_result(
|
|
286
|
+
out: Path, demo: Demo, config: Config, seed: int, evaluation: Evaluation | None
|
|
287
|
+
) -> Path:
|
|
288
|
+
"""Save a run's result JSON in the directory `out`, in a file named for the demo,
|
|
289
|
+
the configuration and the seed, and return its path."""
|
|
290
|
+
if evaluation is None:
|
|
291
|
+
raise RuntimeError(f"demo {demo.key} in {config} was not evaluated")
|
|
292
|
+
path = out / f"demo-{demo.key}-{config}-seed-{seed}.json"
|
|
293
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
294
|
+
path.write_bytes(result_json(evaluation).encode("utf-8"))
|
|
295
|
+
return path
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def run_demo(
|
|
299
|
+
admin_dsn: str, demo: Demo, config: Config, seed: int, *, out: Path | None = None
|
|
300
|
+
) -> Summary:
|
|
301
|
+
"""Run one demo in one configuration, in a fresh copy of the template database, and
|
|
302
|
+
summarize it. With `out`, also save the run's result JSON there."""
|
|
303
|
+
name = f"fivefold_demo_{demo.key}_{config.lower()}"
|
|
304
|
+
dsn = create_run_database(admin_dsn, name, DB_DIR / "migrations", replace=True).dsn
|
|
305
|
+
run = InprocRun(demo.spec(config, seed), dsn)
|
|
306
|
+
result = asyncio.run(run.execute())
|
|
307
|
+
if out is not None:
|
|
308
|
+
save_result(out, demo, config, seed, result.evaluation)
|
|
309
|
+
ended = ", ".join(f"{worker}: {end[0]}" for worker, end in sorted(result.ends.items()))
|
|
310
|
+
summary = summarize(dsn, config, ended)
|
|
311
|
+
return replace(summary, verdict=verdict_line(result.evaluation))
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def run_demo_1(admin_dsn: str, config: Config, seed: int) -> Summary:
|
|
315
|
+
"""Run demo 1 in a fresh database and summarize it."""
|
|
316
|
+
return run_demo(admin_dsn, DEMO_BY_KEY["1"], config, seed)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def verdict_line(evaluation: Evaluation | None) -> str:
|
|
320
|
+
"""The verdict and each failing property's reasons, for the properties evaluated."""
|
|
321
|
+
if evaluation is None:
|
|
322
|
+
return "not evaluated"
|
|
323
|
+
parts = [str(evaluation.verdict)]
|
|
324
|
+
for result in evaluation.properties:
|
|
325
|
+
if str(result.verdict) == "fail":
|
|
326
|
+
parts.append(f"property {result.property} fails ({', '.join(result.reason_codes)})")
|
|
327
|
+
evaluated = ", ".join(str(number) for number in evaluation.evaluated)
|
|
328
|
+
return "; ".join(parts) + f" [properties evaluated: {evaluated}]"
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def render(summary: Summary) -> str:
|
|
332
|
+
"""A few lines a person can read."""
|
|
333
|
+
budget = summary.budget
|
|
334
|
+
left = budget.get("limit_minor", 0) - budget.get("committed_minor", 0)
|
|
335
|
+
lines = [
|
|
336
|
+
f"{summary.config}:",
|
|
337
|
+
f" orders in the ledger: {len(summary.effects)}"
|
|
338
|
+
+ "".join(
|
|
339
|
+
f"{', ' if i else ' '}{e} (approval {a})" for i, (e, a) in enumerate(summary.effects)
|
|
340
|
+
),
|
|
341
|
+
f" budget left: {left / 100:.2f} USD" + (" OVERDRAWN" if left < 0 else ""),
|
|
342
|
+
" report: "
|
|
343
|
+
+ "; ".join(
|
|
344
|
+
f"{c['intent_id']} {c['status']} {c.get('effect_ids') or ''}".strip()
|
|
345
|
+
for c in summary.claims
|
|
346
|
+
),
|
|
347
|
+
" faults: " + ", ".join(f"{f} at {p} (vt {vt})" for f, p, vt in summary.faults),
|
|
348
|
+
*(f" gateway: {line}" for line in summary.gateway),
|
|
349
|
+
f" agent: {summary.ended}",
|
|
350
|
+
f" verdict: {summary.verdict}",
|
|
351
|
+
f" message log: {summary.messages} messages, sha256 {summary.digest[:16]}",
|
|
352
|
+
]
|
|
353
|
+
return "\n".join(lines)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
357
|
+
"""Run the chosen demos for B0 and G2 and print them side by side."""
|
|
358
|
+
parser = argparse.ArgumentParser(
|
|
359
|
+
prog="python -m fivefold.harness.demo",
|
|
360
|
+
description="Run the demos of record 0006 for B0 and G2 from a seed.",
|
|
361
|
+
)
|
|
362
|
+
parser.add_argument(
|
|
363
|
+
"demos",
|
|
364
|
+
nargs="*",
|
|
365
|
+
metavar="DEMO",
|
|
366
|
+
help=f"which demos to run: {', '.join(DEMO_BY_KEY)} (default: all)",
|
|
367
|
+
)
|
|
368
|
+
parser.add_argument("--seed", type=int, default=7, help="the run seed (default 7)")
|
|
369
|
+
parser.add_argument(
|
|
370
|
+
"--dsn",
|
|
371
|
+
default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
|
|
372
|
+
help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
|
|
373
|
+
)
|
|
374
|
+
parser.add_argument(
|
|
375
|
+
"--out",
|
|
376
|
+
type=Path,
|
|
377
|
+
metavar="DIR",
|
|
378
|
+
help="save each run's result JSON in DIR, as demo-KEY-CONFIG-seed-SEED.json",
|
|
379
|
+
)
|
|
380
|
+
args = parser.parse_args(argv)
|
|
381
|
+
unknown = [key for key in args.demos if key not in DEMO_BY_KEY]
|
|
382
|
+
if unknown:
|
|
383
|
+
parser.error(f"no demo {', '.join(unknown)}; choose from {', '.join(DEMO_BY_KEY)}")
|
|
384
|
+
chosen = [DEMO_BY_KEY[key] for key in args.demos] or list(DEMOS)
|
|
385
|
+
bootstrap_server(args.dsn, DB_DIR)
|
|
386
|
+
for demo in chosen:
|
|
387
|
+
scenario = load_scenario_file(demo.scenario)
|
|
388
|
+
print(f"Demo {demo.key}: {demo.title} ({scenario.id}, {demo.contract}, seed {args.seed})")
|
|
389
|
+
print(f"{demo.faults}\n")
|
|
390
|
+
for config in ("B0", "G2"):
|
|
391
|
+
print(render(run_demo(args.dsn, demo, config, args.seed, out=args.out)))
|
|
392
|
+
print()
|
|
393
|
+
return 0
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
if __name__ == "__main__":
|
|
397
|
+
sys.exit(main())
|