fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,209 @@
1
+ """The run's virtual clock, and the only wall-clock access in the harness (0009).
2
+
3
+ Every component reads time only from the run's `VirtualClock`: grant expiry, timeouts,
4
+ `action_event.vt`, `attempt.sent_vt` and `effect.committed_vt` are virtual milliseconds.
5
+ This module and `rng.py` are the only places allowed to use wall-clock time or system
6
+ randomness; the lint configuration in pyproject.toml enforces it (0007). Wall-clock time
7
+ is read here for metrics only, never for decisions.
8
+
9
+ `settle` lets every other task of the event loop run until each one waits on something
10
+ the scheduler controls. It reads two internals of CPython 3.12's asyncio event loop, the
11
+ ready queue (`_ready`) and the timer heap (`_scheduled`), checked in
12
+ `Lib/asyncio/base_events.py` of Python 3.12.3. The project pins Python 3.12
13
+ (pyproject.toml), and `test_scheduler.py` fails loudly if either internal disappears.
14
+
15
+ A task that waits on the real world, as a live model call waits on its provider, does so
16
+ inside a real wait (`RealWaits.hold`). Network I/O registers nothing that `settle` can
17
+ see: an HTTP client with no timeout leaves no timer in the heap. So while a real wait is
18
+ in flight, `settle` waits for it to end, in wall time, and does not return. Virtual time
19
+ stands still meanwhile, and the call takes no virtual time of its own (0009).
20
+
21
+ In the split topology, virtual time follows the host's monotonic clock (0009).
22
+ `HostClock` reads it, counting milliseconds from the run's origin, and its timers are the
23
+ event loop's, in wall time. Every process of one split run reads the same origin, so on
24
+ one host they all read the same virtual time.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import asyncio
30
+ import time
31
+ from collections import deque
32
+ from collections.abc import Awaitable, Callable, Sequence
33
+ from typing import TypeVar, cast
34
+
35
+ T = TypeVar("T")
36
+
37
+ TIMER_SLACK_S = 0.000_5
38
+ """How much later than due a host timer fires, in seconds, so that the clock reads at least
39
+ its due time when it fires, whatever the rounding to whole milliseconds."""
40
+
41
+
42
+ class VirtualClock:
43
+ """Virtual milliseconds since the run started. Only the scheduler moves it."""
44
+
45
+ def __init__(self, start_ms: int = 0) -> None:
46
+ if start_ms < 0:
47
+ raise ValueError("virtual time starts at or after 0")
48
+ self._now = start_ms
49
+
50
+ @property
51
+ def now(self) -> int:
52
+ """The current virtual time, in milliseconds."""
53
+ return self._now
54
+
55
+ def advance_to(self, at_ms: int) -> None:
56
+ """Move the clock forward to `at_ms`. Virtual time never runs backwards."""
57
+ if at_ms < self._now:
58
+ raise ValueError(f"virtual time cannot go back from {self._now} to {at_ms}")
59
+ self._now = at_ms
60
+
61
+
62
+ class ClockBehind(RuntimeError):
63
+ """The host's clock reads earlier than the run's origin: the process is on another host."""
64
+
65
+
66
+ class HostClock(VirtualClock):
67
+ """The split topology's clock: virtual time follows the host's monotonic clock (0009).
68
+
69
+ Virtual milliseconds are the host's monotonic milliseconds since `origin_ms`, so the
70
+ processes of a run on one host, which share its origin, read the same time. The host
71
+ moves it; nothing else can. It has no events of its own: `call_at` hands a callback to
72
+ the running event loop, which calls it in wall time.
73
+ """
74
+
75
+ def __init__(self, origin_ms: int) -> None:
76
+ super().__init__(0)
77
+ self._origin = origin_ms
78
+
79
+ @staticmethod
80
+ def host_ms() -> int:
81
+ """The host's monotonic clock, in whole milliseconds."""
82
+ return time.monotonic_ns() // 1_000_000
83
+
84
+ @property
85
+ def origin_ms(self) -> int:
86
+ """The host's monotonic milliseconds at virtual time 0."""
87
+ return self._origin
88
+
89
+ @property
90
+ def now(self) -> int:
91
+ """Milliseconds since the origin. Raises `ClockBehind` before the origin."""
92
+ elapsed = self.host_ms() - self._origin
93
+ if elapsed < 0:
94
+ raise ClockBehind(
95
+ f"the host's clock reads {-elapsed} ms before the run's origin: every process"
96
+ " of a split run must run on one host"
97
+ )
98
+ return elapsed
99
+
100
+ def advance_to(self, at_ms: int) -> None:
101
+ """Refused: only the host moves this clock."""
102
+ raise RuntimeError("virtual time follows the host's clock here; nothing can move it")
103
+
104
+ def call_at(self, at_ms: int, callback: Callable[[], None]) -> asyncio.TimerHandle:
105
+ """Call `callback` on the running event loop once virtual time reaches `at_ms`."""
106
+ due_ns = (self._origin + at_ms) * 1_000_000
107
+ delay_s = max(0.0, (due_ns - time.monotonic_ns()) / 1e9) + TIMER_SLACK_S
108
+ return asyncio.get_running_loop().call_later(delay_s, callback)
109
+
110
+
111
+ def wall_ms() -> float:
112
+ """Wall-clock milliseconds from a monotonic clock, for metrics only (0009)."""
113
+ return time.monotonic() * 1000.0
114
+
115
+
116
+ class LoopInternalsMissing(RuntimeError):
117
+ """The running event loop lacks the CPython internals `settle` reads."""
118
+
119
+
120
+ def _ready_queue(loop: asyncio.AbstractEventLoop) -> deque[asyncio.Handle]:
121
+ ready = getattr(loop, "_ready", None)
122
+ if not isinstance(ready, deque):
123
+ raise LoopInternalsMissing("the event loop has no _ready deque (CPython 3.12 asyncio)")
124
+ return cast(deque[asyncio.Handle], ready)
125
+
126
+
127
+ def _timer_heap(loop: asyncio.AbstractEventLoop) -> Sequence[asyncio.TimerHandle]:
128
+ scheduled = getattr(loop, "_scheduled", None)
129
+ if not isinstance(scheduled, list):
130
+ raise LoopInternalsMissing("the event loop has no _scheduled list (CPython 3.12 asyncio)")
131
+ return cast(list[asyncio.TimerHandle], scheduled)
132
+
133
+
134
+ async def yield_to_loop() -> None:
135
+ """Let every callback that is ready now run once, then continue."""
136
+ await asyncio.sleep(0)
137
+
138
+
139
+ async def within[T](awaitable: Awaitable[T], limit_s: float) -> T:
140
+ """Await `awaitable`, a wait on the real world such as a live model call, for at most
141
+ `limit_s` wall-clock seconds. A wait still in flight then is cancelled and raises
142
+ `TimeoutError`, so a stalled provider cannot hold the run forever. The limit only
143
+ bounds a real wait; nothing in the run is decided by wall-clock time (0009)."""
144
+ return await asyncio.wait_for(awaitable, timeout=limit_s)
145
+
146
+
147
+ class RealWaits:
148
+ """The real waits in flight, such as live model calls awaiting their provider (0009).
149
+
150
+ While one is in flight, virtual time stands still: `settle` does not return, so the
151
+ scheduler fires no event, and the run waits for it in wall time. What a real wait
152
+ awaits must never wait on the scheduler, which cannot move until the wait ends.
153
+ """
154
+
155
+ def __init__(self) -> None:
156
+ self._count = 0
157
+ self._waiters: list[asyncio.Future[None]] = []
158
+
159
+ @property
160
+ def active(self) -> bool:
161
+ """Whether a real wait is in flight."""
162
+ return self._count > 0
163
+
164
+ async def hold(self, awaitable: Awaitable[T]) -> T:
165
+ """Await `awaitable`, a real wait, with virtual time held still until it ends."""
166
+ self._count += 1
167
+ try:
168
+ return await awaitable
169
+ finally:
170
+ self._count -= 1
171
+ if self._count == 0:
172
+ waiters, self._waiters = self._waiters, []
173
+ for waiter in waiters:
174
+ if not waiter.done():
175
+ waiter.set_result(None)
176
+
177
+ async def ended(self) -> None:
178
+ """Return once no real wait is in flight."""
179
+ if self._count == 0:
180
+ return
181
+ waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future()
182
+ self._waiters.append(waiter)
183
+ await waiter
184
+
185
+
186
+ async def settle(real: RealWaits | None = None) -> None:
187
+ """Return once every other task waits on something the scheduler controls.
188
+
189
+ Ready callbacks run first. A task in one of `real`'s real waits, such as a live model
190
+ call awaiting its provider, holds the run until the wait ends. A task that sleeps on a
191
+ wall-clock timer, such as LangGraph's retry backoff (0008), holds it until its timer
192
+ fires. Virtual time must not move during either: they change wall-clock metrics only
193
+ (0009). While a real wait is in flight, `settle` waits for that wait alone, so a timer
194
+ the real wait itself set, such as an HTTP client's timeout, never keeps the run
195
+ asleep after the answer arrives.
196
+ """
197
+ loop = asyncio.get_running_loop()
198
+ ready = _ready_queue(loop)
199
+ while True:
200
+ await yield_to_loop()
201
+ if ready:
202
+ continue
203
+ if real is not None and real.active:
204
+ await real.ended()
205
+ continue
206
+ pending = [timer.when() for timer in _timer_heap(loop) if not timer.cancelled()]
207
+ if not pending:
208
+ return
209
+ await asyncio.sleep(max(0.0, min(pending) - loop.time()))
@@ -0,0 +1,397 @@
1
+ """The demos of record 0006, run from a seed: `python -m fivefold.harness.demo` (`make demo`).
2
+
3
+ Each demo runs B0 and G2 on the same scenario and seed and shows them side by side
4
+ (0006, Demos):
5
+
6
+ 1. "Order placed, reply lost" (dev-001, C2): F2 drops the purchase order's committed
7
+ answer, and F11 kills the agent right after it sends that call, resuming it from its
8
+ checkpoint 10,000 virtual ms later.
9
+ 2. "Approval goes stale" (dev-002, C2): a human approves the order, then, before the
10
+ gateway sends it, F7 raises the quote's price (2a) or F6 revokes the approval (2b).
11
+ B0 meets the fault as the approval reaches the agent. dev-002 is dev-001 with the
12
+ auto-approval threshold below the order's amount, so the approval is a human's.
13
+ 3. "Outcome unknown" (dev-001, C0): F2 drops the order's committed answer, and no
14
+ resolver attests what happened.
15
+
16
+ Every run gets its own database, as role `ff_*` connections need (0008): a fresh copy of
17
+ the migrated template database (0009, `fivefold.harness.migrations.create_run_database`),
18
+ which replaces the database of the demo's last run. Each run writes its row in the run
19
+ registry, so the viewer lists it (0012). The admin connection comes from
20
+ `FIVEFOLD_ADMIN_DSN`, as for the tests. A demo needs no API key: the model is scripted
21
+ (0009).
22
+
23
+ The demo reads the gateway's records only to show what the gateway did and told the
24
+ agent; the verdict comes from the evaluator, which never reads them to decide (0004).
25
+
26
+ `--out DIR` saves each run's result JSON in `DIR`, one file per demo, configuration and
27
+ seed, for `python -m fivefold.evaluator report`.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import argparse
33
+ import asyncio
34
+ import hashlib
35
+ import json
36
+ import os
37
+ import sys
38
+ from collections.abc import Callable, Sequence
39
+ from dataclasses import dataclass, replace
40
+ from pathlib import Path
41
+ from typing import Any
42
+
43
+ import psycopg
44
+
45
+ from fivefold.domain.contracts import Contract
46
+ from fivefold.domain.scenario import Scenario, load_scenario_file
47
+ from fivefold.evaluator import RunResult as Evaluation
48
+ from fivefold.evaluator import result_json
49
+ from fivefold.harness.migrations import (
50
+ DEFAULT_ADMIN_DSN,
51
+ bootstrap_server,
52
+ create_run_database,
53
+ )
54
+ from fivefold.harness.run import GATEWAY_CONFIGS, Config, InprocRun, RunSpec
55
+ from fivefold.paths import DB_DIR
56
+ from fivefold.paths import DEV_001 as DEV_001
57
+ from fivefold.paths import DEV_002 as DEV_002
58
+
59
+ ORDER = "submit_purchase_order"
60
+
61
+
62
+ def _lost_reply(config: Config) -> dict[str, Any]:
63
+ """F2 on the purchase order's committed answer, on the stack's hop to the simulator."""
64
+ hop = "gw_sim" if config in GATEWAY_CONFIGS else "agent_sim"
65
+ return {
66
+ "fault": "F2",
67
+ "at": {
68
+ "point": f"net.{hop}.answer",
69
+ "match": {"api": ORDER, "answer": "committed"},
70
+ "occurrence": 1,
71
+ },
72
+ }
73
+
74
+
75
+ def _before_the_send(config: Config) -> tuple[str, dict[str, str]]:
76
+ """Where demo 2's faults fire: at the gateway's dispatch check in G1 and G2, and in B0 as the
77
+ approval answer reaches the agent (0006, F6 and F7 early)."""
78
+ if config in GATEWAY_CONFIGS:
79
+ return "gw.dispatch.before_check", {}
80
+ return "net.agent_sim.answer", {"api": "get_approval_status"}
81
+
82
+
83
+ def demo_1_plan(config: Config) -> list[dict[str, Any]]:
84
+ """Demo 1's faults (0006): F2 on the order's answer, F11 after the order call."""
85
+ return [
86
+ _lost_reply(config),
87
+ {
88
+ "fault": "F11",
89
+ "at": {
90
+ "point": "agent.after_tool_call",
91
+ "match": {"tool": ORDER},
92
+ "occurrence": 1,
93
+ },
94
+ "params": {"restart_delay_ms": 10_000},
95
+ },
96
+ ]
97
+
98
+
99
+ def demo_2a_plan(config: Config) -> list[dict[str, Any]]:
100
+ """Demo 2a's fault (0006): F7 early, `price_up`."""
101
+ point, match = _before_the_send(config)
102
+ return [
103
+ {
104
+ "fault": "F7",
105
+ "variant": "price_up",
106
+ "at": {"point": point, "match": match, "occurrence": 1},
107
+ }
108
+ ]
109
+
110
+
111
+ def demo_2b_plan(config: Config) -> list[dict[str, Any]]:
112
+ """Demo 2b's fault (0006): F6 early."""
113
+ point, match = _before_the_send(config)
114
+ return [
115
+ {
116
+ "fault": "F6",
117
+ "variant": "early",
118
+ "at": {"point": point, "match": match, "occurrence": 1},
119
+ }
120
+ ]
121
+
122
+
123
+ def demo_3_plan(config: Config) -> list[dict[str, Any]]:
124
+ """Demo 3's fault (0006): F2, with no resolver."""
125
+ return [_lost_reply(config)]
126
+
127
+
128
+ @dataclass(frozen=True)
129
+ class Demo:
130
+ """One demo of record 0006: what it shows, and how each configuration runs it."""
131
+
132
+ key: str
133
+ title: str
134
+ scenario: Path
135
+ contract: Contract
136
+ faults: str
137
+ plan: Callable[[Config], list[dict[str, Any]]]
138
+
139
+ def spec(self, config: Config, seed: int, scenario: Scenario | None = None) -> RunSpec:
140
+ """The demo's run in one configuration."""
141
+ loaded = scenario if scenario is not None else load_scenario_file(self.scenario)
142
+ return RunSpec(loaded, config, self.contract, seed, plan=self.plan(config))
143
+
144
+
145
+ DEMOS: tuple[Demo, ...] = (
146
+ Demo(
147
+ "1",
148
+ "Order placed, reply lost",
149
+ DEV_001,
150
+ Contract.C2,
151
+ "F2 drops the order's committed answer; F11 kills the agent after the order call.",
152
+ demo_1_plan,
153
+ ),
154
+ Demo(
155
+ "2a",
156
+ "Approval goes stale: quote",
157
+ DEV_002,
158
+ Contract.C2,
159
+ "A human approves the order; then F7 raises the quote's price before it is sent.",
160
+ demo_2a_plan,
161
+ ),
162
+ Demo(
163
+ "2b",
164
+ "Approval goes stale: revoked",
165
+ DEV_002,
166
+ Contract.C2,
167
+ "A human approves the order; then F6 revokes the approval before it is sent.",
168
+ demo_2b_plan,
169
+ ),
170
+ Demo(
171
+ "3",
172
+ "Outcome unknown",
173
+ DEV_001,
174
+ Contract.C0,
175
+ "F2 drops the order's committed answer; C0 offers no lookup, and no resolver attests.",
176
+ demo_3_plan,
177
+ ),
178
+ )
179
+ """The demos of record 0006, in order."""
180
+
181
+ DEMO_BY_KEY = {demo.key: demo for demo in DEMOS}
182
+
183
+
184
+ def demo_1_spec(config: Config, seed: int, scenario: Scenario | None = None) -> RunSpec:
185
+ """Demo 1 in one configuration."""
186
+ return DEMO_BY_KEY["1"].spec(config, seed, scenario)
187
+
188
+
189
+ @dataclass(frozen=True)
190
+ class Summary:
191
+ """What a run did, read back from its database."""
192
+
193
+ config: str
194
+ effects: list[tuple[str, str]]
195
+ budget: dict[str, int]
196
+ claims: list[dict[str, Any]]
197
+ faults: list[tuple[str, str, int]]
198
+ messages: int
199
+ digest: str
200
+ ended: str
201
+ verdict: str = ""
202
+ gateway: tuple[str, ...] = ()
203
+ """What the gateway did with each action, and what it told the agent to reconcile."""
204
+
205
+
206
+ def summarize(dsn: str, config: str, ended: str) -> Summary:
207
+ """Read a finished run's ledger, report, faults and message log."""
208
+ with psycopg.connect(dsn) as conn:
209
+ effects = [
210
+ (str(effect_id), str(approval_id))
211
+ for effect_id, approval_id in conn.execute(
212
+ "SELECT effect_id, approval_id FROM sim.effect ORDER BY commit_seq"
213
+ )
214
+ ]
215
+ budget_row = conn.execute(
216
+ "SELECT value FROM sim.fact_version WHERE key = 'budget/CC-OPS'"
217
+ " ORDER BY version DESC LIMIT 1"
218
+ ).fetchone()
219
+ report = conn.execute(
220
+ "SELECT report FROM harness.final_report WHERE valid ORDER BY seq DESC LIMIT 1"
221
+ ).fetchone()
222
+ faults = [
223
+ (str(fault), str(point), int(vt))
224
+ for fault, point, vt in conn.execute(
225
+ "SELECT fault, point, vt FROM harness.fault_log ORDER BY seq"
226
+ )
227
+ ]
228
+ log = conn.execute(
229
+ "SELECT seq, hop, kind, sender, receiver, api, correlation, intent_id,"
230
+ " idempotency_key, answer_class, body, sent_vt, fate, drop_reason,"
231
+ " delivered_seq, delivered_vt FROM harness.message_log ORDER BY seq"
232
+ ).fetchall()
233
+ gateway = _gateway_lines(conn) if config != "B0" else ()
234
+ text = json.dumps([list(row) for row in log], sort_keys=True, default=str)
235
+ return Summary(
236
+ config=config,
237
+ effects=effects,
238
+ budget=dict(budget_row[0]) if budget_row else {},
239
+ claims=list(report[0]["claims"]) if report else [],
240
+ faults=faults,
241
+ messages=len(log),
242
+ digest=hashlib.sha256(text.encode("utf-8")).hexdigest(),
243
+ ended=ended,
244
+ gateway=gateway,
245
+ )
246
+
247
+
248
+ def _gateway_lines(conn: psycopg.Connection[Any]) -> tuple[str, ...]:
249
+ """One line per action: its state, why, the stale facts it named, and its approval."""
250
+ lines: list[str] = []
251
+ actions = conn.execute(
252
+ "SELECT a.action_id, a.state, g.approval_id FROM gw.action a"
253
+ ' LEFT JOIN gw."grant" g USING (action_id) ORDER BY a.action_id'
254
+ ).fetchall()
255
+ for action_id, state, approval_id in actions:
256
+ event = conn.execute(
257
+ "SELECT reason, details FROM gw.action_event WHERE action_id = %s"
258
+ " AND to_state IS NOT NULL ORDER BY seq DESC LIMIT 1",
259
+ (action_id,),
260
+ ).fetchone()
261
+ reason, details = (str(event[0]), event[1]) if event else ("", None)
262
+ line = f"{action_id} {state} ({reason}), approval {approval_id}"
263
+ stale = details.get("stale") if isinstance(details, dict) else None
264
+ for fact in stale or ():
265
+ changed = ", ".join(fact.get("changed_fields") or ())
266
+ line += (
267
+ f"; stale: {fact['key']} version {fact['cited_version']} to"
268
+ f" {fact['current_version']} ({changed})"
269
+ )
270
+ lines.append(line)
271
+ told = conn.execute(
272
+ "SELECT result->'reconcile' FROM gw.tool_call"
273
+ " WHERE result->'reconcile' IS NOT NULL AND result->'reconcile' <> 'null'::jsonb"
274
+ " ORDER BY call_no DESC LIMIT 1"
275
+ ).fetchone()
276
+ if told is not None:
277
+ reconcile = told[0]
278
+ lines.append(
279
+ f"told the agent to reconcile: {reconcile['how']} ({reconcile['escalation_id']}):"
280
+ f" {reconcile['what']}"
281
+ )
282
+ return tuple(lines)
283
+
284
+
285
+ def save_result(
286
+ out: Path, demo: Demo, config: Config, seed: int, evaluation: Evaluation | None
287
+ ) -> Path:
288
+ """Save a run's result JSON in the directory `out`, in a file named for the demo,
289
+ the configuration and the seed, and return its path."""
290
+ if evaluation is None:
291
+ raise RuntimeError(f"demo {demo.key} in {config} was not evaluated")
292
+ path = out / f"demo-{demo.key}-{config}-seed-{seed}.json"
293
+ out.mkdir(parents=True, exist_ok=True)
294
+ path.write_bytes(result_json(evaluation).encode("utf-8"))
295
+ return path
296
+
297
+
298
+ def run_demo(
299
+ admin_dsn: str, demo: Demo, config: Config, seed: int, *, out: Path | None = None
300
+ ) -> Summary:
301
+ """Run one demo in one configuration, in a fresh copy of the template database, and
302
+ summarize it. With `out`, also save the run's result JSON there."""
303
+ name = f"fivefold_demo_{demo.key}_{config.lower()}"
304
+ dsn = create_run_database(admin_dsn, name, DB_DIR / "migrations", replace=True).dsn
305
+ run = InprocRun(demo.spec(config, seed), dsn)
306
+ result = asyncio.run(run.execute())
307
+ if out is not None:
308
+ save_result(out, demo, config, seed, result.evaluation)
309
+ ended = ", ".join(f"{worker}: {end[0]}" for worker, end in sorted(result.ends.items()))
310
+ summary = summarize(dsn, config, ended)
311
+ return replace(summary, verdict=verdict_line(result.evaluation))
312
+
313
+
314
+ def run_demo_1(admin_dsn: str, config: Config, seed: int) -> Summary:
315
+ """Run demo 1 in a fresh database and summarize it."""
316
+ return run_demo(admin_dsn, DEMO_BY_KEY["1"], config, seed)
317
+
318
+
319
+ def verdict_line(evaluation: Evaluation | None) -> str:
320
+ """The verdict and each failing property's reasons, for the properties evaluated."""
321
+ if evaluation is None:
322
+ return "not evaluated"
323
+ parts = [str(evaluation.verdict)]
324
+ for result in evaluation.properties:
325
+ if str(result.verdict) == "fail":
326
+ parts.append(f"property {result.property} fails ({', '.join(result.reason_codes)})")
327
+ evaluated = ", ".join(str(number) for number in evaluation.evaluated)
328
+ return "; ".join(parts) + f" [properties evaluated: {evaluated}]"
329
+
330
+
331
+ def render(summary: Summary) -> str:
332
+ """A few lines a person can read."""
333
+ budget = summary.budget
334
+ left = budget.get("limit_minor", 0) - budget.get("committed_minor", 0)
335
+ lines = [
336
+ f"{summary.config}:",
337
+ f" orders in the ledger: {len(summary.effects)}"
338
+ + "".join(
339
+ f"{', ' if i else ' '}{e} (approval {a})" for i, (e, a) in enumerate(summary.effects)
340
+ ),
341
+ f" budget left: {left / 100:.2f} USD" + (" OVERDRAWN" if left < 0 else ""),
342
+ " report: "
343
+ + "; ".join(
344
+ f"{c['intent_id']} {c['status']} {c.get('effect_ids') or ''}".strip()
345
+ for c in summary.claims
346
+ ),
347
+ " faults: " + ", ".join(f"{f} at {p} (vt {vt})" for f, p, vt in summary.faults),
348
+ *(f" gateway: {line}" for line in summary.gateway),
349
+ f" agent: {summary.ended}",
350
+ f" verdict: {summary.verdict}",
351
+ f" message log: {summary.messages} messages, sha256 {summary.digest[:16]}",
352
+ ]
353
+ return "\n".join(lines)
354
+
355
+
356
+ def main(argv: Sequence[str] | None = None) -> int:
357
+ """Run the chosen demos for B0 and G2 and print them side by side."""
358
+ parser = argparse.ArgumentParser(
359
+ prog="python -m fivefold.harness.demo",
360
+ description="Run the demos of record 0006 for B0 and G2 from a seed.",
361
+ )
362
+ parser.add_argument(
363
+ "demos",
364
+ nargs="*",
365
+ metavar="DEMO",
366
+ help=f"which demos to run: {', '.join(DEMO_BY_KEY)} (default: all)",
367
+ )
368
+ parser.add_argument("--seed", type=int, default=7, help="the run seed (default 7)")
369
+ parser.add_argument(
370
+ "--dsn",
371
+ default=os.environ.get("FIVEFOLD_ADMIN_DSN", DEFAULT_ADMIN_DSN),
372
+ help="admin connection string (default: FIVEFOLD_ADMIN_DSN)",
373
+ )
374
+ parser.add_argument(
375
+ "--out",
376
+ type=Path,
377
+ metavar="DIR",
378
+ help="save each run's result JSON in DIR, as demo-KEY-CONFIG-seed-SEED.json",
379
+ )
380
+ args = parser.parse_args(argv)
381
+ unknown = [key for key in args.demos if key not in DEMO_BY_KEY]
382
+ if unknown:
383
+ parser.error(f"no demo {', '.join(unknown)}; choose from {', '.join(DEMO_BY_KEY)}")
384
+ chosen = [DEMO_BY_KEY[key] for key in args.demos] or list(DEMOS)
385
+ bootstrap_server(args.dsn, DB_DIR)
386
+ for demo in chosen:
387
+ scenario = load_scenario_file(demo.scenario)
388
+ print(f"Demo {demo.key}: {demo.title} ({scenario.id}, {demo.contract}, seed {args.seed})")
389
+ print(f"{demo.faults}\n")
390
+ for config in ("B0", "G2"):
391
+ print(render(run_demo(args.dsn, demo, config, args.seed, out=args.out)))
392
+ print()
393
+ return 0
394
+
395
+
396
+ if __name__ == "__main__":
397
+ sys.exit(main())