fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,152 @@
1
+ """A split run's directory: what its services and commands share (0009).
2
+
3
+ `start` writes it; the services, `end` and `score` read it. Its files:
4
+
5
+ - `run.json`: the run's id, its database, configuration, contract level and seed;
6
+ - `scenario.yaml`: the scenario, exactly as the run recorded its digest;
7
+ - `callers.json`: the SHA-256 of each holder's secret (`credentials.Callers`);
8
+ - `gateway.token`, `agent-1.token` and `harness.token`: each holder's secret, which only
9
+ the file's owner may read (mode 0600);
10
+ - `ended`: the virtual time the run stopped at, written once `end` has stopped it.
11
+
12
+ It holds no database password: the component roles have none (`db/bootstrap.sql`), and
13
+ each service connects as its own role to the database `run.json` names.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import os
20
+ from dataclasses import dataclass
21
+ from pathlib import Path
22
+ from typing import Literal, cast
23
+
24
+ from psycopg.conninfo import make_conninfo
25
+
26
+ from fivefold.domain.contracts import Contract
27
+ from fivefold.domain.scenario import Scenario, load_scenario_file
28
+ from fivefold.harness.split.credentials import Callers
29
+
30
+ Config = Literal["B0", "G1", "G2"]
31
+
32
+ RUN_FILE = "run.json"
33
+ SCENARIO_FILE = "scenario.yaml"
34
+ CALLERS_FILE = "callers.json"
35
+ ENDED_FILE = "ended"
36
+ TOKEN_SUFFIX = ".token"
37
+
38
+ FILES = (RUN_FILE, SCENARIO_FILE, CALLERS_FILE, ENDED_FILE)
39
+ """The directory's files, besides the tokens."""
40
+
41
+
42
+ class RunDirError(Exception):
43
+ """The run directory is missing, incomplete, or holds another run."""
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class SplitRun:
48
+ """A split run, as its directory describes it."""
49
+
50
+ directory: Path
51
+ run_id: str
52
+ database: str
53
+ config: Config
54
+ contract: Contract
55
+ seed: int
56
+
57
+ @property
58
+ def has_gateway(self) -> bool:
59
+ """Whether the agent writes through the gateway: G1 and G2 (0008)."""
60
+ return self.config != "B0"
61
+
62
+ @property
63
+ def ended(self) -> bool:
64
+ """Whether `end` has stopped the run."""
65
+ return (self.directory / ENDED_FILE).exists()
66
+
67
+ def scenario(self) -> Scenario:
68
+ """The run's scenario."""
69
+ return load_scenario_file(self.directory / SCENARIO_FILE)
70
+
71
+ def callers(self) -> Callers:
72
+ """The digests of the run's secrets."""
73
+ data = json.loads((self.directory / CALLERS_FILE).read_text(encoding="utf-8"))
74
+ return Callers({str(holder): str(known) for holder, known in data.items()})
75
+
76
+ def secret(self, holder: str) -> str:
77
+ """`holder`'s secret."""
78
+ path = self.directory / f"{holder}{TOKEN_SUFFIX}"
79
+ if not path.exists():
80
+ raise RunDirError(f"{self.directory} holds no secret for {holder}")
81
+ return path.read_text(encoding="utf-8").strip()
82
+
83
+ def conninfo(self, base_dsn: str, role: str) -> str:
84
+ """A connection to the run's database as `role`, on the server `base_dsn` names."""
85
+ return make_conninfo(base_dsn, dbname=self.database, user=role)
86
+
87
+ def mark_ended(self, stopped_vt: int) -> None:
88
+ """Note that the run stopped at `stopped_vt`."""
89
+ (self.directory / ENDED_FILE).write_text(f"{stopped_vt}\n", encoding="utf-8")
90
+
91
+
92
+ def write_run(
93
+ run: SplitRun, *, scenario_text: str, secrets: dict[str, str], callers: Callers
94
+ ) -> None:
95
+ """Write `run`'s directory. The directory must hold no run."""
96
+ directory = run.directory
97
+ directory.mkdir(parents=True, exist_ok=True)
98
+ if (directory / RUN_FILE).exists():
99
+ raise RunDirError(f"{directory} already holds a run")
100
+ (directory / SCENARIO_FILE).write_text(scenario_text, encoding="utf-8")
101
+ (directory / CALLERS_FILE).write_text(
102
+ json.dumps(dict(callers.digests), indent=2, sort_keys=True) + "\n", encoding="utf-8"
103
+ )
104
+ for holder, secret in sorted(secrets.items()):
105
+ _write_private(directory / f"{holder}{TOKEN_SUFFIX}", secret + "\n")
106
+ description = {
107
+ "run_id": run.run_id,
108
+ "database": run.database,
109
+ "config": run.config,
110
+ "contract": str(run.contract),
111
+ "seed": run.seed,
112
+ }
113
+ (directory / RUN_FILE).write_text(
114
+ json.dumps(description, indent=2, sort_keys=True) + "\n", encoding="utf-8"
115
+ )
116
+
117
+
118
+ def _write_private(path: Path, text: str) -> None:
119
+ """Write `text` to a file only its owner may read or write."""
120
+ descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
121
+ with os.fdopen(descriptor, "w", encoding="utf-8") as file:
122
+ file.write(text)
123
+
124
+
125
+ def read_run(directory: Path) -> SplitRun:
126
+ """The run that `directory` describes. Raises `RunDirError`."""
127
+ path = directory / RUN_FILE
128
+ if not path.exists():
129
+ raise RunDirError(f"{directory} holds no split run: start one first")
130
+ try:
131
+ data = json.loads(path.read_text(encoding="utf-8"))
132
+ config = str(data["config"])
133
+ if config not in ("B0", "G1", "G2"):
134
+ raise ValueError(f"unknown configuration {config!r}")
135
+ return SplitRun(
136
+ directory=directory,
137
+ run_id=str(data["run_id"]),
138
+ database=str(data["database"]),
139
+ config=cast(Config, config),
140
+ contract=Contract(str(data["contract"])),
141
+ seed=int(data["seed"]),
142
+ )
143
+ except (KeyError, TypeError, ValueError) as error:
144
+ raise RunDirError(f"{path} does not describe a run: {error}") from error
145
+
146
+
147
+ def clear_run(directory: Path) -> None:
148
+ """Remove a run's files from `directory`, and nothing else."""
149
+ for name in FILES:
150
+ (directory / name).unlink(missing_ok=True)
151
+ for token in directory.glob(f"*{TOKEN_SUFFIX}"):
152
+ token.unlink()
@@ -0,0 +1,157 @@
1
+ """What the split topology's two services share: the run's records, clock and plan, and
2
+ some HTTP plumbing (0009).
3
+
4
+ A service opens its run from the run's directory: it connects to the run's database as
5
+ `ff_harness`, checks that the database holds the split run the directory names, with the
6
+ same scenario, and reads the run's seeded fault plan from `harness.run`. Its clock is a
7
+ `HostClock` with the run's origin, which the first service to start sets (migration
8
+ 0013), so both services read the same virtual time on one host.
9
+
10
+ Each service starts once per run. Its fault counters, whether the agents' phase is over,
11
+ and the scripted approver's timers live in its memory, so a service started again
12
+ mid-run would fire a fault again from its first occurrence, serve an agent the run has
13
+ stopped hearing, and never decide an approval it had scheduled. A service refuses to open
14
+ a run that has ended: `end` stopped it (the directory's `ended`), or the simulator
15
+ recorded its stop (`harness.run_stop`).
16
+
17
+ Checked in starlette 1.10.0: `Request.receive` is the ASGI receive callable, and after the
18
+ request's body is read, uvicorn 0.54.0's receive (`protocols/http/h11_impl.py`) returns
19
+ `http.disconnect` once the client closes the connection, and not before. Checked in
20
+ uvicorn 0.54.0 (`config.py`, `server.py`): `Config(app, host=, port=, lifespan=,
21
+ log_level=, timeout_graceful_shutdown=)` and `Server(config).run(sockets=)`.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import asyncio
27
+ from collections.abc import Mapping
28
+ from dataclasses import dataclass
29
+ from typing import Any
30
+
31
+ import psycopg
32
+ import uvicorn
33
+ from psycopg.rows import TupleRow
34
+ from starlette.requests import Request
35
+ from starlette.responses import JSONResponse
36
+
37
+ from fivefold.domain.scenario import Scenario, scenario_digest
38
+ from fivefold.faults.plan import PlannedFault, parse_plan
39
+ from fivefold.harness.clock import HostClock
40
+ from fivefold.harness.log import SharedPgHarnessLog
41
+ from fivefold.harness.scheduler import HostScheduler
42
+ from fivefold.harness.split.rundir import RunDirError, SplitRun
43
+
44
+ GRACEFUL_SHUTDOWN_S = 5
45
+ """How long a service lets its open requests finish when it shuts down, in seconds."""
46
+
47
+
48
+ @dataclass
49
+ class OpenRun:
50
+ """A service's hold on its run: the harness's connection, log, clock and plan."""
51
+
52
+ run: SplitRun
53
+ scenario: Scenario
54
+ conn: psycopg.Connection[TupleRow]
55
+ log: SharedPgHarnessLog
56
+ scheduler: HostScheduler
57
+ plan: tuple[PlannedFault, ...]
58
+ clock: HostClock
59
+ """The scheduler's clock, which still calls back once the scheduler has stopped."""
60
+
61
+ def stop_records(self) -> None:
62
+ """Write no more harness records: the run has stopped. Any later write fails."""
63
+ self.conn.close()
64
+
65
+ def close(self) -> None:
66
+ """Stop the run's timers and tasks here, and close the harness's connection."""
67
+ if not self.scheduler.stopped:
68
+ self.scheduler.stop()
69
+ self.conn.close()
70
+
71
+
72
+ def open_run(run: SplitRun, base_dsn: str) -> OpenRun:
73
+ """Open `run` for a service. Raises `RunDirError` when the database holds another run,
74
+ or the run has ended."""
75
+ scenario = run.scenario()
76
+ conn = psycopg.connect(run.conninfo(base_dsn, "ff_harness"), autocommit=True)
77
+ try:
78
+ row = conn.execute(
79
+ "SELECT run_id, scenario_digest, topology, params FROM harness.run"
80
+ ).fetchone()
81
+ if row is None or row[0] != run.run_id or row[2] != "split":
82
+ raise RunDirError(f"database {run.database} holds no split run {run.run_id}")
83
+ if row[1] != scenario_digest(scenario):
84
+ raise RunDirError(f"{run.directory}'s scenario is not the one run {run.run_id} has")
85
+ _refuse_ended(run, conn)
86
+ params: Mapping[str, Any] = row[3]
87
+ plan = parse_plan(params.get("plan", []))
88
+ clock = HostClock(clock_origin(conn))
89
+ _ = clock.now
90
+ scheduler = HostScheduler(clock)
91
+ return OpenRun(run, scenario, conn, SharedPgHarnessLog(conn), scheduler, plan, clock)
92
+ except BaseException:
93
+ conn.close()
94
+ raise
95
+
96
+
97
+ def refuse_ended(run: SplitRun, base_dsn: str) -> None:
98
+ """Raise `RunDirError` if `run` has ended, before a service starts for it."""
99
+ with psycopg.connect(run.conninfo(base_dsn, "ff_harness"), autocommit=True) as conn:
100
+ _refuse_ended(run, conn)
101
+
102
+
103
+ def _refuse_ended(run: SplitRun, conn: psycopg.Connection[TupleRow]) -> None:
104
+ stopped = conn.execute("SELECT stopped_vt FROM harness.run_stop").fetchone()
105
+ if run.ended or stopped is not None:
106
+ raise RunDirError(
107
+ f"run {run.run_id} has ended, and its services cannot serve it again: start a new run"
108
+ )
109
+
110
+
111
+ def clock_origin(conn: psycopg.Connection[TupleRow]) -> int:
112
+ """The run's clock origin; the first process to ask sets it to the host's clock now."""
113
+ conn.execute(
114
+ "INSERT INTO harness.clock_origin (origin_ms) VALUES (%s) ON CONFLICT DO NOTHING",
115
+ (HostClock.host_ms(),),
116
+ )
117
+ row = conn.execute("SELECT origin_ms FROM harness.clock_origin").fetchone()
118
+ if row is None:
119
+ raise RuntimeError("harness.clock_origin is empty after it was set")
120
+ return int(row[0])
121
+
122
+
123
+ async def until_gone(request: Request, stopping: asyncio.Event) -> None:
124
+ """Hold `request` until its caller closes the connection, or the service stops.
125
+
126
+ Read the request's body first: the receive callable gives the body before it reports
127
+ a disconnect.
128
+ """
129
+
130
+ async def gone() -> None:
131
+ while (await request.receive())["type"] != "http.disconnect":
132
+ pass
133
+
134
+ waits = {asyncio.ensure_future(gone()), asyncio.ensure_future(stopping.wait())}
135
+ try:
136
+ await asyncio.wait(waits, return_when=asyncio.FIRST_COMPLETED)
137
+ finally:
138
+ for wait in waits:
139
+ wait.cancel()
140
+
141
+
142
+ def unavailable(detail: str) -> JSONResponse:
143
+ """503: the service no longer takes this request, because the run has moved on."""
144
+ return JSONResponse({"error": "unavailable", "detail": detail}, status_code=503)
145
+
146
+
147
+ def serve(app: Any, *, host: str, port: int) -> None:
148
+ """Serve `app` with uvicorn until the process is told to stop."""
149
+ config = uvicorn.Config(
150
+ app,
151
+ host=host,
152
+ port=port,
153
+ lifespan="on",
154
+ log_level="info",
155
+ timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_S,
156
+ )
157
+ uvicorn.Server(config).run()