fleetproof 0.2.1__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fleetproof-0.2.1/src/fleetproof.egg-info → fleetproof-0.3.1}/PKG-INFO +87 -11
- {fleetproof-0.2.1 → fleetproof-0.3.1}/README.md +86 -10
- {fleetproof-0.2.1 → fleetproof-0.3.1}/pyproject.toml +1 -1
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/__init__.py +1 -1
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/cli.py +222 -3
- fleetproof-0.3.1/src/fleetproof/config.py +131 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/hookgate.py +135 -17
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/ledger.py +264 -9
- fleetproof-0.3.1/src/fleetproof/telemetry.py +783 -0
- fleetproof-0.3.1/src/fleetproof/telemetry_export.py +690 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1/src/fleetproof.egg-info}/PKG-INFO +87 -11
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof.egg-info/SOURCES.txt +6 -1
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_cli.py +73 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_hookgate.py +249 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_ledger.py +129 -0
- fleetproof-0.3.1/tests/test_telemetry.py +585 -0
- fleetproof-0.3.1/tests/test_telemetry_export.py +384 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/LICENSE +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/setup.cfg +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/__main__.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/checker.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/checks.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/report.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof/runlog.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof.egg-info/dependency_links.txt +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof.egg-info/entry_points.txt +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof.egg-info/requires.txt +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/src/fleetproof.egg-info/top_level.txt +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_adversarial_regressions.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_checker.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_checks.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_report.py +0 -0
- {fleetproof-0.2.1 → fleetproof-0.3.1}/tests/test_runlog.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fleetproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Independent, out-of-band verification for agent fleets.
|
|
5
5
|
Author: Gathrazio
|
|
6
6
|
License: MIT
|
|
@@ -74,6 +74,7 @@ No language model sits in the grading path. Grading is comparison.
|
|
|
74
74
|
| SubagentStop hook | The per-subagent gate: records what the agent claimed, grades it at that agent's tier, blocks a false "done". |
|
|
75
75
|
| `fleetproof fleet` | The dispatch board: every dispatch, its state, its tier, and whether anything graded it. |
|
|
76
76
|
| `fleetproof report` | One self-contained HTML file: per run, claimed-done vs. independently-verified. |
|
|
77
|
+
| `fleetproof telemetry` | v0.3: per-dispatch outcome records, a local reliability summary, and a strict-allowlist export. |
|
|
77
78
|
|
|
78
79
|
Runtime dependencies: none (Python standard library only). A tool whose job is
|
|
79
80
|
being trustworthy should add as little dependency and supply-chain surface as it can.
|
|
@@ -160,10 +161,12 @@ SubagentStop is the gate. Each step is its own reason to refuse the stop:
|
|
|
160
161
|
2. Pin drift. Every dispatch pins the SHA-256 of the check spec that was in force
|
|
161
162
|
when the work was ordered. If the spec changed mid-flight, the stop is blocked
|
|
162
163
|
rather than graded against a spec the agent could have edited itself.
|
|
163
|
-
3. That agent's tier of the spec
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
164
|
+
3. That agent's tier of the spec, unioned with any checks declared on the
|
|
165
|
+
dispatch's own manifest (`{"id", "cmd"}` entries — blocking, expect-exit0).
|
|
166
|
+
A blocking failure records `contradicted` and blocks; a pass records
|
|
167
|
+
`verified` and closes the dispatch. If nothing is runnable — the tier selects
|
|
168
|
+
no repo checks and the manifest declares none — no verdict is recorded: an
|
|
169
|
+
absent grade must never read as a passing one.
|
|
167
170
|
|
|
168
171
|
The Stop hook keeps its v0.1 job and adds a ledger sweep. A dispatch that reported
|
|
169
172
|
and then never terminated blocks the bridge from stopping: something started
|
|
@@ -178,6 +181,10 @@ fleetproof dispatch new --prompt "..." # --prompt-file for a real, long o
|
|
|
178
181
|
fleetproof dispatch report <run-id> --report report.json
|
|
179
182
|
fleetproof dispatch close <run-id>
|
|
180
183
|
|
|
184
|
+
fleetproof dispatch intent --agent recon --prompt-file p.md \
|
|
185
|
+
[--manifest m.json] [--tier lane] # declare the NEXT spawn of an agent
|
|
186
|
+
# type; the start capture consumes it
|
|
187
|
+
|
|
181
188
|
fleetproof fleet # the board, newest first
|
|
182
189
|
fleetproof fleet --open # only what has not terminated
|
|
183
190
|
fleetproof fleet --format json # full records, for an agent to read
|
|
@@ -193,12 +200,16 @@ transition trail, and which process wrote each transition.
|
|
|
193
200
|
|
|
194
201
|
### What the ledger does not do yet
|
|
195
202
|
|
|
196
|
-
- The spawn prompt is in neither subagent hook payload
|
|
197
|
-
a placeholder saying so, rather than inventing a prompt it never
|
|
198
|
-
real prompt
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
203
|
+
- The spawn prompt is in neither subagent hook payload, so an undeclared spawn
|
|
204
|
+
still records a placeholder saying so, rather than inventing a prompt it never
|
|
205
|
+
saw. To get the real prompt (and manifest, and tier) onto a captured dispatch,
|
|
206
|
+
write an intent sidecar first: `fleetproof dispatch intent --agent <type>
|
|
207
|
+
--prompt-file <file>` writes `.fleetproof/intents/<agent_type>.json`, and the
|
|
208
|
+
next SubagentStart of that agent type consumes it — one intent, one spawn.
|
|
209
|
+
- Captured subagents without a declared intent tier are recorded as `lane` tier.
|
|
210
|
+
Real nesting depth is not visible from the payload — there is no parent-agent
|
|
211
|
+
field — so declaring one beats inferring it wrongly. What that costs in
|
|
212
|
+
practice is a pilot question.
|
|
202
213
|
- An agent killed mid-turn can reach SubagentStop with an empty final message,
|
|
203
214
|
which report-before-idle blocks. Blocking the stop of an agent that is already
|
|
204
215
|
gone is not useful; how often it happens is under observation.
|
|
@@ -207,6 +218,71 @@ transition trail, and which process wrote each transition.
|
|
|
207
218
|
is also why the lookup is scoped to non-terminal dispatches: an agent id can be
|
|
208
219
|
reused once a dispatch is closed out.
|
|
209
220
|
|
|
221
|
+
## Verification telemetry (v0.3)
|
|
222
|
+
|
|
223
|
+
The ledger records what happened to each dispatch. v0.3 turns those records into
|
|
224
|
+
reliability data an operator can actually read — and, when they choose to, share.
|
|
225
|
+
|
|
226
|
+
Enable it per repo with `.fleetproof/config.json`:
|
|
227
|
+
|
|
228
|
+
```json
|
|
229
|
+
{ "telemetry_era": "2026-08-21" }
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
From that date, every dispatch gains a `telemetry.json`, written on the checker/hook
|
|
233
|
+
side at verdict or close time — never by the agent being measured. Runs from before
|
|
234
|
+
the date read back as pre-telemetry and are never back-filled or guessed.
|
|
235
|
+
|
|
236
|
+
Each finished dispatch derives one of nine **outcome classes** from its transition
|
|
237
|
+
history — bookkeeping, not judgment:
|
|
238
|
+
|
|
239
|
+
- `verified` — the claim survived the checks.
|
|
240
|
+
- `near_miss` — a gate-blocked retry whose work product *changed* before passing:
|
|
241
|
+
a real failure, caught before acceptance.
|
|
242
|
+
- `verifier_flake` — the retry passed with the work product *unchanged*: the
|
|
243
|
+
contradiction was wrong, not the work. Counted separately so flaky checks can't
|
|
244
|
+
inflate the near-miss number.
|
|
245
|
+
- `contradicted` — the claim did not survive.
|
|
246
|
+
- `ungraded` — checks existed but no verdict ever landed. This is a control
|
|
247
|
+
failure and every summary says so; it is never folded into a benign class.
|
|
248
|
+
- `unverifiable` — reported, but nothing in the claim was checkable. Never counts
|
|
249
|
+
as success.
|
|
250
|
+
- `silent_idle` / `terminated_unreported` / `terminated_unclassified` — died
|
|
251
|
+
without reporting, split by the recorded terminate reason.
|
|
252
|
+
|
|
253
|
+
The commands:
|
|
254
|
+
|
|
255
|
+
```
|
|
256
|
+
fleetproof telemetry summary # local-only reliability read
|
|
257
|
+
fleetproof telemetry loss <run-id> # one question per failure: hours or dollars
|
|
258
|
+
fleetproof telemetry export --recipient X # allowlist extract for sharing
|
|
259
|
+
fleetproof telemetry anchor # record the current chain head
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
`summary` prints outcome and severity distributions with their numerators and
|
|
263
|
+
denominators spelled out, override rates, and the corpus's own integrity rates
|
|
264
|
+
(stop-only captures, missing telemetry files) — a dataset that can't measure its
|
|
265
|
+
own holes isn't worth reading.
|
|
266
|
+
|
|
267
|
+
`export` is built the opposite way from most exports: a strict per-field
|
|
268
|
+
**allowlist** — closed enums, counts, bands, versions, and salted identifiers.
|
|
269
|
+
Free text, prompts, paths, and every name you chose yourself never leave the
|
|
270
|
+
machine; timestamps coarsen to dates. Each export carries a completeness manifest
|
|
271
|
+
(runs per day per class, gaps visible) and a methodology note that says plainly
|
|
272
|
+
what `verified` means here: conformance to *your own declared checks* at a
|
|
273
|
+
measured coverage — not a judgment of work quality.
|
|
274
|
+
|
|
275
|
+
The honest trust model, stated rather than implied: local records are
|
|
276
|
+
operator-attested. The checker writes verdicts from a separate process, and v0.3
|
|
277
|
+
chains a rolling hash over the records as they're written (`anchor` gives you a
|
|
278
|
+
head you can sign or store elsewhere) — but a local ledger on a shared filesystem
|
|
279
|
+
is not tamper-proof against everything that can write to it, and FleetProof will
|
|
280
|
+
not pretend otherwise. The dispatch-intent sidecar (0.3.1) sits in the same
|
|
281
|
+
trust class as `checks.json` itself: operator-attested, exactly as forgeable as
|
|
282
|
+
the rest of `.fleetproof/`, no worse — anything that can write the repo can
|
|
283
|
+
write an intent, and the manifest checks it carries run as shell commands at
|
|
284
|
+
the stop gate the same way repo checks do.
|
|
285
|
+
|
|
210
286
|
## Install (each line is one command in Claude Code)
|
|
211
287
|
|
|
212
288
|
```
|
|
@@ -53,6 +53,7 @@ No language model sits in the grading path. Grading is comparison.
|
|
|
53
53
|
| SubagentStop hook | The per-subagent gate: records what the agent claimed, grades it at that agent's tier, blocks a false "done". |
|
|
54
54
|
| `fleetproof fleet` | The dispatch board: every dispatch, its state, its tier, and whether anything graded it. |
|
|
55
55
|
| `fleetproof report` | One self-contained HTML file: per run, claimed-done vs. independently-verified. |
|
|
56
|
+
| `fleetproof telemetry` | v0.3: per-dispatch outcome records, a local reliability summary, and a strict-allowlist export. |
|
|
56
57
|
|
|
57
58
|
Runtime dependencies: none (Python standard library only). A tool whose job is
|
|
58
59
|
being trustworthy should add as little dependency and supply-chain surface as it can.
|
|
@@ -139,10 +140,12 @@ SubagentStop is the gate. Each step is its own reason to refuse the stop:
|
|
|
139
140
|
2. Pin drift. Every dispatch pins the SHA-256 of the check spec that was in force
|
|
140
141
|
when the work was ordered. If the spec changed mid-flight, the stop is blocked
|
|
141
142
|
rather than graded against a spec the agent could have edited itself.
|
|
142
|
-
3. That agent's tier of the spec
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
143
|
+
3. That agent's tier of the spec, unioned with any checks declared on the
|
|
144
|
+
dispatch's own manifest (`{"id", "cmd"}` entries — blocking, expect-exit0).
|
|
145
|
+
A blocking failure records `contradicted` and blocks; a pass records
|
|
146
|
+
`verified` and closes the dispatch. If nothing is runnable — the tier selects
|
|
147
|
+
no repo checks and the manifest declares none — no verdict is recorded: an
|
|
148
|
+
absent grade must never read as a passing one.
|
|
146
149
|
|
|
147
150
|
The Stop hook keeps its v0.1 job and adds a ledger sweep. A dispatch that reported
|
|
148
151
|
and then never terminated blocks the bridge from stopping: something started
|
|
@@ -157,6 +160,10 @@ fleetproof dispatch new --prompt "..." # --prompt-file for a real, long o
|
|
|
157
160
|
fleetproof dispatch report <run-id> --report report.json
|
|
158
161
|
fleetproof dispatch close <run-id>
|
|
159
162
|
|
|
163
|
+
fleetproof dispatch intent --agent recon --prompt-file p.md \
|
|
164
|
+
[--manifest m.json] [--tier lane] # declare the NEXT spawn of an agent
|
|
165
|
+
# type; the start capture consumes it
|
|
166
|
+
|
|
160
167
|
fleetproof fleet # the board, newest first
|
|
161
168
|
fleetproof fleet --open # only what has not terminated
|
|
162
169
|
fleetproof fleet --format json # full records, for an agent to read
|
|
@@ -172,12 +179,16 @@ transition trail, and which process wrote each transition.
|
|
|
172
179
|
|
|
173
180
|
### What the ledger does not do yet
|
|
174
181
|
|
|
175
|
-
- The spawn prompt is in neither subagent hook payload
|
|
176
|
-
a placeholder saying so, rather than inventing a prompt it never
|
|
177
|
-
real prompt
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
182
|
+
- The spawn prompt is in neither subagent hook payload, so an undeclared spawn
|
|
183
|
+
still records a placeholder saying so, rather than inventing a prompt it never
|
|
184
|
+
saw. To get the real prompt (and manifest, and tier) onto a captured dispatch,
|
|
185
|
+
write an intent sidecar first: `fleetproof dispatch intent --agent <type>
|
|
186
|
+
--prompt-file <file>` writes `.fleetproof/intents/<agent_type>.json`, and the
|
|
187
|
+
next SubagentStart of that agent type consumes it — one intent, one spawn.
|
|
188
|
+
- Captured subagents without a declared intent tier are recorded as `lane` tier.
|
|
189
|
+
Real nesting depth is not visible from the payload — there is no parent-agent
|
|
190
|
+
field — so declaring one beats inferring it wrongly. What that costs in
|
|
191
|
+
practice is a pilot question.
|
|
181
192
|
- An agent killed mid-turn can reach SubagentStop with an empty final message,
|
|
182
193
|
which report-before-idle blocks. Blocking the stop of an agent that is already
|
|
183
194
|
gone is not useful; how often it happens is under observation.
|
|
@@ -186,6 +197,71 @@ transition trail, and which process wrote each transition.
|
|
|
186
197
|
is also why the lookup is scoped to non-terminal dispatches: an agent id can be
|
|
187
198
|
reused once a dispatch is closed out.
|
|
188
199
|
|
|
200
|
+
## Verification telemetry (v0.3)
|
|
201
|
+
|
|
202
|
+
The ledger records what happened to each dispatch. v0.3 turns those records into
|
|
203
|
+
reliability data an operator can actually read — and, when they choose to, share.
|
|
204
|
+
|
|
205
|
+
Enable it per repo with `.fleetproof/config.json`:
|
|
206
|
+
|
|
207
|
+
```json
|
|
208
|
+
{ "telemetry_era": "2026-08-21" }
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
From that date, every dispatch gains a `telemetry.json`, written on the checker/hook
|
|
212
|
+
side at verdict or close time — never by the agent being measured. Runs from before
|
|
213
|
+
the date read back as pre-telemetry and are never back-filled or guessed.
|
|
214
|
+
|
|
215
|
+
Each finished dispatch derives one of nine **outcome classes** from its transition
|
|
216
|
+
history — bookkeeping, not judgment:
|
|
217
|
+
|
|
218
|
+
- `verified` — the claim survived the checks.
|
|
219
|
+
- `near_miss` — a gate-blocked retry whose work product *changed* before passing:
|
|
220
|
+
a real failure, caught before acceptance.
|
|
221
|
+
- `verifier_flake` — the retry passed with the work product *unchanged*: the
|
|
222
|
+
contradiction was wrong, not the work. Counted separately so flaky checks can't
|
|
223
|
+
inflate the near-miss number.
|
|
224
|
+
- `contradicted` — the claim did not survive.
|
|
225
|
+
- `ungraded` — checks existed but no verdict ever landed. This is a control
|
|
226
|
+
failure and every summary says so; it is never folded into a benign class.
|
|
227
|
+
- `unverifiable` — reported, but nothing in the claim was checkable. Never counts
|
|
228
|
+
as success.
|
|
229
|
+
- `silent_idle` / `terminated_unreported` / `terminated_unclassified` — died
|
|
230
|
+
without reporting, split by the recorded terminate reason.
|
|
231
|
+
|
|
232
|
+
The commands:
|
|
233
|
+
|
|
234
|
+
```
|
|
235
|
+
fleetproof telemetry summary # local-only reliability read
|
|
236
|
+
fleetproof telemetry loss <run-id> # one question per failure: hours or dollars
|
|
237
|
+
fleetproof telemetry export --recipient X # allowlist extract for sharing
|
|
238
|
+
fleetproof telemetry anchor # record the current chain head
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
`summary` prints outcome and severity distributions with their numerators and
|
|
242
|
+
denominators spelled out, override rates, and the corpus's own integrity rates
|
|
243
|
+
(stop-only captures, missing telemetry files) — a dataset that can't measure its
|
|
244
|
+
own holes isn't worth reading.
|
|
245
|
+
|
|
246
|
+
`export` is built the opposite way from most exports: a strict per-field
|
|
247
|
+
**allowlist** — closed enums, counts, bands, versions, and salted identifiers.
|
|
248
|
+
Free text, prompts, paths, and every name you chose yourself never leave the
|
|
249
|
+
machine; timestamps coarsen to dates. Each export carries a completeness manifest
|
|
250
|
+
(runs per day per class, gaps visible) and a methodology note that says plainly
|
|
251
|
+
what `verified` means here: conformance to *your own declared checks* at a
|
|
252
|
+
measured coverage — not a judgment of work quality.
|
|
253
|
+
|
|
254
|
+
The honest trust model, stated rather than implied: local records are
|
|
255
|
+
operator-attested. The checker writes verdicts from a separate process, and v0.3
|
|
256
|
+
chains a rolling hash over the records as they're written (`anchor` gives you a
|
|
257
|
+
head you can sign or store elsewhere) — but a local ledger on a shared filesystem
|
|
258
|
+
is not tamper-proof against everything that can write to it, and FleetProof will
|
|
259
|
+
not pretend otherwise. The dispatch-intent sidecar (0.3.1) sits in the same
|
|
260
|
+
trust class as `checks.json` itself: operator-attested, exactly as forgeable as
|
|
261
|
+
the rest of `.fleetproof/`, no worse — anything that can write the repo can
|
|
262
|
+
write an intent, and the manifest checks it carries run as shell commands at
|
|
263
|
+
the stop gate the same way repo checks do.
|
|
264
|
+
|
|
189
265
|
## Install (each line is one command in Claude Code)
|
|
190
266
|
|
|
191
267
|
```
|
|
@@ -13,8 +13,11 @@ surface as it can. Subcommands:
|
|
|
13
13
|
record PostToolUse-hook entry: append an evidence record from hook stdin
|
|
14
14
|
subagent-start SubagentStart-hook entry: put a spawning subagent on the ledger
|
|
15
15
|
subagent-stop SubagentStop-hook entry: the per-subagent report-and-verify gate
|
|
16
|
-
dispatch ledger verbs: new / report / close a dispatch
|
|
16
|
+
dispatch ledger verbs: new / report / close a dispatch; intent writes the
|
|
17
|
+
sidecar the SubagentStart capture consumes
|
|
17
18
|
fleet the dispatch board — every dispatch, its state, and whether it reported
|
|
19
|
+
telemetry summary (local-only aggregates) / export (allowlisted bundle) /
|
|
20
|
+
anchor (chain head) / loss (the one raw-loss question per failure)
|
|
18
21
|
|
|
19
22
|
Output is plain ASCII on purpose: these commands get read in cp1252 consoles on
|
|
20
23
|
Windows, where a stray unicode glyph is a UnicodeEncodeError, not a nicer table.
|
|
@@ -27,7 +30,7 @@ import json
|
|
|
27
30
|
import os
|
|
28
31
|
import shutil
|
|
29
32
|
import sys
|
|
30
|
-
from datetime import datetime, timedelta, timezone
|
|
33
|
+
from datetime import date, datetime, timedelta, timezone
|
|
31
34
|
from pathlib import Path
|
|
32
35
|
|
|
33
36
|
# The CLI must not record its own invocations.
|
|
@@ -51,12 +54,15 @@ from .hookgate import (
|
|
|
51
54
|
subagent_stop_main,
|
|
52
55
|
)
|
|
53
56
|
from .ledger import (
|
|
57
|
+
REASON_OPERATOR_CLOSE,
|
|
58
|
+
VALID_TERMINATE_REASONS,
|
|
54
59
|
LedgerError,
|
|
55
60
|
close_dispatch,
|
|
56
61
|
create_dispatch,
|
|
57
62
|
list_dispatches,
|
|
58
63
|
load_dispatch,
|
|
59
64
|
record_report,
|
|
65
|
+
write_intent,
|
|
60
66
|
)
|
|
61
67
|
# The board and the HTML report describe a dispatch with the same words, defined
|
|
62
68
|
# once in report.py. Two surfaces disagreeing about what a state is called is how
|
|
@@ -76,6 +82,7 @@ from .runlog import (
|
|
|
76
82
|
project_root,
|
|
77
83
|
runs_dir,
|
|
78
84
|
)
|
|
85
|
+
from .telemetry import TelemetryError, build_telemetry, record_failure_loss
|
|
79
86
|
|
|
80
87
|
|
|
81
88
|
def _cmd_init(args: argparse.Namespace) -> int:
|
|
@@ -328,6 +335,49 @@ def _cmd_dispatch_new(args: argparse.Namespace) -> int:
|
|
|
328
335
|
return 0
|
|
329
336
|
|
|
330
337
|
|
|
338
|
+
def _cmd_dispatch_intent(args: argparse.Namespace) -> int:
|
|
339
|
+
"""Write the dispatch-intent sidecar the next SubagentStart capture consumes.
|
|
340
|
+
|
|
341
|
+
Exists so dispatchers never hand-write the JSON: the prompt comes from a
|
|
342
|
+
file (same rationale as ``dispatch new --prompt-file`` — shell-quoting a
|
|
343
|
+
real prompt on Windows is how prompts get mangled), the manifest is
|
|
344
|
+
validated here where an error still has someone to land on, and the printed
|
|
345
|
+
path is the audit surface: the file that will vanish when the spawn
|
|
346
|
+
consumes it.
|
|
347
|
+
"""
|
|
348
|
+
try:
|
|
349
|
+
prompt = Path(args.prompt_file).read_text(encoding="utf-8")
|
|
350
|
+
except OSError as e:
|
|
351
|
+
_emit_error("bad_prompt",
|
|
352
|
+
f"Could not read prompt file {args.prompt_file}: {e}", args.format)
|
|
353
|
+
return 2
|
|
354
|
+
|
|
355
|
+
manifest = None
|
|
356
|
+
if args.manifest:
|
|
357
|
+
try:
|
|
358
|
+
manifest = _load_json_file(args.manifest, "manifest")
|
|
359
|
+
except ValueError as e:
|
|
360
|
+
_emit_error("bad_manifest", str(e), args.format)
|
|
361
|
+
return 2
|
|
362
|
+
# Same tolerance as dispatch new: a bare manifest object or a wrapper
|
|
363
|
+
# with a "manifest" key both work.
|
|
364
|
+
inner = manifest.get("manifest")
|
|
365
|
+
if isinstance(inner, dict):
|
|
366
|
+
manifest = inner
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
path = write_intent(args.agent, prompt, manifest=manifest, tier=args.tier)
|
|
370
|
+
except LedgerError as e:
|
|
371
|
+
_emit_error("ledger_error", str(e), args.format)
|
|
372
|
+
return 1
|
|
373
|
+
if args.format == "json":
|
|
374
|
+
print(json.dumps({"ok": True, "agent_type": args.agent,
|
|
375
|
+
"intent_path": str(path)}))
|
|
376
|
+
else:
|
|
377
|
+
print(str(path))
|
|
378
|
+
return 0
|
|
379
|
+
|
|
380
|
+
|
|
331
381
|
def _cmd_dispatch_report(args: argparse.Namespace) -> int:
|
|
332
382
|
try:
|
|
333
383
|
payload = _load_json_file(args.report, "report")
|
|
@@ -348,10 +398,19 @@ def _cmd_dispatch_report(args: argparse.Namespace) -> int:
|
|
|
348
398
|
|
|
349
399
|
def _cmd_dispatch_close(args: argparse.Namespace) -> int:
|
|
350
400
|
try:
|
|
351
|
-
record = close_dispatch(args.run_id)
|
|
401
|
+
record = close_dispatch(args.run_id, reason=args.reason)
|
|
352
402
|
except LedgerError as e:
|
|
353
403
|
_emit_error("ledger_error", str(e), args.format)
|
|
354
404
|
return 1
|
|
405
|
+
# The close finalizes the lifecycle, so the telemetry record derives here.
|
|
406
|
+
# Best-effort: a telemetry failure must not turn a successful close into a
|
|
407
|
+
# failed command — the summary surfaces the missing record as an integrity
|
|
408
|
+
# defect instead.
|
|
409
|
+
try:
|
|
410
|
+
build_telemetry(record.run_id)
|
|
411
|
+
except Exception as e:
|
|
412
|
+
print(f"[fleetproof] telemetry build failed for {record.run_id}: {e}",
|
|
413
|
+
file=sys.stderr)
|
|
355
414
|
if args.format == "json":
|
|
356
415
|
print(json.dumps(record.to_dict(), indent=2))
|
|
357
416
|
else:
|
|
@@ -416,6 +475,105 @@ def _cmd_fleet(args: argparse.Namespace) -> int:
|
|
|
416
475
|
return 0
|
|
417
476
|
|
|
418
477
|
|
|
478
|
+
def _parse_date(value: str | None, label: str) -> "date | None":
|
|
479
|
+
if value is None:
|
|
480
|
+
return None
|
|
481
|
+
try:
|
|
482
|
+
return date.fromisoformat(value)
|
|
483
|
+
except ValueError as e:
|
|
484
|
+
raise ValueError(f"--{label} must be YYYY-MM-DD: {e}") from e
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _cmd_telemetry_summary(args: argparse.Namespace) -> int:
|
|
488
|
+
from .telemetry_export import summarize
|
|
489
|
+
summary = summarize()
|
|
490
|
+
if args.format == "json":
|
|
491
|
+
print(json.dumps(summary, indent=2))
|
|
492
|
+
return 0
|
|
493
|
+
print("Telemetry summary (LOCAL-ONLY: clear-text names and exact times;")
|
|
494
|
+
print("anything shareable goes through `fleetproof telemetry export`).")
|
|
495
|
+
for name, block in summary["windows"].items():
|
|
496
|
+
counts = block["counts"]
|
|
497
|
+
shown = {k: v for k, v in counts.items() if v}
|
|
498
|
+
print(f"\n[{name}] {block['total_dispatches']} dispatch(es), "
|
|
499
|
+
f"{block['classifiable_dispatches']} classifiable")
|
|
500
|
+
print(f" outcomes: {shown if shown else 'none'}")
|
|
501
|
+
for metric in ("delivery_failure_rate", "false_claim_rate",
|
|
502
|
+
"near_miss_rate", "verifier_flake_rate",
|
|
503
|
+
"ungraded_rate", "unverifiable_rate",
|
|
504
|
+
"telemetry_missing_rate", "stop_only_fraction"):
|
|
505
|
+
m = block[metric]
|
|
506
|
+
rate = f"{m['rate']:.3f}" if m["rate"] is not None else "n/a"
|
|
507
|
+
print(f" {metric}: {m['numerator']}/{m['denominator']} = {rate}")
|
|
508
|
+
sev = block["severity_distribution"]
|
|
509
|
+
print(f" severity: {({k: v for k, v in sev.items() if v}) or 'no failures'}")
|
|
510
|
+
override = block["override_rate_by_source"]
|
|
511
|
+
for source in ("hook", "operator"):
|
|
512
|
+
m = override[source]
|
|
513
|
+
rate = f"{m['rate']:.3f}" if m["rate"] is not None else "n/a"
|
|
514
|
+
print(f" override_rate[{source}]: {m['numerator']}/{m['denominator']} = {rate}")
|
|
515
|
+
timeline = summary["spec_hash_timeline"]
|
|
516
|
+
if timeline:
|
|
517
|
+
print("\nspec-hash timeline:")
|
|
518
|
+
for entry in timeline:
|
|
519
|
+
print(f" {entry['first_seen']} {short_spec_hash(entry['spec_sha256'])}")
|
|
520
|
+
return 0
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _cmd_telemetry_export(args: argparse.Namespace) -> int:
|
|
524
|
+
from .telemetry_export import export_telemetry
|
|
525
|
+
try:
|
|
526
|
+
since = _parse_date(args.since, "since")
|
|
527
|
+
until = _parse_date(args.until, "until")
|
|
528
|
+
except ValueError as e:
|
|
529
|
+
_emit_error("bad_date", str(e), args.format)
|
|
530
|
+
return 2
|
|
531
|
+
try:
|
|
532
|
+
out = export_telemetry(
|
|
533
|
+
args.recipient,
|
|
534
|
+
out_dir=Path(args.output) if args.output else None,
|
|
535
|
+
since=since, until=until)
|
|
536
|
+
except TelemetryError as e:
|
|
537
|
+
_emit_error("telemetry_error", str(e), args.format)
|
|
538
|
+
return 1
|
|
539
|
+
if args.format == "json":
|
|
540
|
+
print(json.dumps({"ok": True, "export_dir": str(out)}))
|
|
541
|
+
else:
|
|
542
|
+
print(f"Wrote export bundle to {out}")
|
|
543
|
+
print("Label: identified, minimized (single-party exports are "
|
|
544
|
+
"identified by construction).")
|
|
545
|
+
return 0
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _cmd_telemetry_anchor(args: argparse.Namespace) -> int:
|
|
549
|
+
from .telemetry_export import anchor_chain
|
|
550
|
+
anchor = anchor_chain()
|
|
551
|
+
if args.format == "json":
|
|
552
|
+
print(json.dumps(anchor, indent=2))
|
|
553
|
+
else:
|
|
554
|
+
print(f"chain head: {anchor['chain_head']}")
|
|
555
|
+
print(f"entries: {anchor['chain_entries']}")
|
|
556
|
+
print(anchor["note"])
|
|
557
|
+
return 0
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _cmd_telemetry_loss(args: argparse.Namespace) -> int:
|
|
561
|
+
try:
|
|
562
|
+
telemetry = record_failure_loss(
|
|
563
|
+
args.run_id, args.value, args.unit,
|
|
564
|
+
default_accepted=args.default_accepted or None)
|
|
565
|
+
except TelemetryError as e:
|
|
566
|
+
_emit_error("telemetry_error", str(e), args.format)
|
|
567
|
+
return 1
|
|
568
|
+
if args.format == "json":
|
|
569
|
+
print(json.dumps(telemetry, indent=2))
|
|
570
|
+
else:
|
|
571
|
+
print(f"{args.run_id}: severity {telemetry['failure.severity_band']} "
|
|
572
|
+
f"(floor {telemetry['failure.severity_floor']}), "
|
|
573
|
+
f"loss {telemetry['failure.estimated_loss']}")
|
|
574
|
+
return 0
|
|
575
|
+
|
|
576
|
+
|
|
419
577
|
def _emit_error(code: str, message: str, fmt: str) -> None:
|
|
420
578
|
if fmt == "json":
|
|
421
579
|
print(json.dumps({"ok": False, "error_code": code, "message": message}), file=sys.stderr)
|
|
@@ -500,6 +658,22 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
500
658
|
_add_format(p_dnew)
|
|
501
659
|
p_dnew.set_defaults(func=_cmd_dispatch_new)
|
|
502
660
|
|
|
661
|
+
p_dint = dsub.add_parser(
|
|
662
|
+
"intent",
|
|
663
|
+
help="Write the dispatch-intent sidecar for the next spawn of one agent type.")
|
|
664
|
+
p_dint.add_argument("--agent", required=True,
|
|
665
|
+
help="The agent_type the harness will report for the spawn.")
|
|
666
|
+
p_dint.add_argument("--prompt-file", required=True,
|
|
667
|
+
help="File holding the dispatch prompt, verbatim.")
|
|
668
|
+
p_dint.add_argument("--manifest", default=None,
|
|
669
|
+
help="JSON file with the dispatch manifest "
|
|
670
|
+
"(bare object or a {'manifest': ...} wrapper).")
|
|
671
|
+
p_dint.add_argument("--tier", choices=sorted(VALID_TIERS), default=None,
|
|
672
|
+
help="Declare the tier. Omit to record the captured-subagent "
|
|
673
|
+
"default (lane).")
|
|
674
|
+
_add_format(p_dint)
|
|
675
|
+
p_dint.set_defaults(func=_cmd_dispatch_intent)
|
|
676
|
+
|
|
503
677
|
p_drep = dsub.add_parser("report", help="Record what a dispatched agent claimed.")
|
|
504
678
|
p_drep.add_argument("run_id")
|
|
505
679
|
p_drep.add_argument("--report", required=True, help="JSON file holding the report.")
|
|
@@ -508,9 +682,54 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
508
682
|
|
|
509
683
|
p_dclose = dsub.add_parser("close", help="Terminate a dispatch.")
|
|
510
684
|
p_dclose.add_argument("run_id")
|
|
685
|
+
# Defaulted, not required: the CLI is the operator's close path, so the
|
|
686
|
+
# honest machine-set reason for it is operator-close. Scripted sweepers and
|
|
687
|
+
# session-teardown callers override it to say what they actually are.
|
|
688
|
+
p_dclose.add_argument(
|
|
689
|
+
"--reason", choices=sorted(VALID_TERMINATE_REASONS),
|
|
690
|
+
default=REASON_OPERATOR_CLOSE,
|
|
691
|
+
help="Why this dispatch is being terminated (default: operator-close).")
|
|
511
692
|
_add_format(p_dclose)
|
|
512
693
|
p_dclose.set_defaults(func=_cmd_dispatch_close)
|
|
513
694
|
|
|
695
|
+
p_tel = sub.add_parser("telemetry", help="Verification-telemetry surfaces.")
|
|
696
|
+
tsub = p_tel.add_subparsers(dest="telemetry_command", required=True)
|
|
697
|
+
|
|
698
|
+
p_tsum = tsub.add_parser(
|
|
699
|
+
"summary",
|
|
700
|
+
help="Local-only outcome/severity/integrity summary with trailing windows.")
|
|
701
|
+
_add_format(p_tsum)
|
|
702
|
+
p_tsum.set_defaults(func=_cmd_telemetry_summary)
|
|
703
|
+
|
|
704
|
+
p_texp = tsub.add_parser(
|
|
705
|
+
"export",
|
|
706
|
+
help="Write the allowlisted, date-bucketed export bundle for one recipient.")
|
|
707
|
+
p_texp.add_argument("--recipient", required=True,
|
|
708
|
+
help="Counterparty label; selects (or mints) the export salt.")
|
|
709
|
+
p_texp.add_argument("-o", "--output", default=None, help="Output directory.")
|
|
710
|
+
p_texp.add_argument("--since", default=None, help="Window start, YYYY-MM-DD.")
|
|
711
|
+
p_texp.add_argument("--until", default=None, help="Window end, YYYY-MM-DD.")
|
|
712
|
+
_add_format(p_texp)
|
|
713
|
+
p_texp.set_defaults(func=_cmd_telemetry_export)
|
|
714
|
+
|
|
715
|
+
p_tanc = tsub.add_parser(
|
|
716
|
+
"anchor",
|
|
717
|
+
help="Record the telemetry chain head as an anchorable value.")
|
|
718
|
+
_add_format(p_tanc)
|
|
719
|
+
p_tanc.set_defaults(func=_cmd_telemetry_anchor)
|
|
720
|
+
|
|
721
|
+
p_tloss = tsub.add_parser(
|
|
722
|
+
"loss",
|
|
723
|
+
help="Record the one raw loss quantity for a failed run (hours or usd).")
|
|
724
|
+
p_tloss.add_argument("run_id")
|
|
725
|
+
p_tloss.add_argument("--value", type=float, required=True,
|
|
726
|
+
help="The raw quantity; the band is derived, never chosen.")
|
|
727
|
+
p_tloss.add_argument("--unit", choices=["hours", "usd"], required=True)
|
|
728
|
+
p_tloss.add_argument("--default-accepted", action="store_true",
|
|
729
|
+
help="The suggested default was accepted unchanged.")
|
|
730
|
+
_add_format(p_tloss)
|
|
731
|
+
p_tloss.set_defaults(func=_cmd_telemetry_loss)
|
|
732
|
+
|
|
514
733
|
p_fleet = sub.add_parser("fleet", help="The dispatch board, newest first.")
|
|
515
734
|
p_fleet.add_argument("--open", action="store_true",
|
|
516
735
|
help="Only dispatches that have not terminated.")
|