shapeup-sdlc 1.7.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.example.json +5 -5
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +17 -3
- package/README.md +124 -103
- package/SECURITY.md +37 -30
- package/bin/init.mjs +77 -47
- package/bin/lib/grant.mjs +145 -0
- package/commands/build.md +7 -6
- package/commands/ship.md +29 -16
- package/commands/wire.md +1 -1
- package/hooks/dispatch-receipt.mjs +195 -0
- package/hooks/gate-intake.mjs +3 -3
- package/hooks/gate-zerowork.mjs +25 -24
- package/hooks/hooks.json +9 -48
- package/hooks/lib/decision.mjs +19 -8
- package/hooks/safety-spine.mjs +3 -3
- package/hooks/sandbox-guard.mjs +84 -40
- package/{skills/tech-lead/scripts/compile-order.mjs → kernel/compile.mjs} +283 -41
- package/{skills/tech-lead/scripts/gate-answers.mjs → kernel/gate.mjs} +58 -14
- package/kernel/harness.mjs +134 -0
- package/{skills/tech-lead/scripts/fit-check.mjs → kernel/init/fit.mjs} +15 -12
- package/{skills/tech-lead/scripts/init-run.mjs → kernel/init/run.mjs} +152 -31
- package/{skills/tech-lead/scripts → kernel}/lib/argv.mjs +52 -31
- package/{skills/tech-lead/scripts/lib/contract-md.mjs → kernel/lib/contract.mjs} +251 -7
- package/kernel/lib/paths.mjs +491 -0
- package/kernel/probe/concurrency.mjs +510 -0
- package/{skills/tech-lead/scripts/aegis-digest.mjs → kernel/probe/digest.mjs} +9 -10
- package/kernel/probe/eval.mjs +77 -0
- package/kernel/probe/leg.mjs +125 -0
- package/{skills/tech-lead/scripts/resume-state.mjs → kernel/probe/resume.mjs} +204 -63
- package/{skills/tech-lead/scripts → kernel/probe}/stats.mjs +14 -9
- package/kernel/probe/t0.mjs +66 -0
- package/{skills/ba-pitch-analyzer/scripts/board-derive.mjs → kernel/reduce/board.mjs} +99 -23
- package/kernel/reduce/graph.mjs +437 -0
- package/{skills/tech-lead/scripts/hill-derive.mjs → kernel/reduce/hill.mjs} +28 -9
- package/kernel/reduce/ingest.mjs +633 -0
- package/{hooks/slop-cleaner.mjs → kernel/reduce/leftovers.mjs} +40 -57
- package/{skills/tech-lead/scripts/ship-report.mjs → kernel/reduce/ship.mjs} +61 -13
- package/{skills/tech-lead/scripts/run-snapshot.mjs → kernel/reduce/snapshot.mjs} +21 -15
- package/{skills/spec-evaluator/scripts/verdict-ledger.mjs → kernel/reduce/verdict.mjs} +13 -7
- package/kernel/report/export.mjs +321 -0
- package/kernel/report/facts.mjs +209 -0
- package/{skills/tech-lead/scripts/budget-check.mjs → kernel/verify/budget.mjs} +15 -12
- package/kernel/verify/dispatch.mjs +114 -0
- package/{skills/tech-lead/scripts/validate-envelope.mjs → kernel/verify/envelope.mjs} +20 -15
- package/{skills/tech-lead/scripts/lib → kernel/verify}/ratchet-tree.mjs +36 -14
- package/kernel/verify/skills.mjs +125 -0
- package/kernel/verify/spec.mjs +559 -0
- package/{skills/tech-lead/scripts/t0-verify.mjs → kernel/verify/t0.mjs} +134 -24
- package/{skills/tech-lead/scripts/trace-lint.mjs → kernel/verify/trace.mjs} +26 -11
- package/oracles/http-oracle.mjs +1 -1
- package/oracles/process-oracle.mjs +1 -1
- package/oracles/snapshot-oracle.mjs +1 -1
- package/oracles/test-oracle.mjs +1 -1
- package/package.json +11 -9
- package/skills/ba-pitch-analyzer/SKILL.md +5 -5
- package/skills/ba-pitch-analyzer/assets/templates/_index.tmpl.md +1 -1
- package/skills/ba-pitch-analyzer/assets/templates/api-feasibility.tmpl.md +12 -7
- package/skills/ba-pitch-analyzer/assets/templates/contracts/third-party-api.contract.tmpl.md +6 -3
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/migration-plan.tmpl.md +5 -4
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/team-handoff.tmpl.md +23 -13
- package/skills/ba-pitch-analyzer/assets/templates/scope-summary.tmpl.md +17 -12
- package/skills/ba-pitch-analyzer/assets/templates/synthesis.tmpl.md +17 -38
- package/skills/ba-pitch-analyzer/assets/templates/task.tmpl.md +3 -0
- package/skills/ba-pitch-analyzer/references/contract-patterns.md +8 -4
- package/skills/ba-pitch-analyzer/references/doc-schemas.md +2 -0
- package/skills/ba-pitch-analyzer/references/task-generation.md +2 -2
- package/skills/qa-edge-hunter/SKILL.md +7 -3
- package/skills/scope-architect/SKILL.md +40 -6
- package/skills/solution-architect/SKILL.md +19 -4
- package/skills/spec-evaluator/SKILL.md +12 -2
- package/skills/task-executor/SKILL.md +7 -6
- package/skills/tech-lead/SKILL.md +50 -38
- package/skills/tech-lead/references/gates.md +71 -20
- package/skills/tech-lead/references/protocol.md +832 -0
- package/skills/tech-lead/schemas/domain.schema.json +341 -66
- package/skills/tech-lead/schemas/work-order.schema.json +11 -2
- package/skills/tech-lead/schemas/work-result.schema.json +1 -1
- package/skills/tech-lead/workflows/shapeup-run.js +1327 -750
- package/hooks/anti-rationalization.mjs +0 -238
- package/hooks/compact-snapshot.mjs +0 -47
- package/hooks/gate-deadline.mjs +0 -147
- package/hooks/gate-l2.mjs +0 -161
- package/hooks/session-rehydrate.mjs +0 -108
- package/skills/ba-pitch-analyzer/scripts/spec-lint.mjs +0 -239
- package/skills/spec-evaluator/README.md +0 -93
- package/skills/tech-lead/README.md +0 -71
- package/skills/tech-lead/references/delegation.md +0 -261
- package/skills/tech-lead/references/hard-rules.md +0 -34
- package/skills/tech-lead/references/invocation.md +0 -45
- package/skills/tech-lead/references/ledger-schema.md +0 -213
- package/skills/tech-lead/references/round-protocol.md +0 -234
- package/skills/tech-lead/references/state-model.md +0 -66
- package/skills/tech-lead/scripts/ingest-result.mjs +0 -258
- package/skills/tech-lead/scripts/lib/is-main.mjs +0 -81
- package/skills/tech-lead/scripts/lib/paths.mjs +0 -280
- package/skills/tech-lead/scripts/run-workflow.mjs +0 -381
- package/skills/translator/README.md +0 -66
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// probe concurrency — "how many legs actually ran at once, and what did that buy?"
|
|
3
|
+
//
|
|
4
|
+
// WHY IT EXISTS. Fanning the scope loop out has three acceptance questions — did two scopes ever
|
|
5
|
+
// run at the same time, did shared state survive it, did wall-clock improve — and all three are
|
|
6
|
+
// MEASUREMENTS. The pipeline emitted no measurement: the first claim that legs ran in parallel came
|
|
7
|
+
// from a probe that counted green results, which is satisfied by three legs run one after another.
|
|
8
|
+
// This is the missing instrument. It writes nothing and answers only from records on disk.
|
|
9
|
+
//
|
|
10
|
+
// THE RULE IT IS BUILT AROUND. A predicate that an ABSENCE can satisfy must report that absence in
|
|
11
|
+
// the same value. `max_concurrent: 1` over four legs and one usable record is not a small number,
|
|
12
|
+
// it is a false one — and it reads exactly like a run that was genuinely sequential. So every
|
|
13
|
+
// figure here travels with the completeness of the record set it came from and with a `bound`
|
|
14
|
+
// saying whether it is exact or a floor, and a speedup that cannot be supported is `null` with the
|
|
15
|
+
// reason, never a plausible-looking ratio.
|
|
16
|
+
//
|
|
17
|
+
// WHERE THE TWO ENDS OF A LEG COME FROM.
|
|
18
|
+
//
|
|
19
|
+
// start `receipts/dispatch.jsonl` — a PostToolUse row, written when the Skill RESOLVED. Measured
|
|
20
|
+
// across real runs it lands 1.8–47 s after the order was compiled, on legs that then ran
|
|
21
|
+
// for minutes: it is a start, and being hook-written and append-only it is the one start
|
|
22
|
+
// that survives a relaunch re-writing the order file.
|
|
23
|
+
// end `legs.jsonl` — the reducer's leg-completion row. EXACT.
|
|
24
|
+
// Falling back, for a run recorded before that ledger existed: the last T0 trial the leg
|
|
25
|
+
// wrote inside its own dispatch window. That is a landmark near the end, not the end, so
|
|
26
|
+
// every figure derived through it is marked a LOWER bound.
|
|
27
|
+
//
|
|
28
|
+
// WHAT IS DELIBERATELY NOT USED: result-file mtimes. A trace that has been copied carries the copy
|
|
29
|
+
// time on every file, so a measurement built on them is fabricated rather than approximate.
|
|
30
|
+
//
|
|
31
|
+
// Usage: node kernel/harness.mjs probe concurrency --slug <slug> [--cwd <dir>] [--run-root <dir>]
|
|
32
|
+
// [--round N] [--gap-s N] [--format json|table]
|
|
33
|
+
// Exit: 0 = a report was produced with at least one usable leg · 1 = ran, and no leg in scope had
|
|
34
|
+
// a usable interval (the report still prints, and says why) · 2 = malformed argv.
|
|
35
|
+
|
|
36
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
37
|
+
import { join, resolve } from "node:path";
|
|
38
|
+
import { runArgs } from "../lib/argv.mjs";
|
|
39
|
+
import { localRoot, runIdFromRoot, RECEIPT_FILE } from "../lib/paths.mjs";
|
|
40
|
+
|
|
41
|
+
/** The round-addressed order id forms `<scope>-r<N>-a<M>` and `<phase>-r<N>`. */
|
|
42
|
+
const ROUND_SUFFIX = /^(.*?)-r(\d+)(?:-a(\d+))?$/;
|
|
43
|
+
|
|
44
|
+
/** Default fan-out width when a run's `run-args.json` declares none — mirrors the orchestrator. */
|
|
45
|
+
export const DEFAULT_MAX_PARALLEL_SCOPES = 4;
|
|
46
|
+
|
|
47
|
+
/** Default relaunch gap, in seconds. A run resumed after this long is a different launch. */
|
|
48
|
+
export const DEFAULT_GAP_S = 900;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Read a JSONL ledger into rows, skipping torn lines rather than throwing.
|
|
52
|
+
* @param {string} path - Path to a `.jsonl` file.
|
|
53
|
+
* @returns {{rows:Array<object>, malformed:number, present:boolean}} Parsed rows and how many
|
|
54
|
+
* lines could not be parsed — reported, because a ledger half of which is unreadable must not
|
|
55
|
+
* look like a short one.
|
|
56
|
+
*/
|
|
57
|
+
export function readLedger(path) {
|
|
58
|
+
if (!existsSync(path)) return { rows: [], malformed: 0, present: false };
|
|
59
|
+
let malformed = 0;
|
|
60
|
+
const rows = [];
|
|
61
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
62
|
+
if (!line.trim()) continue;
|
|
63
|
+
try { rows.push(JSON.parse(line)); } catch { malformed++; }
|
|
64
|
+
}
|
|
65
|
+
return { rows, malformed, present: true };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Split an order id into the scope/phase name and the round-attempt address it encodes.
|
|
70
|
+
* @param {string} orderId - e.g. `todo-cli/foundation-r2-a1`.
|
|
71
|
+
* @returns {{name:(string|null), round:(number|null), attempt:(number|null)}}
|
|
72
|
+
*/
|
|
73
|
+
export function addressOf(orderId) {
|
|
74
|
+
const suffix = String(orderId ?? "").split("/").slice(1).join("/");
|
|
75
|
+
if (!suffix) return { name: null, round: null, attempt: null };
|
|
76
|
+
const m = suffix.match(ROUND_SUFFIX);
|
|
77
|
+
if (!m) return { name: suffix, round: null, attempt: null };
|
|
78
|
+
return { name: m[1], round: Number(m[2]), attempt: m[3] === undefined ? null : Number(m[3]) };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Every build dispatch's start, from the hook-written receipt ledger.
|
|
83
|
+
*
|
|
84
|
+
* BUILD LEGS ONLY, and the count of what was skipped travels with the answer. ORIENT, ANALYZE,
|
|
85
|
+
* WIRE, EVAL and GATE H are sequential phases in every lane; folding them into a concurrency figure
|
|
86
|
+
* would dilute exactly the number the fan-out is judged on. A leg qualifies by carrying an attempt
|
|
87
|
+
* address (`-r<N>-a<M>`), which only a scope build order does.
|
|
88
|
+
*
|
|
89
|
+
* @param {Array<object>} receipts - Rows from `receipts/dispatch.jsonl`.
|
|
90
|
+
* @returns {{starts:Array<object>, phase_dispatches:number}} Starts sorted by time.
|
|
91
|
+
*/
|
|
92
|
+
export function startsFrom(receipts) {
|
|
93
|
+
const starts = [];
|
|
94
|
+
let phase = 0;
|
|
95
|
+
for (const r of receipts) {
|
|
96
|
+
const { name, round, attempt } = addressOf(r.order_id);
|
|
97
|
+
const at = Date.parse(r.at);
|
|
98
|
+
if (attempt === null || !Number.isFinite(at)) { phase++; continue; }
|
|
99
|
+
starts.push({ order_id: r.order_id, scope_id: name, round, attempt, start: at, start_source: "dispatch-receipt", run_id: r.run_id ?? null });
|
|
100
|
+
}
|
|
101
|
+
starts.sort((a, b) => a.start - b.start || String(a.order_id).localeCompare(String(b.order_id)));
|
|
102
|
+
return { starts, phase_dispatches: phase };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Attach an end to every start.
|
|
107
|
+
*
|
|
108
|
+
* THE PAIRING PROBLEM THIS SOLVES. One order id is dispatched many times — a relaunch re-uses the
|
|
109
|
+
* path verbatim — so "the completion row for this order" is ambiguous and picking the newest would
|
|
110
|
+
* hand every earlier dispatch the last one's end. Each start therefore claims only what falls
|
|
111
|
+
* inside its own window: from its own instant up to the NEXT dispatch OF THE SAME SCOPE AND ROUND.
|
|
112
|
+
*
|
|
113
|
+
* The window is keyed by scope+round rather than by order id, and the difference is not cosmetic.
|
|
114
|
+
* Windowing by order id let one archived leg claim a T0 artifact written by a LATER dispatch of the
|
|
115
|
+
* same scope under a different attempt number — a leg that had actually died came back with a
|
|
116
|
+
* seventeen-minute duration. A leg for scope S in round R is over the moment anything else is
|
|
117
|
+
* dispatched for S in R, whatever attempt it is addressed by.
|
|
118
|
+
*
|
|
119
|
+
* Exact ends come from the leg ledger. When a start's window holds none — every run recorded before
|
|
120
|
+
* that ledger existed, and every leg that died before ingest — the last T0 trial the scope wrote
|
|
121
|
+
* inside the same window stands in, and the leg is marked a lower bound rather than a fact.
|
|
122
|
+
*
|
|
123
|
+
* @param {Array<object>} starts - From {@link startsFrom}.
|
|
124
|
+
* @param {Array<object>} legRows - Rows from `legs.jsonl`.
|
|
125
|
+
* @param {Array<object>} trials - Rows from `t0/trials.jsonl`.
|
|
126
|
+
* @returns {Array<object>} One leg per start, `end` null when nothing closed it.
|
|
127
|
+
*/
|
|
128
|
+
export function pairLegs(starts, legRows, trials) {
|
|
129
|
+
const sameLeg = (a, b) => a.scope_id === b.scope_id && a.round === b.round;
|
|
130
|
+
const nextOf = new Map();
|
|
131
|
+
for (let i = 0; i < starts.length; i++) {
|
|
132
|
+
for (let j = i + 1; j < starts.length; j++) {
|
|
133
|
+
if (sameLeg(starts[j], starts[i])) { nextOf.set(i, starts[j].start); break; }
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return starts.map((s, i) => {
|
|
137
|
+
const until = nextOf.get(i) ?? Infinity;
|
|
138
|
+
const inWindow = (t) => Number.isFinite(t) && t >= s.start && t < until;
|
|
139
|
+
|
|
140
|
+
const landmarks = trials
|
|
141
|
+
.filter((t) => t.scope_id === s.scope_id && Number(t.round) === s.round && inWindow(Date.parse(t.at)))
|
|
142
|
+
.map((t) => Date.parse(t.at)).sort((a, b) => a - b);
|
|
143
|
+
// The INNER end, kept alongside the outer one rather than used only as a fallback. A T0 trial
|
|
144
|
+
// was written BY THE LEG, mid-flight, so `[start, inner_end]` is an interval the leg provably
|
|
145
|
+
// occupied. `ingested_at` is when the WRITER ran, which is normally the leg's last act and is
|
|
146
|
+
// sometimes not the leg at all — an out-of-band ingest (a repair, a `--no-receipt-check` rescue,
|
|
147
|
+
// a relaunch re-applying a result) stamps a later instant onto the same row and stretches the
|
|
148
|
+
// leg over time it was not running. That is not hypothetical: one leg ingested out of band, minutes
|
|
149
|
+
// after its own T0 landmark while its neighbours closed within seconds of theirs, is enough to
|
|
150
|
+
// report two legs overlapping on a run whose dial was 1 and which built one scope at a time.
|
|
151
|
+
const innerEnd = landmarks.length ? landmarks[landmarks.length - 1] : null;
|
|
152
|
+
|
|
153
|
+
const exact = legRows
|
|
154
|
+
.filter((r) => r.order_id === s.order_id && inWindow(Date.parse(r.ingested_at)))
|
|
155
|
+
.map((r) => Date.parse(r.ingested_at)).sort((a, b) => a - b);
|
|
156
|
+
if (exact.length) {
|
|
157
|
+
return { ...s, end: exact[exact.length - 1], end_source: "leg-record", bound: "exact", inner_end: innerEnd };
|
|
158
|
+
}
|
|
159
|
+
if (innerEnd !== null) {
|
|
160
|
+
return { ...s, end: innerEnd, end_source: "t0-landmark", bound: "lower", inner_end: innerEnd };
|
|
161
|
+
}
|
|
162
|
+
return { ...s, end: null, end_source: null, bound: null, inner_end: null };
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Split legs into launches.
|
|
168
|
+
*
|
|
169
|
+
* A run keeps one `run_id` across every relaunch — the key is derived from the receipt, and a
|
|
170
|
+
* resumed run reads the same receipt — so a span taken over all of a run's legs fuses launches that
|
|
171
|
+
* were hours apart. Measured on one archived run: four launches under one key, 17:18 to 03:35, of
|
|
172
|
+
* which the great majority is a laptop being closed.
|
|
173
|
+
*
|
|
174
|
+
* There is no launch id in the record set, so this is a GAP HEURISTIC and it says so: the rule and
|
|
175
|
+
* its threshold are in the report, because a heuristic that hides is indistinguishable from a fact.
|
|
176
|
+
*
|
|
177
|
+
* @param {Array<object>} legs - Legs with a usable interval, sorted by start.
|
|
178
|
+
* @param {number} gapMs - A quiet period longer than this starts a new launch.
|
|
179
|
+
* @returns {Array<Array<object>>} One group per launch.
|
|
180
|
+
*/
|
|
181
|
+
export function segment(legs, gapMs) {
|
|
182
|
+
if (!legs.length) return [];
|
|
183
|
+
const out = [[legs[0]]];
|
|
184
|
+
let high = legs[0].end;
|
|
185
|
+
for (const leg of legs.slice(1)) {
|
|
186
|
+
if (leg.start - high > gapMs) { out.push([leg]); } else { out[out.length - 1].push(leg); }
|
|
187
|
+
high = Math.max(high, leg.end);
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* The greatest number of legs open at one instant, and when it happened.
|
|
194
|
+
*
|
|
195
|
+
* Ends are processed before starts at the same millisecond, so a leg that closes exactly as the
|
|
196
|
+
* next one opens is not counted as an overlap. Two legs are concurrent only if they were both
|
|
197
|
+
* genuinely running, and a tie at the boundary is the case where that is least certain.
|
|
198
|
+
*
|
|
199
|
+
* @param {Array<object>} legs - Legs with `start` and `end`.
|
|
200
|
+
* @param {string} [endField] - Which end to measure against: `end` (the outer end, an ingest) or
|
|
201
|
+
* `inner_end` (the last T0 trial the leg wrote, provably mid-flight). Legs lacking the named field
|
|
202
|
+
* are dropped, so the caller must read the returned `legs` count before trusting `max`.
|
|
203
|
+
* @returns {{max:(number|null), at:(number|null), legs:number}} `null` when there is nothing to
|
|
204
|
+
* measure — never `0` or `1`, which a reader would take for a sequential run.
|
|
205
|
+
*/
|
|
206
|
+
export function maxConcurrent(legs, endField = "end") {
|
|
207
|
+
const usable = legs.filter((l) => Number.isFinite(l[endField]));
|
|
208
|
+
if (!usable.length) return { max: null, at: null, legs: 0 };
|
|
209
|
+
const events = [];
|
|
210
|
+
for (const l of usable) { events.push([l.start, 1]); events.push([l[endField], -1]); }
|
|
211
|
+
events.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
212
|
+
let open = 0, max = 0, at = null;
|
|
213
|
+
for (const [t, delta] of events) {
|
|
214
|
+
open += delta;
|
|
215
|
+
if (open > max) { max = open; at = t; }
|
|
216
|
+
}
|
|
217
|
+
return { max, at, legs: usable.length };
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The waves the run actually ran, derived from the timings rather than from the scheduler.
|
|
222
|
+
*
|
|
223
|
+
* A wave is a connected component of overlapping intervals: legs that ran together, bounded by
|
|
224
|
+
* quiet moments where nothing was in flight. This is an INDEPENDENT reading of the schedule — if
|
|
225
|
+
* the scheduler claims a dependency ordering, this is where it either shows up or does not.
|
|
226
|
+
*
|
|
227
|
+
* @param {Array<object>} legs - Legs with a usable interval, sorted by start.
|
|
228
|
+
* @returns {Array<{from:string, to:string, scopes:string[]}>} Waves in start order.
|
|
229
|
+
*/
|
|
230
|
+
export function wavesObserved(legs) {
|
|
231
|
+
const waves = [];
|
|
232
|
+
for (const leg of legs) {
|
|
233
|
+
const last = waves[waves.length - 1];
|
|
234
|
+
if (last && leg.start < last.endMs) {
|
|
235
|
+
last.members.push(leg);
|
|
236
|
+
last.endMs = Math.max(last.endMs, leg.end);
|
|
237
|
+
} else {
|
|
238
|
+
waves.push({ startMs: leg.start, endMs: leg.end, members: [leg] });
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return waves.map((w) => ({
|
|
242
|
+
from: new Date(w.startMs).toISOString(),
|
|
243
|
+
to: new Date(w.endMs).toISOString(),
|
|
244
|
+
scopes: [...new Set(w.members.map((m) => m.scope_id))].sort(),
|
|
245
|
+
}));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Summarise one launch: overlap, span, work, and whether a speedup can honestly be stated.
|
|
250
|
+
*
|
|
251
|
+
* WHY A SPEEDUP IS REFUSED ON ANY LOWER-BOUND LEG, and it is not caution. Truncating every leg at
|
|
252
|
+
* its T0 landmark shortens the numerator once per leg and the denominator only for whichever leg
|
|
253
|
+
* finished last, so the ratio moves in a direction that depends on the shape of the round. Measured
|
|
254
|
+
* on a real round that demonstrably ran four scopes at once, the truncated figure is 0.90 — a
|
|
255
|
+
* "speedup" below 1. A ratio of two lower bounds is not a bound on the ratio.
|
|
256
|
+
*
|
|
257
|
+
* @param {number} index - 1-based launch number.
|
|
258
|
+
* @param {Array<object>} legs - The launch's legs, each with a usable interval.
|
|
259
|
+
* @returns {object} The launch block of the report.
|
|
260
|
+
*/
|
|
261
|
+
export function summarise(index, legs) {
|
|
262
|
+
const { max, at } = maxConcurrent(legs);
|
|
263
|
+
// The same question asked of the inner intervals — dispatch to the leg's own T0 trial. Two legs
|
|
264
|
+
// overlapping THERE were both provably mid-flight at one instant; two legs overlapping only on the
|
|
265
|
+
// outer intervals may be one leg plus somebody else's later ingest of it.
|
|
266
|
+
const inner = maxConcurrent(legs, "inner_end");
|
|
267
|
+
const exact = legs.every((l) => l.bound === "exact");
|
|
268
|
+
const from = Math.min(...legs.map((l) => l.start));
|
|
269
|
+
const to = Math.max(...legs.map((l) => l.end));
|
|
270
|
+
const span = to - from;
|
|
271
|
+
const sum = legs.reduce((a, l) => a + (l.end - l.start), 0);
|
|
272
|
+
const bound = exact ? "exact" : "lower";
|
|
273
|
+
return {
|
|
274
|
+
launch: index,
|
|
275
|
+
from: new Date(from).toISOString(),
|
|
276
|
+
to: new Date(to).toISOString(),
|
|
277
|
+
legs: legs.length,
|
|
278
|
+
max_concurrent: max,
|
|
279
|
+
max_concurrent_at: at === null ? null : new Date(at).toISOString(),
|
|
280
|
+
max_concurrent_bound: bound,
|
|
281
|
+
// Reported beside the outer figure, never instead of it, and with its own coverage count: an
|
|
282
|
+
// inner figure over 2 of 6 legs is not evidence the other four ran alone.
|
|
283
|
+
max_concurrent_inner: inner.max,
|
|
284
|
+
max_concurrent_inner_legs: inner.legs,
|
|
285
|
+
// The two ends disagreeing is a fact about the RECORD, not about the run, and it is the only
|
|
286
|
+
// signal that separates a leg that ran long from a leg something ingested later.
|
|
287
|
+
ends_disputed: max !== null && inner.max !== null && inner.legs === legs.length && max > inner.max,
|
|
288
|
+
span_ms: span,
|
|
289
|
+
span_bound: bound,
|
|
290
|
+
sum_leg_ms: sum,
|
|
291
|
+
sum_leg_bound: bound,
|
|
292
|
+
speedup: exact && span > 0 ? Math.round((sum / span) * 100) / 100 : null,
|
|
293
|
+
speedup_refused_because: exact
|
|
294
|
+
? (span > 0 ? null : "the launch's legs span zero milliseconds — nothing to divide by")
|
|
295
|
+
: `${legs.filter((l) => l.bound !== "exact").length} of ${legs.length} leg(s) end at a T0 landmark rather than a completion record, so both the span and the work sum are floors; their ratio bounds nothing`,
|
|
296
|
+
waves_observed: wavesObserved(legs),
|
|
297
|
+
scopes: [...new Set(legs.map((l) => l.scope_id))].sort(),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Which fan-out width the run was launched with — read, never assumed.
|
|
303
|
+
*
|
|
304
|
+
* The dial is written into `run-args.json` by the launcher. It is absent from every run recorded so
|
|
305
|
+
* far, so the honest answer is the effective default WITH the fact that it is a default: reporting
|
|
306
|
+
* `4` unqualified would assert an operator choice nobody made.
|
|
307
|
+
*
|
|
308
|
+
* @param {string} runRoot - The run's LOCAL root.
|
|
309
|
+
* @returns {{max_parallel_scopes:number, source:string}} The value and where it came from.
|
|
310
|
+
*/
|
|
311
|
+
export function dialFrom(runRoot) {
|
|
312
|
+
try {
|
|
313
|
+
const a = JSON.parse(readFileSync(join(runRoot, "run-args.json"), "utf8"));
|
|
314
|
+
const n = Number(a?.maxParallelScopes);
|
|
315
|
+
if (Number.isFinite(n) && n >= 1) return { max_parallel_scopes: n, source: "run-args" };
|
|
316
|
+
return { max_parallel_scopes: DEFAULT_MAX_PARALLEL_SCOPES, source: "default (run-args.json declares none)" };
|
|
317
|
+
} catch {
|
|
318
|
+
return { max_parallel_scopes: DEFAULT_MAX_PARALLEL_SCOPES, source: "default (no run-args.json)" };
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The whole report for one run root.
|
|
324
|
+
*
|
|
325
|
+
* @param {string} runRoot - The run's LOCAL root (`<cwd>/.shapeup/<slug>`, or an archived copy).
|
|
326
|
+
* @param {object} [opts] - `{round, gapS}`.
|
|
327
|
+
* @returns {object} A byte-stable report — no clock reading anywhere in it, arrays sorted, key
|
|
328
|
+
* order fixed by this literal — so two runs over one tree produce identical bytes.
|
|
329
|
+
*/
|
|
330
|
+
export function report(runRoot, { round = null, gapS = DEFAULT_GAP_S } = {}) {
|
|
331
|
+
const receipts = readLedger(join(runRoot, "receipts", "dispatch.jsonl"));
|
|
332
|
+
const legLedger = readLedger(join(runRoot, "legs.jsonl"));
|
|
333
|
+
const trials = readLedger(join(runRoot, "t0", "trials.jsonl"));
|
|
334
|
+
|
|
335
|
+
const { starts, phase_dispatches } = startsFrom(receipts.rows);
|
|
336
|
+
const scoped = round === null ? starts : starts.filter((s) => s.round === round);
|
|
337
|
+
const legs = pairLegs(scoped, legLedger.rows, trials.rows);
|
|
338
|
+
const usable = legs.filter((l) => l.end !== null);
|
|
339
|
+
|
|
340
|
+
const warnings = [];
|
|
341
|
+
if (!receipts.present) {
|
|
342
|
+
warnings.push("no receipts/dispatch.jsonl — nothing recorded a dispatch, so no leg has a start. " +
|
|
343
|
+
"An empty report here means the record set is empty, not that the run was sequential.");
|
|
344
|
+
}
|
|
345
|
+
if (!legLedger.present) {
|
|
346
|
+
warnings.push("no legs.jsonl — this run was recorded before leg completions were written, so every " +
|
|
347
|
+
"end falls back to the T0 landmark and every figure below is a floor.");
|
|
348
|
+
}
|
|
349
|
+
for (const l of [["receipts/dispatch.jsonl", receipts], ["legs.jsonl", legLedger], ["t0/trials.jsonl", trials]]) {
|
|
350
|
+
if (l[1].malformed) warnings.push(`${l[0]}: ${l[1].malformed} unparseable line(s) skipped`);
|
|
351
|
+
}
|
|
352
|
+
const unkeyed = legs.filter((l) => !l.run_id).length;
|
|
353
|
+
if (unkeyed) {
|
|
354
|
+
warnings.push(`${unkeyed} of ${legs.length} leg(s) carry no run_id — order_id alone repeats across ` +
|
|
355
|
+
"runs of one slug, so those rows cannot be attributed to this run rather than another.");
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const launches = segment(usable, gapS * 1000).map((g, i) => summarise(i + 1, g));
|
|
359
|
+
|
|
360
|
+
const best = launches.reduce((a, b) => (b.max_concurrent > (a?.max_concurrent ?? -1) ? b : a), null);
|
|
361
|
+
// The launch the headline verdict is answered from, chosen on the INNER figure. Taking the inner
|
|
362
|
+
// number off `best` would compare two different launches: the widest outer overlap and the widest
|
|
363
|
+
// inner one need not be the same launch, and mixing them would let one launch's ingest artefact
|
|
364
|
+
// stand beside another launch's honest evidence.
|
|
365
|
+
const bestInner = launches.reduce((a, b) => (b.max_concurrent_inner > (a?.max_concurrent_inner ?? -1) ? b : a), null);
|
|
366
|
+
return {
|
|
367
|
+
schema_version: 1,
|
|
368
|
+
run_root: runRoot,
|
|
369
|
+
run_id: runIdFromRoot(runRoot),
|
|
370
|
+
receipt_present: existsSync(join(runRoot, RECEIPT_FILE)),
|
|
371
|
+
round,
|
|
372
|
+
dial: dialFrom(runRoot),
|
|
373
|
+
completeness: {
|
|
374
|
+
// The absence, in the same value as the number. `max_concurrent` below is meaningless without
|
|
375
|
+
// these four counts, so they are not an appendix.
|
|
376
|
+
legs_total: legs.length,
|
|
377
|
+
legs_exact: legs.filter((l) => l.bound === "exact").length,
|
|
378
|
+
legs_lower_bound: legs.filter((l) => l.bound === "lower").length,
|
|
379
|
+
no_completion_record: legs.filter((l) => l.end === null).length,
|
|
380
|
+
legs_unkeyed: unkeyed,
|
|
381
|
+
phase_dispatches_excluded: phase_dispatches,
|
|
382
|
+
end_sources: {
|
|
383
|
+
"leg-record": legs.filter((l) => l.end_source === "leg-record").length,
|
|
384
|
+
"t0-landmark": legs.filter((l) => l.end_source === "t0-landmark").length,
|
|
385
|
+
none: legs.filter((l) => l.end_source === null).length,
|
|
386
|
+
},
|
|
387
|
+
},
|
|
388
|
+
concurrency: {
|
|
389
|
+
max_concurrent: best ? best.max_concurrent : null,
|
|
390
|
+
bound: best ? best.max_concurrent_bound : null,
|
|
391
|
+
at: best ? best.max_concurrent_at : null,
|
|
392
|
+
launch: best ? best.launch : null,
|
|
393
|
+
max_concurrent_inner: bestInner ? bestInner.max_concurrent_inner : null,
|
|
394
|
+
// The acceptance predicate, answered rather than left to the reader — and `null` when the
|
|
395
|
+
// record set cannot answer it, which is a third state and not a "no".
|
|
396
|
+
//
|
|
397
|
+
// ANSWERED FROM THE INNER INTERVALS WHEN THEY COVER EVERY LEG, because that is the only form
|
|
398
|
+
// of the evidence an out-of-band ingest cannot fabricate. A run with dial 1 has been observed
|
|
399
|
+
// reporting `max_concurrent: 2 (exact)` because one leg was ingested long after it finished; the
|
|
400
|
+
// inner intervals said 1, which was the truth. Where the two disagree the answer is
|
|
401
|
+
// `"disputed"` rather than either number — a fourth state, and the honest one.
|
|
402
|
+
two_or_more_concurrent: !best ? null
|
|
403
|
+
: (bestInner && bestInner.max_concurrent_inner_legs === bestInner.legs)
|
|
404
|
+
? (bestInner.max_concurrent_inner >= 2 ? true : (best.max_concurrent >= 2 ? "disputed" : false))
|
|
405
|
+
: best.max_concurrent >= 2,
|
|
406
|
+
disputed_because: (best && bestInner && bestInner.max_concurrent_inner_legs === bestInner.legs
|
|
407
|
+
&& bestInner.max_concurrent_inner < 2 && best.max_concurrent >= 2)
|
|
408
|
+
? "the overlap exists only between intervals that END AT AN INGEST; measured against each leg's own " +
|
|
409
|
+
"T0 trial — an instant the leg provably occupied — no two legs were open together. An ingest run " +
|
|
410
|
+
"outside its leg (a repair, a --no-receipt-check rescue, a relaunch re-applying a result) stamps a " +
|
|
411
|
+
"later end onto the row and manufactures exactly this overlap."
|
|
412
|
+
: null,
|
|
413
|
+
},
|
|
414
|
+
launches,
|
|
415
|
+
segmentation: {
|
|
416
|
+
rule: "a quiet period longer than gap_s starts a new launch",
|
|
417
|
+
gap_s: gapS,
|
|
418
|
+
launches: launches.length,
|
|
419
|
+
note: "heuristic: nothing in the record set identifies a launch, and one run_id spans every relaunch",
|
|
420
|
+
},
|
|
421
|
+
legs: legs.map((l) => ({
|
|
422
|
+
order_id: l.order_id,
|
|
423
|
+
scope_id: l.scope_id,
|
|
424
|
+
round: l.round,
|
|
425
|
+
attempt: l.attempt,
|
|
426
|
+
run_id: l.run_id,
|
|
427
|
+
started_at: new Date(l.start).toISOString(),
|
|
428
|
+
start_source: l.start_source,
|
|
429
|
+
ended_at: l.end === null ? null : new Date(l.end).toISOString(),
|
|
430
|
+
end_source: l.end_source,
|
|
431
|
+
duration_ms: l.end === null ? null : l.end - l.start,
|
|
432
|
+
bound: l.bound,
|
|
433
|
+
// The leg's own mid-flight landmark and how far its recorded end sits past it. Reported for the
|
|
434
|
+
// reader to judge rather than thresholded, deliberately: a leg that closed inside itself and one
|
|
435
|
+
// ingested out of band differ by orders of magnitude on any single machine, and by nothing
|
|
436
|
+
// reliable ACROSS machines — so no constant that separates them here would survive slower
|
|
437
|
+
// hardware. The lag is the fact; what counts as suspicious is the reader's call.
|
|
438
|
+
inner_ended_at: l.inner_end === null ? null : new Date(l.inner_end).toISOString(),
|
|
439
|
+
end_lag_ms: (l.end === null || l.inner_end === null) ? null : l.end - l.inner_end,
|
|
440
|
+
})),
|
|
441
|
+
warnings,
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Render the report as a table for a human, from the already-built object.
|
|
447
|
+
* @param {object} r - A {@link report}.
|
|
448
|
+
* @returns {string} The human view.
|
|
449
|
+
*/
|
|
450
|
+
export function table(r) {
|
|
451
|
+
const c = r.completeness;
|
|
452
|
+
const lines = [
|
|
453
|
+
`run_root ${r.run_root}`,
|
|
454
|
+
`run_id ${r.run_id ?? "(none — records cannot be attributed to a run)"}`,
|
|
455
|
+
`dial maxParallelScopes=${r.dial.max_parallel_scopes} (${r.dial.source})`,
|
|
456
|
+
`legs ${c.legs_total} total · ${c.legs_exact} exact · ${c.legs_lower_bound} lower-bound · ${c.no_completion_record} with NO completion record` +
|
|
457
|
+
`${c.phase_dispatches_excluded ? ` (+${c.phase_dispatches_excluded} non-build dispatch(es) excluded)` : ""}`,
|
|
458
|
+
`concurrency ${r.concurrency.max_concurrent ?? "unmeasurable"}` +
|
|
459
|
+
`${r.concurrency.max_concurrent === null ? "" : ` (${r.concurrency.bound}) at ${r.concurrency.at}`}` +
|
|
460
|
+
`${r.concurrency.max_concurrent_inner === null ? "" : ` · ${r.concurrency.max_concurrent_inner} measured inside the legs' own T0 windows`}`,
|
|
461
|
+
`≥2 concurrent ${r.concurrency.two_or_more_concurrent === null ? "UNKNOWN — no leg in scope has a usable interval" : r.concurrency.two_or_more_concurrent}`,
|
|
462
|
+
...(r.concurrency.disputed_because ? [` ! ${r.concurrency.disputed_because}`] : []),
|
|
463
|
+
];
|
|
464
|
+
for (const l of r.launches) {
|
|
465
|
+
lines.push(` launch ${l.launch} ${l.from} → ${l.to} legs=${l.legs} max=${l.max_concurrent} (${l.max_concurrent_bound})` +
|
|
466
|
+
` span=${(l.span_ms / 1000).toFixed(1)}s work=${(l.sum_leg_ms / 1000).toFixed(1)}s speedup=${l.speedup ?? "refused"}`);
|
|
467
|
+
if (l.speedup === null) lines.push(` refused: ${l.speedup_refused_because}`);
|
|
468
|
+
for (const w of l.waves_observed) lines.push(` wave ${w.scopes.join(" + ")}`);
|
|
469
|
+
}
|
|
470
|
+
for (const w of r.warnings) lines.push(` ! ${w}`);
|
|
471
|
+
return lines.join("\n");
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** The typed argv contract (see `./lib/argv.mjs`). */
|
|
475
|
+
export const ARGV_SPEC = {
|
|
476
|
+
usage: "harness.mjs probe concurrency (--slug <slug> | --run-root <dir>) [--cwd <dir>] [--round N] " +
|
|
477
|
+
"[--gap-s N] [--format json|table]",
|
|
478
|
+
_: { arity: 0, max: 0, name: "(no positional operands)" },
|
|
479
|
+
slug: { type: "str" },
|
|
480
|
+
// The archived-trace and post-export cases, and the same escape `verify t0 --out` has: a caller
|
|
481
|
+
// that already holds the run root should not have to reconstruct a slug from a directory name.
|
|
482
|
+
"run-root": { type: "path" },
|
|
483
|
+
cwd: { type: "path" },
|
|
484
|
+
round: { type: "int", min: 1 },
|
|
485
|
+
"gap-s": { type: "int", min: 1, default: DEFAULT_GAP_S },
|
|
486
|
+
format: { type: "enum", values: ["json", "table"], default: "json" },
|
|
487
|
+
};
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* Report a run's leg concurrency, span and observed waves.
|
|
491
|
+
*
|
|
492
|
+
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
493
|
+
* @returns {void} Exits 0 when at least one leg had a usable interval, 1 when none did — and the
|
|
494
|
+
* report prints either way, because "nothing was measurable" is the answer, not an error.
|
|
495
|
+
*/
|
|
496
|
+
export function cli(rawArgv) {
|
|
497
|
+
const args = runArgs(ARGV_SPEC, rawArgv);
|
|
498
|
+
if (!args.slug && !args.runRoot) {
|
|
499
|
+
process.stderr.write(JSON.stringify({ error: "missing_required", flag: "--slug", expected: "a slug, or --run-root <dir>" }) + "\n");
|
|
500
|
+
process.stderr.write(`usage: ${ARGV_SPEC.usage}\n`);
|
|
501
|
+
process.exit(2);
|
|
502
|
+
}
|
|
503
|
+
const runRoot = args.runRoot
|
|
504
|
+
? resolve(args.runRoot)
|
|
505
|
+
: localRoot(resolve(args.cwd || process.cwd()), args.slug);
|
|
506
|
+
|
|
507
|
+
const r = report(runRoot, { round: args.round ?? null, gapS: args.gapS });
|
|
508
|
+
console.log(args.format === "table" ? table(r) : JSON.stringify(r));
|
|
509
|
+
process.exit(r.launches.length ? 0 : 1);
|
|
510
|
+
}
|
|
@@ -10,8 +10,7 @@
|
|
|
10
10
|
//
|
|
11
11
|
// Zero dependencies, zero network — same discipline as oracles/*.
|
|
12
12
|
|
|
13
|
-
import {
|
|
14
|
-
import { runArgs } from "./lib/argv.mjs";
|
|
13
|
+
import { runArgs } from "../lib/argv.mjs";
|
|
15
14
|
|
|
16
15
|
const PATTERNS = [
|
|
17
16
|
// Node stack frame: " at fn (path/to/file.js:12:34)" or " at path/to/file.js:12:34"
|
|
@@ -92,16 +91,19 @@ export function digest(rawText) {
|
|
|
92
91
|
// --- CLI ---------------------------------------------------------------------
|
|
93
92
|
/** The typed argv contract (see `./lib/argv.mjs`). A bare `-`, or nothing, means stdin. */
|
|
94
93
|
export const ARGV_SPEC = {
|
|
95
|
-
usage: "
|
|
94
|
+
usage: "harness.mjs probe digest [<log-file>|-] (no file → stdin)",
|
|
96
95
|
_: { arity: 0, max: 1, name: "log-file" },
|
|
97
96
|
};
|
|
98
97
|
|
|
99
98
|
/**
|
|
100
|
-
*
|
|
101
|
-
*
|
|
99
|
+
* Digest a build log into AEGIS triples on stdout.
|
|
100
|
+
*
|
|
101
|
+
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
102
|
+
* @returns {(Promise<void>|void)} Settles when the subcommand has written its output; most paths
|
|
103
|
+
* call `process.exit()` with the subcommand's documented code rather than returning.
|
|
102
104
|
*/
|
|
103
|
-
async function
|
|
104
|
-
const arg = runArgs(ARGV_SPEC)._[0];
|
|
105
|
+
export async function cli(rawArgv) {
|
|
106
|
+
const arg = runArgs(ARGV_SPEC, rawArgv)._[0];
|
|
105
107
|
let raw;
|
|
106
108
|
if (arg && arg !== "-") {
|
|
107
109
|
const { readFileSync } = await import("node:fs");
|
|
@@ -119,6 +121,3 @@ async function main() {
|
|
|
119
121
|
process.exit(0);
|
|
120
122
|
}
|
|
121
123
|
|
|
122
|
-
if (isMain(import.meta.url)) {
|
|
123
|
-
main();
|
|
124
|
-
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// probe eval — "what did round N's EVAL WorkResult actually say?"
|
|
2
|
+
//
|
|
3
|
+
// CONTRACT. A bounded, read-only query over the evaluate WorkResult ingest already wrote. Prints
|
|
4
|
+
// `{ok, overall, bug_count, report_path}` on stdout; exits 0 when found and readable, 1 when the
|
|
5
|
+
// result is missing (nothing ran, or ingest hasn't landed yet), 2 on a bad argv. Writes nothing.
|
|
6
|
+
//
|
|
7
|
+
// WHY THIS EXISTS. `shapeup-run.js` cannot read a file itself (a Workflow script has no filesystem
|
|
8
|
+
// of its own — see this repo's own note on why it may not call `Date.now()`), so every fact it
|
|
9
|
+
// branches on has to cross an `agent()`/`query()` boundary. Before this command existed, the EVAL
|
|
10
|
+
// round-loop branch (`verdict = e.overall === "PASS" ? "pass" : "fail"`) trusted the DISPATCHING
|
|
11
|
+
// sub-agent's own end-of-turn summary — a value composed from memory/judgment after three other
|
|
12
|
+
// steps (compile, dispatch, ingest), schema-checked for SHAPE only. Nothing re-verified that
|
|
13
|
+
// summary against the WorkResult `reduce ingest` had just written to disk. Measured live
|
|
14
|
+
// (2026-08-19, todo-cli): `results/evaluate-r1.json`'s `verdict.overall` was `"FAIL"` (2 cited
|
|
15
|
+
// bugs, `spec-evaluator` correctly failing `SC-ERR`), and the run proceeded straight to QA and
|
|
16
|
+
// GATE H anyway — the exact shape `verdict = e.overall === "PASS"` allows when `e.overall` is a
|
|
17
|
+
// self-report that drifted from the artifact it was supposed to summarize. This command closes
|
|
18
|
+
// that gap the same way `probe t0` already closes it for the T0 ratchet: a narrow, single-purpose
|
|
19
|
+
// "transcribe this JSON verbatim" query over the artifact itself, not a multi-step summary.
|
|
20
|
+
//
|
|
21
|
+
// WHY IT READS `results/evaluate-r<N>.json` AND NOT THE `.md` REPORT. The WorkResult is the
|
|
22
|
+
// machine-checked envelope `reduce ingest` validated against `EVAL`'s schema before ever touching
|
|
23
|
+
// shared state; the `.md` report is prose for a human. Reading the prose to re-derive a verdict a
|
|
24
|
+
// schema already carries structurally is the paraphrase channel this repo's hooks exist to close
|
|
25
|
+
// everywhere else.
|
|
26
|
+
|
|
27
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
28
|
+
import { join, resolve } from "node:path";
|
|
29
|
+
import { runArgs } from "../lib/argv.mjs";
|
|
30
|
+
import { resultsDir } from "../lib/paths.mjs";
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Read one round's EVAL verdict straight from the WorkResult `reduce ingest` wrote.
|
|
34
|
+
*
|
|
35
|
+
* @param {string} cwd - Project root.
|
|
36
|
+
* @param {string} slug - Feature slug.
|
|
37
|
+
* @param {number} round - The EVAL round (`evaluate-r<N>.json`).
|
|
38
|
+
* @returns {{found: boolean, overall: (string|null), bug_count: (number|null),
|
|
39
|
+
* report_path: (string|null)}} `found: false` when no result exists yet — a fact, not a guess.
|
|
40
|
+
*/
|
|
41
|
+
export function evalVerdict(cwd, slug, round) {
|
|
42
|
+
const path = join(resultsDir(cwd, slug), `evaluate-r${round}.json`);
|
|
43
|
+
if (!existsSync(path)) return { found: false, overall: null, bug_count: null, report_path: null };
|
|
44
|
+
let doc;
|
|
45
|
+
try { doc = JSON.parse(readFileSync(path, "utf8")); }
|
|
46
|
+
catch { return { found: false, overall: null, bug_count: null, report_path: null }; }
|
|
47
|
+
const v = doc?.verdict || {};
|
|
48
|
+
const overall = v.overall === "PASS" || v.overall === "FAIL" ? v.overall : null;
|
|
49
|
+
return {
|
|
50
|
+
found: overall !== null,
|
|
51
|
+
overall,
|
|
52
|
+
bug_count: Array.isArray(v.bugs) ? v.bugs.length : null,
|
|
53
|
+
report_path: typeof v.report_path === "string" ? v.report_path : null,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const ARGV_SPEC = {
|
|
58
|
+
usage: "harness.mjs probe eval --slug <slug> --round N [--cwd <dir>]",
|
|
59
|
+
_: { arity: 0, max: 0, name: "(no positional operands)" },
|
|
60
|
+
slug: { type: "str", required: true },
|
|
61
|
+
round: { type: "int", min: 1, required: true },
|
|
62
|
+
cwd: { type: "path" },
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Report round N's EVAL verdict, mechanically, from the WorkResult on disk.
|
|
67
|
+
*
|
|
68
|
+
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
69
|
+
* @returns {void} Exits 0 when a verdict was found, 1 when none exists yet for this round.
|
|
70
|
+
*/
|
|
71
|
+
export function cli(rawArgv) {
|
|
72
|
+
const args = runArgs(ARGV_SPEC, rawArgv);
|
|
73
|
+
const cwd = resolve(args.cwd || process.cwd());
|
|
74
|
+
const { found, overall, bug_count, report_path } = evalVerdict(cwd, args.slug, args.round);
|
|
75
|
+
console.log(JSON.stringify({ ok: found, overall, bug_count, report_path, round: args.round }));
|
|
76
|
+
process.exit(found ? 0 : 1);
|
|
77
|
+
}
|