shapeup-sdlc 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.example.json +20 -0
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +18 -0
- package/.env.shapeup.example +14 -0
- package/AGENTS.md +133 -0
- package/LICENSE +21 -0
- package/README.md +362 -0
- package/SECURITY.md +72 -0
- package/bin/init.mjs +329 -0
- package/commands/build.md +14 -0
- package/commands/eval.md +15 -0
- package/commands/hammer.md +12 -0
- package/commands/orient.md +11 -0
- package/commands/qa.md +12 -0
- package/commands/retro.md +13 -0
- package/commands/scopes.md +14 -0
- package/commands/shape.md +12 -0
- package/commands/ship.md +53 -0
- package/commands/wire.md +11 -0
- package/hooks/anti-rationalization.mjs +244 -0
- package/hooks/compact-snapshot.mjs +47 -0
- package/hooks/gate-deadline.mjs +151 -0
- package/hooks/gate-intake.mjs +110 -0
- package/hooks/gate-l2.mjs +161 -0
- package/hooks/gate-zerowork.mjs +264 -0
- package/hooks/hooks.json +118 -0
- package/hooks/lib/decision.mjs +183 -0
- package/hooks/safety-spine.mjs +296 -0
- package/hooks/sandbox-guard.mjs +172 -0
- package/hooks/session-rehydrate.mjs +109 -0
- package/hooks/slop-cleaner.mjs +176 -0
- package/oracles/_shared.mjs +46 -0
- package/oracles/http-oracle.mjs +155 -0
- package/oracles/index.mjs +36 -0
- package/oracles/process-oracle.mjs +146 -0
- package/oracles/snapshot-oracle.mjs +119 -0
- package/oracles/test-oracle.mjs +138 -0
- package/package.json +49 -0
- package/skills/advisor-protocol/SKILL.md +171 -0
- package/skills/ba-pitch-analyzer/SKILL.md +175 -0
- package/skills/ba-pitch-analyzer/assets/templates/_index.tmpl.md +84 -0
- package/skills/ba-pitch-analyzer/assets/templates/api-feasibility.tmpl.md +65 -0
- package/skills/ba-pitch-analyzer/assets/templates/assess-report.tmpl.md +127 -0
- package/skills/ba-pitch-analyzer/assets/templates/contracts/be-service.contract.tmpl.md +62 -0
- package/skills/ba-pitch-analyzer/assets/templates/contracts/offline-storage.contract.tmpl.md +92 -0
- package/skills/ba-pitch-analyzer/assets/templates/contracts/third-party-api.contract.tmpl.md +66 -0
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/context-map.tmpl.md +64 -0
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/event-choreography.tmpl.md +77 -0
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/migration-plan.tmpl.md +104 -0
- package/skills/ba-pitch-analyzer/assets/templates/cross-context/team-handoff.tmpl.md +79 -0
- package/skills/ba-pitch-analyzer/assets/templates/domain-model.tmpl.md +79 -0
- package/skills/ba-pitch-analyzer/assets/templates/feedback.tmpl.md +87 -0
- package/skills/ba-pitch-analyzer/assets/templates/integration.tmpl.md +64 -0
- package/skills/ba-pitch-analyzer/assets/templates/run-state.tmpl.md +77 -0
- package/skills/ba-pitch-analyzer/assets/templates/scope-summary.tmpl.md +105 -0
- package/skills/ba-pitch-analyzer/assets/templates/synthesis.tmpl.md +215 -0
- package/skills/ba-pitch-analyzer/assets/templates/task-board.tmpl.md +35 -0
- package/skills/ba-pitch-analyzer/assets/templates/task-spike.tmpl.md +86 -0
- package/skills/ba-pitch-analyzer/assets/templates/task.tmpl.md +84 -0
- package/skills/ba-pitch-analyzer/assets/templates/usecase.tmpl.md +105 -0
- package/skills/ba-pitch-analyzer/assets/templates/ux-behavior.tmpl.md +65 -0
- package/skills/ba-pitch-analyzer/references/contract-patterns.md +152 -0
- package/skills/ba-pitch-analyzer/references/ddd-patterns.md +173 -0
- package/skills/ba-pitch-analyzer/references/doc-schemas.md +275 -0
- package/skills/ba-pitch-analyzer/references/integration-analysis.md +152 -0
- package/skills/ba-pitch-analyzer/references/task-generation.md +623 -0
- package/skills/ba-pitch-analyzer/references/test-surface.md +102 -0
- package/skills/ba-pitch-analyzer/references/ux-behavior-patterns.md +269 -0
- package/skills/ba-pitch-analyzer/scripts/board-derive.mjs +241 -0
- package/skills/ba-pitch-analyzer/scripts/spec-lint.mjs +239 -0
- package/skills/coach/SKILL.md +202 -0
- package/skills/orient/SKILL.md +262 -0
- package/skills/qa-edge-hunter/SKILL.md +369 -0
- package/skills/scope-architect/SKILL.md +111 -0
- package/skills/scope-hammer/SKILL.md +186 -0
- package/skills/shapeup/SKILL.md +403 -0
- package/skills/shapeup/resources/breadboard-reflection.md +88 -0
- package/skills/shapeup/resources/breadboarding.md +334 -0
- package/skills/shapeup/resources/context-compaction.md +240 -0
- package/skills/shapeup/resources/framing-doc.md +92 -0
- package/skills/shapeup/resources/kickoff-doc.md +95 -0
- package/skills/shapeup/resources/shaping.md +194 -0
- package/skills/shapeup/resources/spike.md +86 -0
- package/skills/solution-architect/SKILL.md +129 -0
- package/skills/spec-evaluator/README.md +93 -0
- package/skills/spec-evaluator/SKILL.md +213 -0
- package/skills/spec-evaluator/references/anti-leniency.md +50 -0
- package/skills/spec-evaluator/references/dimension-contract.md +126 -0
- package/skills/spec-evaluator/references/dimensions/_registry.md +40 -0
- package/skills/spec-evaluator/references/dimensions/completeness.md +102 -0
- package/skills/spec-evaluator/references/dimensions/integration.md +129 -0
- package/skills/spec-evaluator/references/dimensions/performance.md +48 -0
- package/skills/spec-evaluator/references/dimensions/security.md +60 -0
- package/skills/spec-evaluator/references/dimensions/spec-conformance.md +134 -0
- package/skills/spec-evaluator/references/dimensions/tdd-surface.md +110 -0
- package/skills/spec-evaluator/references/dimensions/test-surface-conformance.md +100 -0
- package/skills/spec-evaluator/references/probing.md +177 -0
- package/skills/spec-evaluator/references/report-schema.md +101 -0
- package/skills/spec-evaluator/references/verdict-ledger.md +92 -0
- package/skills/spec-evaluator/scripts/verdict-ledger.mjs +166 -0
- package/skills/task-executor/SKILL.md +194 -0
- package/skills/tech-lead/README.md +71 -0
- package/skills/tech-lead/SKILL.md +459 -0
- package/skills/tech-lead/references/delegation.md +254 -0
- package/skills/tech-lead/references/gates.md +379 -0
- package/skills/tech-lead/references/invocation.md +45 -0
- package/skills/tech-lead/references/ledger-schema.md +214 -0
- package/skills/tech-lead/references/round-protocol.md +184 -0
- package/skills/tech-lead/references/state-model.md +66 -0
- package/skills/tech-lead/references/tiny-lane.md +52 -0
- package/skills/tech-lead/schemas/domain.schema.json +2294 -0
- package/skills/tech-lead/schemas/gate-answers.schema.json +92 -0
- package/skills/tech-lead/schemas/work-order.schema.json +21 -0
- package/skills/tech-lead/schemas/work-result.schema.json +40 -0
- package/skills/tech-lead/scripts/aegis-digest.mjs +124 -0
- package/skills/tech-lead/scripts/budget-check.mjs +156 -0
- package/skills/tech-lead/scripts/compile-order.mjs +518 -0
- package/skills/tech-lead/scripts/fit-check.mjs +196 -0
- package/skills/tech-lead/scripts/gate-answers.mjs +338 -0
- package/skills/tech-lead/scripts/ingest-result.mjs +270 -0
- package/skills/tech-lead/scripts/init-run.mjs +326 -0
- package/skills/tech-lead/scripts/lib/argv.mjs +224 -0
- package/skills/tech-lead/scripts/lib/contract-md.mjs +481 -0
- package/skills/tech-lead/scripts/lib/is-main.mjs +82 -0
- package/skills/tech-lead/scripts/lib/paths.mjs +278 -0
- package/skills/tech-lead/scripts/lib/ratchet-tree.mjs +112 -0
- package/skills/tech-lead/scripts/run-snapshot.mjs +269 -0
- package/skills/tech-lead/scripts/ship-report.mjs +275 -0
- package/skills/tech-lead/scripts/stats.mjs +423 -0
- package/skills/tech-lead/scripts/t0-verify.mjs +470 -0
- package/skills/tech-lead/scripts/trace-lint.mjs +367 -0
- package/skills/tech-lead/scripts/validate-envelope.mjs +283 -0
- package/skills/translator/README.md +66 -0
- package/skills/translator/SKILL.md +258 -0
- package/skills/translator/references/preservation-rules.md +102 -0
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// T0 mechanical verification layer (design spec v1.1 §3.5, Blueprint A/E).
|
|
3
|
+
//
|
|
4
|
+
// Runs a scope's e2e fixtures + DB probe (zero LLM tokens), then — on green — the seesaw
|
|
5
|
+
// regression check (re-runs every FINISHED scope's fixtures from the registry). Writes one
|
|
6
|
+
// verdict artifact per attempt that spec-evaluator (T1) must cite; a verdict without it is
|
|
7
|
+
// structurally invalid (PA4 countermeasure, DD-7). No agent can fabricate this file's contents
|
|
8
|
+
// because it is produced by actually running the commands.
|
|
9
|
+
//
|
|
10
|
+
// Zero dependencies, zero network — same discipline as oracles/* and gate-l2.mjs.
|
|
11
|
+
//
|
|
12
|
+
// THE PAWL (v1.5). This script also owns the ratchet's comparison and its history. The attempt
|
|
13
|
+
// loop used to be a BUDGETED RETRY LOOP wearing a ratchet's shape: `computeVerdict` returned four
|
|
14
|
+
// booleans, N fixture outcomes collapsed through a single AND, and an attempt that moved 2-of-5
|
|
15
|
+
// fixtures to 4-of-5 was recorded identically to one that moved 2-of-5 to 0-of-5 — `red`. With no
|
|
16
|
+
// scalar there was nothing to compare, so `better()` could not exist, so nothing was ever kept or
|
|
17
|
+
// reverted on the strength of having improved. Three additions close that, and all three reduce
|
|
18
|
+
// over data this script ALREADY wrote to disk:
|
|
19
|
+
//
|
|
20
|
+
// • `score()` — the comparable outcome vector (§2.1). No new measurement is taken.
|
|
21
|
+
// • `better()` — strict lexicographic comparison; a tie is NOT better, because a tie that counts
|
|
22
|
+
// as an improvement makes a sawtooth look like a ratchet.
|
|
23
|
+
// • `trials.jsonl` — one append-only row per T0 run, with `baseline_trial` as the parent link.
|
|
24
|
+
// That single field is the experiment DAG (lineage + SUPERSEDES) with no graph
|
|
25
|
+
// store, and it is what `compile-order` reads back as `inspect()`'s history.
|
|
26
|
+
//
|
|
27
|
+
// The consequence that matters: an attempt that moves 2/5 → 4/5 fixtures is RED BUT BETTER, and is
|
|
28
|
+
// therefore KEPT. The ratchet retains improvements, not just greens — which is what lets attempt
|
|
29
|
+
// N+1 build on attempt N instead of restarting from unexplained code.
|
|
30
|
+
//
|
|
31
|
+
// Usage:
|
|
32
|
+
// node "${CLAUDE_PLUGIN_ROOT}/skills/tech-lead/scripts/t0-verify.mjs" <scope-contract.json> \
|
|
33
|
+
// --round N --attempt M [--cwd <dir>] [--out <dir>] [--seesaw-registry <path>] [--no-seesaw]
|
|
34
|
+
// [--no-ratchet]
|
|
35
|
+
//
|
|
36
|
+
// Exit code: 0 = overall green, 1 = overall red (mirrors the oracle convention), 2 = bad argv.
|
|
37
|
+
|
|
38
|
+
import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, readdirSync } from "node:fs";
|
|
39
|
+
import { join, dirname } from "node:path";
|
|
40
|
+
import { spawnSync } from "node:child_process";
|
|
41
|
+
import { createHash } from "node:crypto";
|
|
42
|
+
import { digest } from "./aegis-digest.mjs";
|
|
43
|
+
import { isMain } from "./lib/is-main.mjs";
|
|
44
|
+
import { runArgs } from "./lib/argv.mjs";
|
|
45
|
+
import { snapshot, restore, keptRef } from "./lib/ratchet-tree.mjs";
|
|
46
|
+
import { readContract, SCOPE_CONTRACT } from "./lib/contract-md.mjs";
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Run one shell command and capture its outcome (10-minute timeout).
|
|
50
|
+
* @param {string} cmd - The command line to run in a shell.
|
|
51
|
+
* @param {string} cwd - Working directory to run it in.
|
|
52
|
+
* @returns {{cmd:string, exit:number, pass:boolean, stdout:string, stderr:string}} The command,
|
|
53
|
+
* its exit code (1 when null), whether it exited 0, and captured output.
|
|
54
|
+
*/
|
|
55
|
+
function runCommand(cmd, cwd) {
|
|
56
|
+
const r = spawnSync(cmd, { shell: true, cwd, encoding: "utf8", timeout: 10 * 60 * 1000 });
|
|
57
|
+
const stdout = r.stdout || "";
|
|
58
|
+
const stderr = r.stderr || "";
|
|
59
|
+
// A spawn failure or a timeout is NOT the same fact as "the command ran and failed", and the
|
|
60
|
+
// ratchet grades it differently (`crash` → restore, never `reverted`). Keeping it costs a field.
|
|
61
|
+
const error = r.error ? String(r.error.message || r.error) : null;
|
|
62
|
+
return { cmd, exit: r.status ?? 1, pass: r.status === 0, stdout, stderr, ...(error ? { error } : {}) };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Run every e2e fixture command for a scope.
|
|
67
|
+
* @param {string[]} fixtures - Fixture command lines (null/empty → no commands).
|
|
68
|
+
* @param {string} cwd - Working directory.
|
|
69
|
+
* @returns {{pass:boolean, results:Array<{cmd:string,exit:number,pass:boolean,stdout:string,
|
|
70
|
+
* stderr:string}>}} pass=true iff every fixture passed, plus each command's result.
|
|
71
|
+
*/
|
|
72
|
+
export function runFixtures(fixtures, cwd) {
|
|
73
|
+
const results = (fixtures || []).map((cmd) => runCommand(cmd, cwd));
|
|
74
|
+
return { pass: results.every((r) => r.pass), results };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Run the scope's DB probe, if one is declared.
|
|
79
|
+
* @param {(string|null|undefined)} dbProbeCmd - The probe command, or falsy when none applies.
|
|
80
|
+
* @param {string} cwd - Working directory.
|
|
81
|
+
* @returns {({cmd:string,exit:number,pass:boolean,stdout:string,stderr:string}|null)} The command
|
|
82
|
+
* result, or null when no probe is declared (null never counts as a failure).
|
|
83
|
+
*/
|
|
84
|
+
export function runDbProbe(dbProbeCmd, cwd) {
|
|
85
|
+
if (!dbProbeCmd) return null;
|
|
86
|
+
return runCommand(dbProbeCmd, cwd);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Re-run every FINISHED scope's fixtures from the seesaw registry (regression guard).
|
|
91
|
+
* @param {(string|null)} registryPath - Path to the seesaw registry JSON (absent/unreadable → skipped).
|
|
92
|
+
* @param {string} cwd - Working directory.
|
|
93
|
+
* @returns {{ran:boolean, pass:boolean, scopes_checked:string[], failing:string[], error?:string}}
|
|
94
|
+
* ran=false/pass=true when skipped; otherwise pass=true iff no prior scope regressed, with the
|
|
95
|
+
* scope ids checked and those now failing.
|
|
96
|
+
*/
|
|
97
|
+
export function seesawCheck(registryPath, cwd) {
|
|
98
|
+
if (!registryPath || !existsSync(registryPath)) {
|
|
99
|
+
return { ran: false, pass: true, scopes_checked: [], failing: [] };
|
|
100
|
+
}
|
|
101
|
+
let registry;
|
|
102
|
+
try {
|
|
103
|
+
registry = JSON.parse(readFileSync(registryPath, "utf8"));
|
|
104
|
+
} catch {
|
|
105
|
+
return { ran: false, pass: true, scopes_checked: [], failing: [], error: "registry unparsable" };
|
|
106
|
+
}
|
|
107
|
+
const scopes = registry.scopes || [];
|
|
108
|
+
const failing = [];
|
|
109
|
+
for (const s of scopes) {
|
|
110
|
+
const { pass } = runFixtures(s.fixtures, cwd);
|
|
111
|
+
if (!pass) failing.push(s.scope_id);
|
|
112
|
+
}
|
|
113
|
+
return { ran: true, pass: failing.length === 0, scopes_checked: scopes.map((s) => s.scope_id), failing };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Combine fixtures + DB probe + seesaw into the overall T0 verdict.
|
|
118
|
+
* @param {{fixtures:{pass:boolean}, dbProbe:({pass:boolean}|null),
|
|
119
|
+
* seesaw:{ran:boolean,pass:boolean}}} parts - The three sub-results.
|
|
120
|
+
* @returns {{fixtures_green:boolean, db_probe_green:boolean, seesaw_green:boolean,
|
|
121
|
+
* overall:("green"|"red"), regression:boolean}} Per-arm greens, the overall verdict (green iff
|
|
122
|
+
* all three), and `regression` = fixtures+db green but seesaw red (the rollback-and-retry case).
|
|
123
|
+
*/
|
|
124
|
+
export function computeVerdict({ fixtures, dbProbe, seesaw }) {
|
|
125
|
+
const fixturesGreen = fixtures.pass;
|
|
126
|
+
const dbGreen = dbProbe === null || dbProbe.pass;
|
|
127
|
+
const seesawGreen = !seesaw.ran || seesaw.pass;
|
|
128
|
+
return {
|
|
129
|
+
fixtures_green: fixturesGreen,
|
|
130
|
+
db_probe_green: dbGreen,
|
|
131
|
+
seesaw_green: seesawGreen,
|
|
132
|
+
overall: fixturesGreen && dbGreen && seesawGreen ? "green" : "red",
|
|
133
|
+
// A regression is specifically fixtures/db green but seesaw red — the case that should
|
|
134
|
+
// trigger rollback+retry (spec §3.5) rather than "go fix the new scope's own bug".
|
|
135
|
+
regression: fixturesGreen && dbGreen && !seesawGreen,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The comparable T0 outcome — a VECTOR, not a float, because the three arms are not fungible.
|
|
141
|
+
*
|
|
142
|
+
* Every number here is a reduce over data `writeArtifact` already persists (`fixtures:
|
|
143
|
+
* [{cmd, exit, pass}]`). Nothing new is measured; a number that has always been on disk is
|
|
144
|
+
* finally counted.
|
|
145
|
+
*
|
|
146
|
+
* @param {{fixtures:{results:Array<{pass:boolean}>}, dbProbe:({pass:boolean}|null),
|
|
147
|
+
* seesaw:{ran:boolean, failing:string[]}}} parts - The three T0 sub-results.
|
|
148
|
+
* @returns {{regressions:number, fixtures_passed:number, fixtures_total:number,
|
|
149
|
+
* db_probe:(0|1|null)}} The score vector. `db_probe` is null when no probe is declared, which
|
|
150
|
+
* is never a failure — only an absence.
|
|
151
|
+
*/
|
|
152
|
+
export function score({ fixtures, dbProbe, seesaw }) {
|
|
153
|
+
return {
|
|
154
|
+
regressions: seesaw?.ran ? (seesaw.failing || []).length : 0,
|
|
155
|
+
fixtures_passed: fixtures.results.filter((r) => r.pass).length,
|
|
156
|
+
fixtures_total: fixtures.results.length,
|
|
157
|
+
db_probe: dbProbe === null || dbProbe === undefined ? null : (dbProbe.pass ? 1 : 0),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The pawl. Lexicographic and STRICT.
|
|
163
|
+
*
|
|
164
|
+
* Three decisions worth defending:
|
|
165
|
+
* • A TIE IS NOT BETTER. A tie that counted as an improvement would make a sawtooth look like a
|
|
166
|
+
* ratchet, and the whole point of the Day-1 measurement is to tell those two apart.
|
|
167
|
+
* • REGRESSIONS DOMINATE. Breaking a previously-finished scope is never an improvement, whatever
|
|
168
|
+
* the new scope's fixtures did. This is what lets the old seesaw branch collapse into the
|
|
169
|
+
* general rule rather than needing a special case.
|
|
170
|
+
* • DIFFERENT `fixtures_total` IS INCOMPARABLE, not worse. A scope split or remap changes the
|
|
171
|
+
* denominator; comparing across it is a category error, so the ratchet treats it as a baseline
|
|
172
|
+
* reset (`rebased`) rather than issuing a false verdict.
|
|
173
|
+
*
|
|
174
|
+
* @param {{regressions:number, fixtures_passed:number, fixtures_total:number,
|
|
175
|
+
* db_probe:(0|1|null)}} next - The candidate score.
|
|
176
|
+
* @param {({regressions:number, fixtures_passed:number, fixtures_total:number,
|
|
177
|
+
* db_probe:(0|1|null)}|null)} current - The incumbent score, or null for the first trial.
|
|
178
|
+
* @returns {(boolean|null)} true = strictly better · false = not better · null = incomparable.
|
|
179
|
+
*/
|
|
180
|
+
export function better(next, current) {
|
|
181
|
+
if (current === null || current === undefined) return true; // baseline
|
|
182
|
+
if (next.fixtures_total !== current.fixtures_total) return null; // the contract changed
|
|
183
|
+
if (next.regressions !== current.regressions) return next.regressions < current.regressions;
|
|
184
|
+
if (next.fixtures_passed !== current.fixtures_passed) return next.fixtures_passed > current.fixtures_passed;
|
|
185
|
+
if (next.db_probe !== current.db_probe) return (next.db_probe ?? 0) > (current.db_probe ?? 0);
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* The one rule that replaces the protocol's two red branches.
|
|
191
|
+
*
|
|
192
|
+
* | better(next, current) | action | status |
|
|
193
|
+
* |-----------------------|---------------------------------|------------|
|
|
194
|
+
* | true | keep tree · current = next | `kept` |
|
|
195
|
+
* | false | restore last kept snapshot | `reverted` |
|
|
196
|
+
* | null | keep tree · reset the baseline | `rebased` |
|
|
197
|
+
* | a command crashed | restore last kept snapshot | `crash` |
|
|
198
|
+
*
|
|
199
|
+
* @param {(boolean|null)} verdict - The result of {@link better}.
|
|
200
|
+
* @param {boolean} crashed - True when any command failed to spawn or timed out.
|
|
201
|
+
* @returns {{status:("kept"|"reverted"|"rebased"|"crash"), action:("keep"|"restore")}} The status
|
|
202
|
+
* recorded on the trial row and the tree operation to perform.
|
|
203
|
+
*/
|
|
204
|
+
export function decideStatus(verdict, crashed) {
|
|
205
|
+
if (crashed) return { status: "crash", action: "restore" };
|
|
206
|
+
if (verdict === null) return { status: "rebased", action: "keep" };
|
|
207
|
+
return verdict ? { status: "kept", action: "keep" } : { status: "reverted", action: "restore" };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Human-readable one-line summary of a score change, for the trial row's `delta` field.
|
|
212
|
+
* @param {object} next - The candidate score.
|
|
213
|
+
* @param {(object|null)} current - The incumbent score, or null.
|
|
214
|
+
* @returns {string} e.g. "+2 fixtures", "+1 regression", "baseline", "no change".
|
|
215
|
+
*/
|
|
216
|
+
export function describeDelta(next, current) {
|
|
217
|
+
if (!current) return "baseline";
|
|
218
|
+
if (next.fixtures_total !== current.fixtures_total) {
|
|
219
|
+
return `denominator ${current.fixtures_total} → ${next.fixtures_total}`;
|
|
220
|
+
}
|
|
221
|
+
const parts = [];
|
|
222
|
+
const dr = next.regressions - current.regressions;
|
|
223
|
+
const df = next.fixtures_passed - current.fixtures_passed;
|
|
224
|
+
const dp = (next.db_probe ?? 0) - (current.db_probe ?? 0);
|
|
225
|
+
if (dr) parts.push(`${dr > 0 ? "+" : ""}${dr} regression${Math.abs(dr) === 1 ? "" : "s"}`);
|
|
226
|
+
if (df) parts.push(`${df > 0 ? "+" : ""}${df} fixture${Math.abs(df) === 1 ? "" : "s"}`);
|
|
227
|
+
if (dp) parts.push(`${dp > 0 ? "+" : ""}${dp} db_probe`);
|
|
228
|
+
return parts.length ? parts.join(", ") : "no change";
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Read the append-only trial ledger.
|
|
233
|
+
* @param {string} path - Path to `t0/trials.jsonl`.
|
|
234
|
+
* @returns {Array<object>} One entry per parseable row, in write order; [] when the file is absent
|
|
235
|
+
* (the non-regression path — every caller falls back to today's behaviour).
|
|
236
|
+
*/
|
|
237
|
+
export function readTrials(path) {
|
|
238
|
+
if (!existsSync(path)) return [];
|
|
239
|
+
try {
|
|
240
|
+
return readFileSync(path, "utf8")
|
|
241
|
+
.split("\n")
|
|
242
|
+
.filter((l) => l.trim())
|
|
243
|
+
.map((l) => { try { return JSON.parse(l); } catch { return null; } })
|
|
244
|
+
.filter(Boolean);
|
|
245
|
+
} catch { return []; }
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Append one row to the trial ledger.
|
|
250
|
+
*
|
|
251
|
+
* WRITER RULE. `t0-verify` writes this, not `ingest-result`. The single-writer invariant governs
|
|
252
|
+
* WORKER-DERIVED shared state — board, ledger, verdict — because a worker's claims must pass
|
|
253
|
+
* through one applier. A trial row is a MECHANICAL FACT produced by running commands, in exactly
|
|
254
|
+
* the same class as the verdict artifact this script already owns; routing it through an envelope
|
|
255
|
+
* that carries no worker claim would buy nothing.
|
|
256
|
+
*
|
|
257
|
+
* @param {string} path - Path to `t0/trials.jsonl`.
|
|
258
|
+
* @param {object} row - The row to append.
|
|
259
|
+
* @returns {boolean} True when the row was written. Best-effort: a ledger that can break a build
|
|
260
|
+
* would get the ratchet disabled, which costs more than the row is worth.
|
|
261
|
+
*/
|
|
262
|
+
export function appendTrial(path, row) {
|
|
263
|
+
try {
|
|
264
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
265
|
+
appendFileSync(path, JSON.stringify(row) + "\n");
|
|
266
|
+
return true;
|
|
267
|
+
} catch { return false; }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The next retry ordinal for a (round, attempt) pair, read from the verdicts directory.
|
|
272
|
+
*
|
|
273
|
+
* The legacy unsuffixed `r<R>-a<A>.json` written before v1.5 is treated as `t0`, so old artifacts
|
|
274
|
+
* on disk stay readable and can never be collided with.
|
|
275
|
+
*
|
|
276
|
+
* @param {string} dir - The `t0/verdicts` directory.
|
|
277
|
+
* @param {number} round - Round number.
|
|
278
|
+
* @param {number} attempt - Attempt number.
|
|
279
|
+
* @returns {number} The lowest unused ordinal ≥ 1.
|
|
280
|
+
*/
|
|
281
|
+
export function nextTrialNo(dir, round, attempt) {
|
|
282
|
+
let max = 0;
|
|
283
|
+
let entries;
|
|
284
|
+
try { entries = readdirSync(dir); } catch { return 1; }
|
|
285
|
+
const re = new RegExp(`^r${round}-a${attempt}-t(\\d+)\\.json$`);
|
|
286
|
+
for (const f of entries) {
|
|
287
|
+
const m = f.match(re);
|
|
288
|
+
if (m) max = Math.max(max, Number(m[1]));
|
|
289
|
+
}
|
|
290
|
+
return max + 1;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Distill every failing command's output into AEGIS {file,line,core_message} triples.
|
|
295
|
+
* @param {{fixtures:{results:Array<{pass:boolean,stdout:string,stderr:string}>},
|
|
296
|
+
* dbProbe:({pass:boolean,stdout:string,stderr:string}|null)}} parts - The T0 sub-results.
|
|
297
|
+
* @returns {Array<{file:(string|null), line:(number|null), core_message:string, kind:string}>}
|
|
298
|
+
* Deduped triples across all failing logs; [] when nothing failed.
|
|
299
|
+
*/
|
|
300
|
+
export function digestFailures({ fixtures, dbProbe }) {
|
|
301
|
+
const failingLogs = [];
|
|
302
|
+
for (const r of fixtures.results) if (!r.pass) failingLogs.push(r.stdout + "\n" + r.stderr);
|
|
303
|
+
if (dbProbe && !dbProbe.pass) failingLogs.push(dbProbe.stdout + "\n" + dbProbe.stderr);
|
|
304
|
+
return failingLogs.flatMap((log) => digest(log));
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* @param {string} text - Bytes to hash.
|
|
309
|
+
* @returns {string} The lowercase hex SHA-256 digest of `text`.
|
|
310
|
+
*/
|
|
311
|
+
function sha256(text) {
|
|
312
|
+
return createHash("sha256").update(text).digest("hex");
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Write the T0 verdict artifact spec-evaluator (T1) must cite — IMMUTABLY (invariant I4:
|
|
317
|
+
* every superseded object remains addressable).
|
|
318
|
+
*
|
|
319
|
+
* WHAT THIS REPLACED, and why the remedy is `wx` rather than a guard. The address used to be
|
|
320
|
+
* `r<round>-a<attempt>.json`, written with a bare `writeFileSync` — and on a seesaw regression the
|
|
321
|
+
* protocol says stash, then RETRY THIS ATTEMPT, same attempt number. The address had no term for
|
|
322
|
+
* the retry, so the artifact recording the regression was silently replaced by the one recording
|
|
323
|
+
* the recovery, at the same path. Reproduced against the shipped script: two runs at
|
|
324
|
+
* `--round 1 --attempt 2`, one red and one green, left ONE file — the red verdict was gone. This
|
|
325
|
+
* is the same missing-identity-key defect the benchmark logged as its own #9, recurring on the one
|
|
326
|
+
* artifact the evaluator is structurally required to cite.
|
|
327
|
+
*
|
|
328
|
+
* An `existsSync` guard would be check-then-write: still racy, and still a policy expressed in code
|
|
329
|
+
* rather than a property of the store. `flag: "wx"` makes overwriting IMPOSSIBLE — the filesystem
|
|
330
|
+
* refuses with EEXIST and the loop moves to the next ordinal. Same class of move as `lib/is-main.mjs`:
|
|
331
|
+
* replace a fragile comparison with one that cannot silently be wrong.
|
|
332
|
+
*
|
|
333
|
+
* The evaluator's citation contract is unaffected: it re-hashes whatever path it is handed, and
|
|
334
|
+
* `T0Citation` carries `path` + `sha256`, never a filename pattern.
|
|
335
|
+
*
|
|
336
|
+
* @param {string} outDir - Base output dir; the file lands at
|
|
337
|
+
* `<outDir>/t0/verdicts/r<round>-a<attempt>-t<trial>.json`.
|
|
338
|
+
* @param {number} round - Round number.
|
|
339
|
+
* @param {number} attempt - Attempt number within the round.
|
|
340
|
+
* @param {object} verdictBody - Verdict fields to persist (scope_id, per-arm results, score,
|
|
341
|
+
* discovered_tasks…).
|
|
342
|
+
* @returns {{path:string, sha256:string, trial:number}} The artifact path, the sha-256 of its exact
|
|
343
|
+
* bytes (the citation the evaluator's report must include), and the retry ordinal used.
|
|
344
|
+
* Side effect: writes the JSON file, never over an existing one.
|
|
345
|
+
*/
|
|
346
|
+
export function writeArtifact(outDir, round, attempt, verdictBody) {
|
|
347
|
+
const dir = join(outDir, "t0", "verdicts");
|
|
348
|
+
mkdirSync(dir, { recursive: true });
|
|
349
|
+
for (let trial = nextTrialNo(dir, round, attempt); ; trial++) {
|
|
350
|
+
const path = join(dir, `r${round}-a${attempt}-t${trial}.json`);
|
|
351
|
+
const body = {
|
|
352
|
+
schema_version: 2, round, attempt, trial,
|
|
353
|
+
at: new Date().toISOString(), ...verdictBody,
|
|
354
|
+
};
|
|
355
|
+
const text = JSON.stringify(body, null, 2);
|
|
356
|
+
try {
|
|
357
|
+
writeFileSync(path, text, { flag: "wx" }); // EEXIST, never clobber
|
|
358
|
+
return { path, sha256: sha256(text), trial };
|
|
359
|
+
} catch (e) {
|
|
360
|
+
if (e.code !== "EEXIST") throw e;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** The typed argv contract (see `./lib/argv.mjs`). */
|
|
366
|
+
export const ARGV_SPEC = {
|
|
367
|
+
usage: "t0-verify.mjs <scope-contract.json> --round N --attempt M [--cwd <dir>] [--out <dir>] " +
|
|
368
|
+
"[--seesaw-registry <path>] [--no-seesaw] [--no-ratchet]",
|
|
369
|
+
_: { arity: 1, max: 1, name: "scope-contract.json" },
|
|
370
|
+
round: { type: "int", min: 1, required: true },
|
|
371
|
+
attempt: { type: "int", min: 1, required: true },
|
|
372
|
+
cwd: { type: "path" },
|
|
373
|
+
out: { type: "path" },
|
|
374
|
+
"seesaw-registry": { type: "path" },
|
|
375
|
+
"no-seesaw": { type: "flag" },
|
|
376
|
+
"no-ratchet": { type: "flag" },
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* CLI entry: run a scope's fixtures + probe + seesaw, write the verdict artifact, print it, and
|
|
381
|
+
* exit 0 (green) / 1 (red) / 2 (usage).
|
|
382
|
+
* @returns {Promise<void>} Resolves after writing the artifact; the process exit code carries the verdict.
|
|
383
|
+
*/
|
|
384
|
+
async function main() {
|
|
385
|
+
const args = runArgs(ARGV_SPEC);
|
|
386
|
+
const contractPath = args._[0];
|
|
387
|
+
// Markdown first, legacy JSON second (ADR-0001, lib/contract-md.mjs).
|
|
388
|
+
const found = readContract(contractPath, SCOPE_CONTRACT);
|
|
389
|
+
if (!found) { console.error(`t0-verify: no scope contract at ${contractPath} (.md or .json)`); process.exit(2); }
|
|
390
|
+
const contract = found.contract;
|
|
391
|
+
const cwd = args.cwd || process.cwd();
|
|
392
|
+
const outDir = args.out || dirname(dirname(contractPath)); // default: <slug>/ (parent of scopes/)
|
|
393
|
+
const round = args.round;
|
|
394
|
+
const attempt = args.attempt;
|
|
395
|
+
|
|
396
|
+
const fixtures = runFixtures(contract.e2e_verification_fixtures, cwd);
|
|
397
|
+
const dbProbe = runDbProbe(contract.db_probe, cwd);
|
|
398
|
+
// --seesaw-registry is expected explicitly (tech-lead always passes it, delegation.md 3c);
|
|
399
|
+
// standalone CLI use without it simply skips the seesaw check rather than guessing a path.
|
|
400
|
+
const seesawRegistry = args.noSeesaw ? null : args.seesawRegistry || null;
|
|
401
|
+
const seesaw = args.noSeesaw || fixtures.pass === false
|
|
402
|
+
? { ran: false, pass: true, scopes_checked: [], failing: [] } // don't seesaw on an already-red attempt
|
|
403
|
+
: seesawCheck(seesawRegistry, cwd);
|
|
404
|
+
|
|
405
|
+
const verdict = computeVerdict({ fixtures, dbProbe, seesaw });
|
|
406
|
+
const discovered = verdict.overall === "red" ? digestFailures({ fixtures, dbProbe }) : [];
|
|
407
|
+
|
|
408
|
+
// ---- the ratchet ---------------------------------------------------------------------
|
|
409
|
+
// `current` is the incumbent: the score of the most recent trial whose TREE is the one on disk
|
|
410
|
+
// (`kept` or `rebased`). A `reverted` or `crash` trial's tree was thrown away, so its score is
|
|
411
|
+
// history, never the thing to beat.
|
|
412
|
+
const trialsPath = join(outDir, "t0", "trials.jsonl");
|
|
413
|
+
const priorTrials = readTrials(trialsPath).filter((t) => t.scope_id === contract.scope_id);
|
|
414
|
+
const baseline = [...priorTrials].reverse().find((t) => t.status === "kept" || t.status === "rebased") || null;
|
|
415
|
+
const s = score({ fixtures, dbProbe, seesaw });
|
|
416
|
+
const verdictBetter = better(s, baseline ? baseline.score : null);
|
|
417
|
+
const crashed = fixtures.results.some((r) => r.error) || !!dbProbe?.error;
|
|
418
|
+
const { status, action } = decideStatus(verdictBetter, crashed);
|
|
419
|
+
|
|
420
|
+
const { path, sha256: hash, trial } = writeArtifact(outDir, round, attempt, {
|
|
421
|
+
scope_id: contract.scope_id,
|
|
422
|
+
fixtures: fixtures.results.map(({ cmd, exit, pass }) => ({ cmd, exit, pass })),
|
|
423
|
+
db_probe: dbProbe && { cmd: dbProbe.cmd, exit: dbProbe.exit, pass: dbProbe.pass },
|
|
424
|
+
seesaw,
|
|
425
|
+
...verdict,
|
|
426
|
+
score: s,
|
|
427
|
+
discovered_tasks: discovered,
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
// The tree operation. `--no-ratchet` leaves the working tree exactly as the attempt left it —
|
|
431
|
+
// the pre-v1.5 behaviour, kept for standalone CLI use and for any caller managing its own tree.
|
|
432
|
+
let tree = { ok: false, reason: "--no-ratchet" };
|
|
433
|
+
if (!args.noRatchet) {
|
|
434
|
+
tree = action === "keep" ? snapshot(contract.scope_id, cwd) : restore(contract.scope_id, cwd);
|
|
435
|
+
// First trial with nothing to restore to: there is no kept tree yet BY DEFINITION. Take one,
|
|
436
|
+
// so trial 2 has a floor to fall back to instead of inheriting the "no revert at all" defect.
|
|
437
|
+
if (action === "restore" && !tree.ok) tree = { ...snapshot(contract.scope_id, cwd), fell_back_to: "snapshot" };
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// `baseline_trial` is the parent link — the experiment DAG (lineage, PARENT_OF and a genuine
|
|
441
|
+
// SUPERSEDES edge) delivered as one field, with no graph store behind it.
|
|
442
|
+
const trialNo = readTrials(trialsPath).length + 1;
|
|
443
|
+
const row = {
|
|
444
|
+
schema_version: 1,
|
|
445
|
+
trial: trialNo,
|
|
446
|
+
round, attempt,
|
|
447
|
+
scope_id: contract.scope_id,
|
|
448
|
+
at: new Date().toISOString(),
|
|
449
|
+
artifact: path.startsWith(outDir) ? path.slice(outDir.length).replace(/^[/\\]/, "") : path,
|
|
450
|
+
sha256: hash,
|
|
451
|
+
score: s,
|
|
452
|
+
status,
|
|
453
|
+
baseline_trial: baseline ? baseline.trial : null,
|
|
454
|
+
delta: describeDelta(s, baseline ? baseline.score : null),
|
|
455
|
+
tree_ref: tree.ok ? tree.ref : null,
|
|
456
|
+
digest: discovered.slice(0, 8),
|
|
457
|
+
};
|
|
458
|
+
appendTrial(trialsPath, row);
|
|
459
|
+
|
|
460
|
+
console.log(JSON.stringify({
|
|
461
|
+
path, sha256: hash, trial, overall: verdict.overall, regression: verdict.regression,
|
|
462
|
+
score: s, status, baseline_trial: row.baseline_trial, delta: row.delta,
|
|
463
|
+
tree_ref: row.tree_ref ?? keptRef(contract.scope_id),
|
|
464
|
+
}, null, 2));
|
|
465
|
+
process.exit(verdict.overall === "green" ? 0 : 1);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
if (isMain(import.meta.url)) {
|
|
469
|
+
main();
|
|
470
|
+
}
|