shapeup-sdlc 3.18.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude/settings.local.example.json +2 -3
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/AGENTS.md +115 -56
  4. package/README.md +101 -143
  5. package/SECURITY.md +16 -25
  6. package/agents/orient.md +14 -0
  7. package/agents/qa-edge-hunter.md +13 -0
  8. package/agents/spec-evaluator.md +14 -0
  9. package/agents/task-executor.md +14 -0
  10. package/bin/init.mjs +13 -16
  11. package/bin/lib/grant.mjs +8 -19
  12. package/commands/build.md +9 -14
  13. package/commands/eval.md +3 -3
  14. package/commands/hammer.md +9 -5
  15. package/commands/retro.md +12 -11
  16. package/commands/scopes.md +1 -12
  17. package/commands/ship.md +45 -88
  18. package/commands/wire.md +1 -8
  19. package/hooks/dispatch-receipt.mjs +94 -8
  20. package/hooks/hooks.json +0 -41
  21. package/hooks/lib/decision.mjs +6 -9
  22. package/hooks/lib/tier.mjs +50 -0
  23. package/hooks/sandbox-guard.mjs +140 -64
  24. package/kernel/gate.mjs +203 -22
  25. package/kernel/harness.mjs +20 -67
  26. package/kernel/init/run.mjs +175 -523
  27. package/kernel/lib/contract.mjs +9 -93
  28. package/kernel/lib/floor.mjs +64 -0
  29. package/kernel/lib/paths.mjs +39 -112
  30. package/kernel/order.mjs +157 -0
  31. package/kernel/probe/digest.mjs +2 -38
  32. package/kernel/schemas/domain.schema.json +146 -1125
  33. package/kernel/schemas/work-order.schema.json +8 -4
  34. package/kernel/schemas/work-result.schema.json +1 -1
  35. package/kernel/verify/envelope.mjs +2 -2
  36. package/kernel/verify/skills.mjs +27 -8
  37. package/kernel/verify/t0.mjs +2 -2
  38. package/package.json +4 -2
  39. package/skills/building/SKILL.md +144 -0
  40. package/skills/building/resources/gates.md +76 -0
  41. package/skills/building/resources/guidance.md +23 -0
  42. package/skills/{translator/references/preservation-rules.md → building/resources/intake-preservation.md} +1 -1
  43. package/skills/building/resources/intake.md +157 -0
  44. package/skills/building/resources/orders.md +69 -0
  45. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/doc-schemas.md +1 -1
  46. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/task-generation.md +13 -13
  47. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/test-surface.md +6 -7
  48. package/skills/building/resources/plan-scopes.md +95 -0
  49. package/skills/building/resources/plan-spec.md +102 -0
  50. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/_index.tmpl.md +1 -1
  51. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/assess-report.tmpl.md +10 -11
  52. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/run-state.tmpl.md +4 -4
  53. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/scope-summary.tmpl.md +2 -2
  54. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/synthesis.tmpl.md +4 -4
  55. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/usecase.tmpl.md +2 -2
  56. package/skills/building/resources/plan-wire.md +82 -0
  57. package/skills/building/resources/plan.md +98 -0
  58. package/skills/building/resources/retro-research.md +60 -0
  59. package/skills/building/resources/retro-scan.md +49 -0
  60. package/skills/building/resources/retro.md +225 -0
  61. package/skills/building/resources/rounds.md +78 -0
  62. package/skills/building/resources/stop.md +127 -0
  63. package/skills/orient/SKILL.md +37 -37
  64. package/skills/qa-edge-hunter/SKILL.md +29 -14
  65. package/skills/shapeup/SKILL.md +5 -5
  66. package/skills/shapeup/resources/breadboarding.md +1 -1
  67. package/skills/shapeup/resources/context-compaction.md +5 -5
  68. package/skills/shapeup/resources/shaping.md +1 -1
  69. package/skills/spec-evaluator/SKILL.md +17 -21
  70. package/skills/spec-evaluator/references/dimension-contract.md +1 -1
  71. package/skills/spec-evaluator/references/dimensions/completeness.md +3 -3
  72. package/skills/spec-evaluator/references/dimensions/spec-conformance.md +1 -1
  73. package/skills/spec-evaluator/references/dimensions/test-surface-conformance.md +5 -5
  74. package/skills/task-executor/SKILL.md +11 -15
  75. package/commands/hill.md +0 -10
  76. package/hooks/gate-intake.mjs +0 -109
  77. package/hooks/gate-zerowork.mjs +0 -349
  78. package/hooks/tier-guard.mjs +0 -162
  79. package/kernel/compile.mjs +0 -1419
  80. package/kernel/init/fit.mjs +0 -199
  81. package/kernel/init/run-args.mjs +0 -206
  82. package/kernel/lib/breadboard.mjs +0 -165
  83. package/kernel/probe/attempts.mjs +0 -161
  84. package/kernel/probe/concurrency.mjs +0 -535
  85. package/kernel/probe/eval.mjs +0 -405
  86. package/kernel/probe/hunt.mjs +0 -104
  87. package/kernel/probe/leg.mjs +0 -229
  88. package/kernel/probe/owner.mjs +0 -142
  89. package/kernel/probe/requirements.mjs +0 -328
  90. package/kernel/probe/resume.mjs +0 -1110
  91. package/kernel/probe/rounds.mjs +0 -115
  92. package/kernel/probe/staged.mjs +0 -83
  93. package/kernel/probe/stats.mjs +0 -477
  94. package/kernel/probe/t0.mjs +0 -115
  95. package/kernel/reduce/board.mjs +0 -350
  96. package/kernel/reduce/graph.mjs +0 -482
  97. package/kernel/reduce/hill.mjs +0 -322
  98. package/kernel/reduce/ingest.mjs +0 -767
  99. package/kernel/reduce/leftovers.mjs +0 -159
  100. package/kernel/reduce/ship.mjs +0 -529
  101. package/kernel/reduce/snapshot.mjs +0 -291
  102. package/kernel/reduce/verdict.mjs +0 -172
  103. package/kernel/report/export.mjs +0 -384
  104. package/kernel/report/facts.mjs +0 -234
  105. package/kernel/verify/budget.mjs +0 -153
  106. package/kernel/verify/build.mjs +0 -428
  107. package/kernel/verify/dispatch.mjs +0 -114
  108. package/kernel/verify/spec.mjs +0 -975
  109. package/kernel/verify/trace.mjs +0 -604
  110. package/skills/ba-pitch-analyzer/SKILL.md +0 -204
  111. package/skills/coach/SKILL.md +0 -397
  112. package/skills/hill-chart/SKILL.md +0 -154
  113. package/skills/hill-chart/assets/dashboard.template.html +0 -1432
  114. package/skills/scope-architect/SKILL.md +0 -174
  115. package/skills/scope-hammer/SKILL.md +0 -231
  116. package/skills/solution-architect/SKILL.md +0 -158
  117. package/skills/tech-lead/SKILL.md +0 -151
  118. package/skills/tech-lead/references/gates.md +0 -605
  119. package/skills/tech-lead/references/protocol.md +0 -889
  120. package/skills/tech-lead/references/tiny-lane.md +0 -53
  121. package/skills/tech-lead/workflows/shapeup-run.js +0 -2187
  122. package/skills/translator/SKILL.md +0 -258
  123. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/contract-patterns.md +0 -0
  124. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ddd-patterns.md +0 -0
  125. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/integration-analysis.md +0 -0
  126. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ux-behavior-patterns.md +0 -0
  127. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/api-feasibility.tmpl.md +0 -0
  128. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/be-service.contract.tmpl.md +0 -0
  129. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/offline-storage.contract.tmpl.md +0 -0
  130. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/third-party-api.contract.tmpl.md +0 -0
  131. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/context-map.tmpl.md +0 -0
  132. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/event-choreography.tmpl.md +0 -0
  133. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/migration-plan.tmpl.md +0 -0
  134. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/team-handoff.tmpl.md +0 -0
  135. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/domain-model.tmpl.md +0 -0
  136. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/feedback.tmpl.md +0 -0
  137. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/integration.tmpl.md +0 -0
  138. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-board.tmpl.md +0 -0
  139. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-spike.tmpl.md +0 -0
  140. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task.tmpl.md +0 -0
  141. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/ux-behavior.tmpl.md +0 -0
@@ -1,405 +0,0 @@
1
- // probe eval — "what did round N's EVAL WorkResult actually say?"
2
- //
3
- // CONTRACT. A bounded, read-only query over the evaluate WorkResult ingest already wrote. Prints
4
- // `{ok, overall, bug_count, report_path, round, status, reason}` on stdout; exits 0 when the round
5
- // holds a verdict the run may act on, 1 when it does not — nothing ran, ingest hasn't landed, the
6
- // evaluator refused the round, or the verdict is structurally invalid; `reason` says which — and 2
7
- // on a bad argv. Writes nothing.
8
- //
9
- // WHY THIS EXISTS. `shapeup-run.js` cannot read a file itself (a Workflow script has no filesystem
10
- // of its own — see this repo's own note on why it may not call `Date.now()`), so every fact it
11
- // branches on has to cross an `agent()`/`query()` boundary. Before this command existed, the EVAL
12
- // round-loop branch (`verdict = e.overall === "PASS" ? "pass" : "fail"`) trusted the DISPATCHING
13
- // sub-agent's own end-of-turn summary — a value composed from memory/judgment after three other
14
- // steps (compile, dispatch, ingest), schema-checked for SHAPE only. Nothing re-verified that
15
- // summary against the WorkResult `reduce ingest` had just written to disk. Measured live
16
- // (2026-08-19, todo-cli): `results/evaluate-r1.json`'s `verdict.overall` was `"FAIL"` (2 cited
17
- // bugs, `spec-evaluator` correctly failing `SC-ERR`), and the run proceeded straight to QA and
18
- // GATE H anyway — the exact shape `verdict = e.overall === "PASS"` allows when `e.overall` is a
19
- // self-report that drifted from the artifact it was supposed to summarize. This command closes
20
- // that gap the same way `probe t0` already closes it for the T0 ratchet: a narrow, single-purpose
21
- // "transcribe this JSON verbatim" query over the artifact itself, not a multi-step summary.
22
- //
23
- // WHY IT READS `results/evaluate-r<N>.json` AND NOT THE `.md` REPORT. The WorkResult is the
24
- // machine-checked envelope `reduce ingest` validated against `EVAL`'s schema before ever touching
25
- // shared state; the `.md` report is prose for a human. Reading the prose to re-derive a verdict a
26
- // schema already carries structurally is the paraphrase channel this repo's hooks exist to close
27
- // everywhere else.
28
- //
29
- // WHY "NO VERDICT" CARRIES A REASON. An evaluator that refuses a round — a structural precondition
30
- // it cannot meet — still writes `evaluate-r<N>.json`, with `status: failed`, no verdict, and the
31
- // cause as its first deviation. A bare `ok: false` reached the operator as a sub-agent that died
32
- // after retries, while the one sentence naming the actual cause sat in a file nobody was pointed at.
33
-
34
- import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
35
- import { join, resolve, resolve as resolvePath, sep } from "node:path";
36
- import { createHash } from "node:crypto";
37
- import { runArgs } from "../lib/argv.mjs";
38
- import { resultsDir, ordersDir, scopesDir, readRunId, verdictsDir, specDir } from "../lib/paths.mjs";
39
- import { readBoard } from "../compile.mjs";
40
- import { coveringAcs } from "./requirements.mjs";
41
-
42
- /** Longest `reason` reported. A deviation is prose written by a worker and can run to paragraphs. */
43
- const REASON_MAX = 400;
44
-
45
- /**
46
- * Bound a reason to {@link REASON_MAX} characters.
47
- * @param {string} s - The reason.
48
- * @returns {string} `s`, or its first REASON_MAX − 1 characters and an ellipsis.
49
- */
50
- const clip = (s) => (s.length > REASON_MAX ? `${s.slice(0, REASON_MAX - 1)}…` : s);
51
-
52
- /**
53
- * Whether a feature's spec is SCOPED — has scope contracts, the case in which every verdict must
54
- * cite the T0 artifacts it re-hashed.
55
- *
56
- * @param {string} cwd - Project root.
57
- * @param {string} slug - Feature slug.
58
- * @returns {boolean} True when `scopes/` holds at least one contract (`.md`, or a legacy `.json`).
59
- */
60
- export function isScoped(cwd, slug) {
61
- try { return readdirSync(scopesDir(cwd, slug)).some((f) => /\.(md|json)$/.test(f)); }
62
- catch { return false; }
63
- }
64
-
65
- /**
66
- * Why one T0 citation does not resolve, or null when the kernel can positively confirm it does.
67
- *
68
- * Re-checks the two facts a citation can lie about and the schema promises are enforced
69
- * (`T0Citation`: "the evaluator RECOMPUTES sha256 from disk — a handed hash is never trusted"):
70
- * that the bytes at `path` hash to the cited `sha256`, and that what those bytes actually say is a
71
- * green T0 verdict — a PASS/FAIL cannot ride on an artifact that itself recorded red. NOT checked:
72
- * whether `path` is one of the order's own `payload.t0_artifacts` (see the note on
73
- * {@link citationProblem} for why that is left to the operator rather than enforced here).
74
- *
75
- * FAILS OPEN ON "CANNOT TELL", CLOSED ON "PROVEN WRONG" — and the two are not the same fact. A read
76
- * that fails for a reason that says something DEFINITE about what is (or isn't) at `path` is proof,
77
- * same as a hash mismatch: `ENOENT` (nothing there) and `EISDIR` (a directory, never a file — T0
78
- * verdicts are always files, per `writeArtifact`) both mean no such artifact was ever produced, so
79
- * both refuse. `verify t0` writes verdict artifacts immutably and never deletes one
80
- * (`writeArtifact`'s `wx` flag — see kernel/verify/t0.mjs), so that absence or shape mismatch is a
81
- * positive fact about the citation, not a guess. Anything else a read can fail with — permission
82
- * denied, a symlink loop, a transient I/O error — says nothing about the citation's honesty, only
83
- * that THIS MACHINE could not check it just now, so it is not refused on that ground alone: the
84
- * fail-open discipline this repo's guards already use elsewhere for an unproven bad state.
85
- *
86
- * @param {string} cwd - Project root.
87
- * @param {object} citation - One `T0Citation` (`scope_id`, `path`, `sha256`). Read defensively:
88
- * `reduce ingest` only reaches this after the enclosing WorkResult passed schema validation, but
89
- * `probe eval` reaches it over a result file it merely `JSON.parse`s, so a malformed citation must
90
- * fail this check rather than throw.
91
- * @returns {(string|null)} A reason phrased for an operator, or null when the file at `path` exists,
92
- * hashes to the cited `sha256`, and its own `overall` reads "green".
93
- */
94
- function unresolvedCitation(cwd, citation, { round = null, runId = null, verdicts = null } = {}) {
95
- const rel = typeof citation?.path === "string" ? citation.path : "";
96
- if (!rel) return "names no artifact path";
97
- let text;
98
- try {
99
- text = readFileSync(resolve(cwd, rel), "utf8");
100
- } catch (e) {
101
- if (e.code === "ENOENT") return `cites ${rel}, which does not exist on disk`;
102
- if (e.code === "EISDIR") return `cites ${rel}, which is a directory, not a T0 verdict file`;
103
- // EACCES, ELOOP, EIO, EMFILE… — this machine failing to look, not evidence against the
104
- // citation, so it is not refused on that ground.
105
- return null;
106
- }
107
- const actual = createHash("sha256").update(text).digest("hex");
108
- const claimed = typeof citation.sha256 === "string" ? citation.sha256.toLowerCase() : "";
109
- if (actual !== claimed) return `cites ${rel} with sha256 ${citation.sha256}, but the file on disk hashes to ${actual}`;
110
- let body;
111
- try { body = JSON.parse(text); }
112
- catch { return `cites ${rel}, whose bytes match the hash but do not read as a T0 verdict`; }
113
- if (body?.overall !== "green") return `cites ${rel}, whose own verdict is "${body?.overall ?? "unknown"}", not green`;
114
- // INSIDE THIS RUN'S OWN VERDICTS DIRECTORY, resolved — asked of a file that exists, so "does not
115
- // exist" and "resolves somewhere else" stay different answers. Accepted before this check: a
116
- // green artifact in the source tree, one outside the project reached by `../`, one by absolute
117
- // path, and a symlink in the verdicts directory pointing at a forged file. Every one hashed
118
- // correctly, because a digest says the bytes are the file's and nothing about which file it
119
- // should have been.
120
- if (verdicts) {
121
- const real = (() => { try { return realpathSync.native(resolvePath(cwd, rel)); } catch { return resolvePath(cwd, rel); } })();
122
- const home = (() => { try { return realpathSync.native(verdicts); } catch { return verdicts; } })();
123
- if (!real.startsWith(home.endsWith(sep) ? home : home + sep)) {
124
- return `cites ${rel}, which resolves outside this run's own verdicts directory (${home}) — a verdict cites what this run's own verifier wrote, not a file the judge can reach`;
125
- }
126
- }
127
- // THE ARTIFACT HAS TO BE THE ONE THE CITATION SAYS IT IS. A re-hash proves the bytes are the
128
- // file's; it says nothing about whose verdict the file holds. A PASS citing scope alpha's green
129
- // artifact while declaring scope beta, or a prior round's, or a prior run's over the same slug,
130
- // passed the digest check unremarked. The citation's own required `scope_id`, the round being
131
- // judged and the run's key are compared to what the artifact records about itself.
132
- if (typeof citation.scope_id === "string" && body?.scope_id && body.scope_id !== citation.scope_id) {
133
- return `cites ${rel} for scope "${citation.scope_id}", but the artifact records scope "${body.scope_id}"`;
134
- }
135
- if (round != null && typeof body?.round === "number" && body.round !== round) {
136
- return `cites ${rel}, a round ${body.round} artifact, as evidence for round ${round}`;
137
- }
138
- if (runId && body?.run_id && body.run_id !== runId) {
139
- return `cites ${rel}, an artifact of run ${body.run_id}, as evidence for run ${runId}`;
140
- }
141
- return null;
142
- }
143
-
144
- /**
145
- * Why a verdict cannot stand as its round's judgement on T0 grounds, or null when it can.
146
- *
147
- * A PASS or FAIL on a scoped spec that cites no T0 artifact is structurally invalid — the
148
- * evaluator's own contract says so, because T0 is the machine fact a generator cannot fabricate.
149
- * That rule used to live only in the contract, so a verdict citing nothing was ingested, ledgered
150
- * and branched on like any other. It is checked here so the round loop, the resume derivation, the
151
- * hill and ingest all refuse the same verdict for the same reason.
152
- *
153
- * RESOLVES, NOT JUST PRESENT. A citation naming a path and a hash used to be taken on faith: any
154
- * non-empty `t0_citations[]` passed, whatever it pointed at. Measured live: a PASS citing a path
155
- * that does not exist on disk, and a PASS citing a real artifact whose own verdict was red, both
156
- * ingested clean — presence stood in for a re-hash the schema had already promised. Each citation is
157
- * now re-checked through {@link unresolvedCitation}.
158
- *
159
- * WHAT THIS DOES NOT CHECK: whether a citation is drawn from the order's own `payload.t0_artifacts`
160
- * list. That would need the compiled order, which this function's callers do not equally have —
161
- * `reduce ingest` holds it, but `probe eval` (and the round-loop/resume/hill readers behind it)
162
- * knows only (cwd, slug, round), and a scope's T0 attempt can legitimately go green again LATER than
163
- * whatever list was frozen at compile time (`greenVerdict` already treats "newest green" as
164
- * authoritative for exactly this reason — see kernel/probe/t0.mjs). Enforcing membership only where
165
- * the order happens to be on hand would let one channel refuse a citation the other accepts, for
166
- * evidence that may simply be fresher than the order — worse than leaving it unenforced.
167
- *
168
- * @param {string} cwd - Project root.
169
- * @param {string} slug - Feature slug.
170
- * @param {object} verdict - The WorkResult's `verdict` block.
171
- * @returns {(string|null)} The problem, phrased for an operator; null for a verdict whose every
172
- * citation resolves, an unscoped spec, or a block with no PASS/FAIL in it (there is no judgement
173
- * to invalidate).
174
- */
175
- /**
176
- * Why a verdict cannot stand on its own criteria, or null when it can.
177
- *
178
- * `overall` is the judge's field, and nothing recomputed it from the criteria the judge graded: a
179
- * PASS over a failing criterion, or over no criterion at all, validated and ingested, and the
180
- * round loop branched on it. The evaluator's own first rule is that absence of evidence is a FAIL,
181
- * so a PASS criterion with no evidence is no evidence either. Recomputed here, on ingest and on
182
- * read alike: PASS means every graded criterion passed with evidence and at least one was graded;
183
- * FAIL means at least one graded criterion failed.
184
- *
185
- * @param {object} verdict - The WorkResult's `verdict`.
186
- * @returns {(string|null)} A reason phrased for an operator, or null.
187
- */
188
- export function verdictProblem(verdict) {
189
- const overall = verdict?.overall;
190
- if (overall !== "PASS" && overall !== "FAIL") return null;
191
- const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
192
- const fails = criteria.filter((c) => c?.verdict === "FAIL");
193
- const passes = criteria.filter((c) => c?.verdict === "PASS");
194
- const other = criteria.length - fails.length - passes.length;
195
- if (overall === "PASS") {
196
- if (criteria.length === 0) return "the PASS verdict grades no criterion at all — a PASS with no evidence is a claim";
197
- if (fails.length) return `the verdict says PASS while ${fails.length} of its ${criteria.length} criteria read FAIL — overall is derived from the criteria, never declared over them`;
198
- if (other) return `the verdict says PASS while ${other} of its criteria carry no PASS/FAIL verdict`;
199
- const bare = passes.filter((c) => !(typeof c?.evidence === "string" && c.evidence.trim()));
200
- if (bare.length) return `the PASS verdict has ${bare.length} criterion(s) marked PASS with no evidence — absence of evidence is a FAIL by the evaluator's own first rule`;
201
- return null;
202
- }
203
- if (criteria.length === 0) return "the FAIL verdict grades no criterion at all — a FAIL must name what failed";
204
- if (!fails.length) return `the verdict says FAIL while every one of its ${criteria.length} graded criteria reads PASS — a FAIL must cite the criterion it failed`;
205
- return null;
206
- }
207
-
208
- /**
209
- * The Test Surface row ids the spec declares, in spec order.
210
- *
211
- * @param {string} cwd - Project root.
212
- * @param {string} slug - Feature slug.
213
- * @returns {string[]} Every `TS-…` id that opens a table row in a use-case file; [] with no spec tree.
214
- */
215
- export function surfaceRows(cwd, slug) {
216
- const dir = join(specDir(cwd, slug), "usecases");
217
- let files = [];
218
- try { files = readdirSync(dir).filter((f) => /^UC-.*\.md$/.test(f)).sort(); } catch { return []; }
219
- const rows = [];
220
- for (const f of files) {
221
- let text = "";
222
- try { text = readFileSync(join(dir, f), "utf8"); } catch { continue; }
223
- for (const m of text.matchAll(/^\|\s*(TS-[A-Za-z0-9_-]+)\s*\|/gm)) if (!rows.includes(m[1])) rows.push(m[1]);
224
- }
225
- return rows;
226
- }
227
-
228
- /**
229
- * Whether a PASS grades every Test Surface row by name.
230
- *
231
- * A PASS that grades the surface as a whole — "device rows", "local rows" — validated like any other
232
- * and switched off everything that reads a verdict row by row: the requirements matrix read no
233
- * evidence under every covered requirement, a failed row had no criterion to become a bug, and a row
234
- * nobody looked at passed with the rest. A row counts as graded when its id appears in a criterion's
235
- * text or its `traces_to`. Only a PASS is held to it: a FAIL sends the round back either way.
236
- *
237
- * @param {string} cwd - Project root.
238
- * @param {string} slug - Feature slug.
239
- * @param {object} verdict - The WorkResult's `verdict`.
240
- * @returns {(string|null)} A reason naming the rows no criterion graded, or null.
241
- */
242
- export function coverageProblem(cwd, slug, verdict) {
243
- if (verdict?.overall !== "PASS") return null;
244
- const rows = surfaceRows(cwd, slug);
245
- if (!rows.length) return null;
246
- const named = new Set();
247
- for (const c of Array.isArray(verdict.criteria) ? verdict.criteria : []) {
248
- const text = [c?.criterion, ...(Array.isArray(c?.traces_to) ? c.traces_to : [])].map((x) => String(x ?? "")).join(" ");
249
- for (const m of text.matchAll(/\bTS-[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+)*/g)) named.add(m[0]);
250
- }
251
- const missing = rows.filter((r) => !named.has(r));
252
- if (!missing.length) return null;
253
- const shown = missing.slice(0, 8).join(", ") + (missing.length > 8 ? ` and ${missing.length - 8} more` : "");
254
- return `the PASS verdict grades ${rows.length - missing.length} of ${rows.length} Test Surface rows by name — ` +
255
- `each row is its own criterion, its id in the criterion or its traces_to; ungraded: ${shown}`;
256
- }
257
-
258
- /**
259
- * Whether a verdict anchors any criterion to a requirement when the board says which ones it could.
260
- *
261
- * `traces_to` is copied from the `(covers: REQ-…)` clauses of the acceptance criteria a criterion
262
- * grades, and the requirements matrix is joined along it. Two judges on the same spec and the same
263
- * instruction differed: one anchored every row and the matrix read 5/5, the next anchored none and it
264
- * read 0/5 over the same passes. Held only to the all-empty case: which requirement a criterion
265
- * traces to is the judge's reading, but none at all, beside a board that covers some, is a copy
266
- * step skipped.
267
- *
268
- * @param {string} cwd - Project root.
269
- * @param {string} slug - Feature slug.
270
- * @param {object} verdict - The WorkResult's `verdict`.
271
- * @returns {(string|null)} A reason phrased for the judge, or null.
272
- */
273
- export function tracesProblem(cwd, slug, verdict) {
274
- if (verdict?.overall !== "PASS" && verdict?.overall !== "FAIL") return null;
275
- const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
276
- if (!criteria.length) return null;
277
- if (criteria.some((c) => Array.isArray(c?.traces_to) && c.traces_to.length)) return null;
278
- let covered = 0;
279
- try { covered = coveringAcs(readBoard(cwd, slug)).size; } catch { return null; }
280
- if (!covered) return null;
281
- return `the verdict anchors none of its ${criteria.length} criteria to a requirement while the board's acceptance ` +
282
- `criteria cover ${covered} — copy each graded AC's (covers: REQ-…) clause into that criterion's traces_to`;
283
- }
284
-
285
- /**
286
- * A PASS that grades nothing under a dimension its order named.
287
- *
288
- * The order names the dimensions the judge grades, and GATE L4 prints them back as what "PASS"
289
- * covers. Measured: an order named four, the PASS graded criteria under three, and L4 still listed
290
- * the fourth as evaluated. A PASS must grade at least one criterion under every dimension its order
291
- * named; a FAIL already stops the ship, so it is not held to this.
292
- *
293
- * @param {string} cwd - Project root.
294
- * @param {string} slug - Feature slug.
295
- * @param {object} verdict - The WorkResult's `verdict`.
296
- * @param {{round:(number|null)}} [opts] - The EVAL round whose order names the set; null → no check.
297
- * @returns {(string|null)} Why the verdict is refused, or null.
298
- */
299
- export function dimensionsProblem(cwd, slug, verdict, { round = null } = {}) {
300
- if (verdict?.overall !== "PASS" || round == null) return null;
301
- const named = orderDimensions(cwd, slug, round);
302
- if (!named?.length) return null;
303
- const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
304
- const graded = new Set(criteria.map((c) => c?.dimension).filter(Boolean));
305
- const missing = named.filter((d) => !graded.has(d));
306
- if (!missing.length) return null;
307
- return `the PASS grades no criterion under ${missing.join(", ")}, which the order names — grade each named ` +
308
- "dimension's criteria, or FAIL the one you cannot grade with the reason";
309
- }
310
-
311
- export function citationProblem(cwd, slug, verdict, { round = null } = {}) {
312
- if (verdict?.overall !== "PASS" && verdict?.overall !== "FAIL") return null;
313
- if (!isScoped(cwd, slug)) return null;
314
- if (!Array.isArray(verdict.t0_citations) || !verdict.t0_citations.length) {
315
- return `the ${verdict.overall} verdict cites no T0 artifact, and a verdict on a scoped spec must ` +
316
- "cite the T0 verdict it re-hashed (the order lists them under payload.t0_artifacts)";
317
- }
318
- const runId = readRunId(cwd, slug);
319
- const verdicts = verdictsDir(cwd, slug);
320
- for (const citation of verdict.t0_citations) {
321
- const reason = unresolvedCitation(cwd, citation, { round, runId, verdicts });
322
- if (reason) return `the ${verdict.overall} verdict ${reason} — a T0 citation is re-hashed from disk, never taken on the handed word`;
323
- }
324
- return null;
325
- }
326
-
327
- /**
328
- * Read one round's EVAL verdict straight from the WorkResult `reduce ingest` wrote.
329
- *
330
- * @param {string} cwd - Project root.
331
- * @param {string} slug - Feature slug.
332
- * @param {number} round - The EVAL round (`evaluate-r<N>.json`).
333
- * @returns {{found: boolean, overall: (string|null), status: (string|null), reason: (string|null),
334
- * bug_count: (number|null), report_path: (string|null)}} `found` is true only for a PASS/FAIL the
335
- * round may act on; otherwise `reason` says why not — a fact, not a guess.
336
- */
337
- export function evalVerdict(cwd, slug, round) {
338
- const path = join(resultsDir(cwd, slug), `evaluate-r${round}.json`);
339
- const unfit = (reason, status = null, overall = null) =>
340
- ({ found: false, overall, status, reason, bug_count: null, report_path: null });
341
- if (!existsSync(path)) return unfit("no evaluate result for this round yet");
342
- let doc;
343
- try { doc = JSON.parse(readFileSync(path, "utf8")); }
344
- catch { return unfit("the evaluate result is not readable JSON"); }
345
- const status = typeof doc?.status === "string" ? doc.status : null;
346
- const v = doc?.verdict || {};
347
- const overall = v.overall === "PASS" || v.overall === "FAIL" ? v.overall : null;
348
- if (!overall) {
349
- // A worker that refused to grade says why in its FIRST deviation — the only channel it has.
350
- const first = Array.isArray(doc?.deviations) && typeof doc.deviations[0] === "string" ? doc.deviations[0] : "";
351
- return unfit(clip(first
352
- ? `the evaluator returned ${status || "no status"}: ${first}`
353
- : `status ${status || "unknown"} with no PASS/FAIL verdict`), status);
354
- }
355
- const problem = verdictProblem(v) || coverageProblem(cwd, slug, v) || tracesProblem(cwd, slug, v)
356
- || dimensionsProblem(cwd, slug, v, { round }) || citationProblem(cwd, slug, v, { round });
357
- if (problem) return unfit(problem, status, overall);
358
- return {
359
- found: true,
360
- overall,
361
- status,
362
- reason: null,
363
- bug_count: Array.isArray(v.bugs) ? v.bugs.length : null,
364
- report_path: typeof v.report_path === "string" ? v.report_path : null,
365
- };
366
- }
367
-
368
- /**
369
- * The dimensions round N's evaluate order named — the set the judge was asked to grade.
370
- *
371
- * @param {string} cwd - Project root.
372
- * @param {string} slug - Feature slug.
373
- * @param {number} round - The EVAL round (`evaluate-r<N>.json`).
374
- * @returns {(string[]|null)} The order's `payload.dimensions`; null when the order or the list is absent.
375
- */
376
- export function orderDimensions(cwd, slug, round) {
377
- try {
378
- const o = JSON.parse(readFileSync(join(ordersDir(cwd, slug), `evaluate-r${round}.json`), "utf8"));
379
- const d = o?.payload?.dimensions;
380
- return Array.isArray(d) && d.every((x) => typeof x === "string") ? d : null;
381
- } catch { return null; }
382
- }
383
-
384
- export const ARGV_SPEC = {
385
- usage: "harness.mjs probe eval --slug <slug> --round N [--cwd <dir>]",
386
- _: { arity: 0, max: 0, name: "(no positional operands)" },
387
- slug: { type: "str", required: true },
388
- round: { type: "int", min: 1, required: true },
389
- cwd: { type: "path" },
390
- };
391
-
392
- /**
393
- * Report round N's EVAL verdict, mechanically, from the WorkResult on disk.
394
- *
395
- * @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
396
- * @returns {void} Exits 0 when the round holds a verdict the run may act on, 1 when it does not.
397
- */
398
- export function cli(rawArgv) {
399
- const args = runArgs(ARGV_SPEC, rawArgv);
400
- const cwd = resolve(args.cwd || process.cwd());
401
- const { found, overall, status, reason, bug_count, report_path } = evalVerdict(cwd, args.slug, args.round);
402
- const dimensions = orderDimensions(cwd, args.slug, args.round);
403
- console.log(JSON.stringify({ ok: found, overall, bug_count, report_path, round: args.round, status, reason, dimensions }));
404
- process.exit(found ? 0 : 1);
405
- }
@@ -1,104 +0,0 @@
1
- #!/usr/bin/env node
2
- // probe hunt — "did the QA hunt hunt anything?"
3
- //
4
- // CONTRACT. A bounded, read-only query over the hunt's WorkResult, its order and its report.
5
- // Prints `{ok, status, qa, charters_run, findings, reason}` on stdout; exits 0 when the hunt's
6
- // result is one the run may accept, 1 when it is not, and 2 on a bad argv. Writes nothing.
7
- //
8
- // WHY THIS EXISTS. A hunter that reached the app, swept the fault log and then drafted no charter
9
- // still returned `done`, and a `done` with no findings reads as an app with nothing wrong. The hunt
10
- // report's charter count says what actually happened. A `done` hunt over an app its order could
11
- // reach, with no charter run, is refused here and on ingest, and the run sends the hunter back once
12
- // — the same shape as a verdict that grades nothing by name.
13
-
14
- import { existsSync, readFileSync } from "node:fs";
15
- import { join, resolve } from "node:path";
16
- import { runArgs } from "../lib/argv.mjs";
17
- import { resultsDir, ordersDir, qaDir } from "../lib/paths.mjs";
18
-
19
- /**
20
- * The number of charters a hunt report says it ran.
21
- *
22
- * @param {(string|null)} text - The hunt report's text.
23
- * @returns {(number|null)} The run count from its `charters: <run>/<approved>` line, or null when
24
- * there is no report or no such line.
25
- */
26
- export function chartersRun(text) {
27
- const m = String(text ?? "").match(/^charters:\s*(\d+)/m);
28
- return m ? Number(m[1]) : null;
29
- }
30
-
31
- /**
32
- * Whether a hunt result may be accepted as it stands.
33
- *
34
- * Held to it only when the result says `done` and the order gave the hunter a way into the app
35
- * (`launch_cmd` or `app_url`): a hunt that returns `failed` says it could not hunt, which is honest,
36
- * and a hunt with no way in cannot be asked to drive anything.
37
- *
38
- * @param {object} result - The hunt WorkResult.
39
- * @param {object} payload - The hunt order's payload.
40
- * @param {(string|null)} report - The hunt report's text, or null.
41
- * @returns {(string|null)} A reason phrased for the hunter, or null.
42
- */
43
- export function huntProblem(result, payload, report) {
44
- if (result?.status !== "done") return null;
45
- if (!payload?.launch_cmd && !payload?.app_url) return null;
46
- const n = chartersRun(report);
47
- if (n === null) return "the hunt returned done with no hunt report carrying a `charters:` line";
48
- if (n > 0) return null;
49
- return "the hunt returned done over an app its order could reach, having run 0 charters — " +
50
- "draft at least one charter where the EVAL left territory uncovered and hunt it, or return failed saying why the app could not be driven";
51
- }
52
-
53
- /**
54
- * Read a JSON file, or null.
55
- * @param {string} p - Path.
56
- * @returns {(object|null)} The parsed document, or null when absent or unreadable.
57
- */
58
- const readJson = (p) => { try { return JSON.parse(readFileSync(p, "utf8")); } catch { return null; } };
59
-
60
- /**
61
- * What the hunt on disk amounts to.
62
- *
63
- * @param {string} cwd - Project root.
64
- * @param {string} slug - Feature slug.
65
- * @returns {{ok:boolean, status:(string|null), qa:string, charters_run:(number|null), findings:number, reason:(string|null)}}
66
- * `qa` is `run` (charters ran), `not-hunted` (the hunt returned, no charter ran), or `missing`.
67
- */
68
- export function huntOutcome(cwd, slug) {
69
- const result = readJson(join(resultsDir(cwd, slug), "hunt.json"));
70
- if (!result) return { ok: false, status: null, qa: "missing", charters_run: null, findings: 0, reason: "no hunt result on disk" };
71
- const payload = readJson(join(ordersDir(cwd, slug), "hunt.json"))?.payload || {};
72
- const reportPath = join(qaDir(cwd, slug), "hunt-report.md");
73
- const report = existsSync(reportPath) ? readFileSync(reportPath, "utf8") : null;
74
- const n = chartersRun(report);
75
- const reason = huntProblem(result, payload, report);
76
- return {
77
- ok: !reason,
78
- status: typeof result.status === "string" ? result.status : null,
79
- qa: n ? "run" : "not-hunted",
80
- charters_run: n,
81
- findings: Array.isArray(result.discoveries) ? result.discoveries.length : 0,
82
- reason,
83
- };
84
- }
85
-
86
- export const ARGV_SPEC = {
87
- usage: "harness.mjs probe hunt --slug <slug> [--cwd <dir>]",
88
- _: { arity: 0, max: 0, name: "(no positional operands)" },
89
- slug: { type: "str", required: true },
90
- cwd: { type: "path" },
91
- };
92
-
93
- /**
94
- * Report what the hunt on disk amounts to.
95
- *
96
- * @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
97
- * @returns {void} Exits 0 when the hunt result may be accepted, 1 when it may not.
98
- */
99
- export function cli(rawArgv) {
100
- const args = runArgs(ARGV_SPEC, rawArgv);
101
- const out = huntOutcome(resolve(args.cwd || process.cwd()), args.slug);
102
- console.log(JSON.stringify(out));
103
- process.exit(out.ok ? 0 : 1);
104
- }