shapeup-sdlc 3.18.3 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.example.json +2 -3
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +115 -56
- package/README.md +101 -143
- package/SECURITY.md +16 -25
- package/agents/orient.md +14 -0
- package/agents/qa-edge-hunter.md +13 -0
- package/agents/spec-evaluator.md +14 -0
- package/agents/task-executor.md +14 -0
- package/bin/init.mjs +13 -16
- package/bin/lib/grant.mjs +8 -19
- package/commands/build.md +9 -14
- package/commands/eval.md +3 -3
- package/commands/hammer.md +9 -5
- package/commands/retro.md +12 -11
- package/commands/scopes.md +1 -12
- package/commands/ship.md +45 -88
- package/commands/wire.md +1 -8
- package/hooks/dispatch-receipt.mjs +94 -8
- package/hooks/hooks.json +0 -41
- package/hooks/lib/decision.mjs +6 -9
- package/hooks/lib/tier.mjs +50 -0
- package/hooks/sandbox-guard.mjs +140 -64
- package/kernel/gate.mjs +203 -22
- package/kernel/harness.mjs +20 -67
- package/kernel/init/run.mjs +175 -523
- package/kernel/lib/contract.mjs +9 -93
- package/kernel/lib/floor.mjs +64 -0
- package/kernel/lib/paths.mjs +39 -112
- package/kernel/order.mjs +157 -0
- package/kernel/probe/digest.mjs +2 -38
- package/kernel/schemas/domain.schema.json +146 -1125
- package/kernel/schemas/work-order.schema.json +8 -4
- package/kernel/schemas/work-result.schema.json +1 -1
- package/kernel/verify/envelope.mjs +2 -2
- package/kernel/verify/skills.mjs +27 -8
- package/kernel/verify/t0.mjs +2 -2
- package/package.json +4 -2
- package/skills/building/SKILL.md +144 -0
- package/skills/building/resources/gates.md +76 -0
- package/skills/building/resources/guidance.md +23 -0
- package/skills/{translator/references/preservation-rules.md → building/resources/intake-preservation.md} +1 -1
- package/skills/building/resources/intake.md +157 -0
- package/skills/building/resources/orders.md +69 -0
- package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/doc-schemas.md +1 -1
- package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/task-generation.md +13 -13
- package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/test-surface.md +6 -7
- package/skills/building/resources/plan-scopes.md +95 -0
- package/skills/building/resources/plan-spec.md +102 -0
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/_index.tmpl.md +1 -1
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/assess-report.tmpl.md +10 -11
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/run-state.tmpl.md +4 -4
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/scope-summary.tmpl.md +2 -2
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/synthesis.tmpl.md +4 -4
- package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/usecase.tmpl.md +2 -2
- package/skills/building/resources/plan-wire.md +82 -0
- package/skills/building/resources/plan.md +98 -0
- package/skills/building/resources/retro-research.md +60 -0
- package/skills/building/resources/retro-scan.md +49 -0
- package/skills/building/resources/retro.md +225 -0
- package/skills/building/resources/rounds.md +78 -0
- package/skills/building/resources/stop.md +127 -0
- package/skills/orient/SKILL.md +37 -37
- package/skills/qa-edge-hunter/SKILL.md +14 -14
- package/skills/shapeup/SKILL.md +5 -5
- package/skills/shapeup/resources/breadboarding.md +1 -1
- package/skills/shapeup/resources/context-compaction.md +5 -5
- package/skills/shapeup/resources/shaping.md +1 -1
- package/skills/spec-evaluator/SKILL.md +17 -21
- package/skills/spec-evaluator/references/dimension-contract.md +1 -1
- package/skills/spec-evaluator/references/dimensions/completeness.md +3 -3
- package/skills/spec-evaluator/references/dimensions/spec-conformance.md +1 -1
- package/skills/spec-evaluator/references/dimensions/test-surface-conformance.md +5 -5
- package/skills/task-executor/SKILL.md +11 -15
- package/commands/hill.md +0 -10
- package/hooks/gate-intake.mjs +0 -109
- package/hooks/gate-zerowork.mjs +0 -349
- package/hooks/tier-guard.mjs +0 -162
- package/kernel/compile.mjs +0 -1419
- package/kernel/init/fit.mjs +0 -199
- package/kernel/init/run-args.mjs +0 -206
- package/kernel/lib/breadboard.mjs +0 -165
- package/kernel/probe/attempts.mjs +0 -161
- package/kernel/probe/concurrency.mjs +0 -535
- package/kernel/probe/eval.mjs +0 -405
- package/kernel/probe/hunt.mjs +0 -104
- package/kernel/probe/leg.mjs +0 -229
- package/kernel/probe/owner.mjs +0 -142
- package/kernel/probe/requirements.mjs +0 -328
- package/kernel/probe/resume.mjs +0 -1110
- package/kernel/probe/rounds.mjs +0 -115
- package/kernel/probe/staged.mjs +0 -83
- package/kernel/probe/stats.mjs +0 -477
- package/kernel/probe/t0.mjs +0 -115
- package/kernel/reduce/board.mjs +0 -350
- package/kernel/reduce/graph.mjs +0 -482
- package/kernel/reduce/hill.mjs +0 -322
- package/kernel/reduce/ingest.mjs +0 -767
- package/kernel/reduce/leftovers.mjs +0 -159
- package/kernel/reduce/ship.mjs +0 -529
- package/kernel/reduce/snapshot.mjs +0 -291
- package/kernel/reduce/verdict.mjs +0 -172
- package/kernel/report/export.mjs +0 -384
- package/kernel/report/facts.mjs +0 -234
- package/kernel/verify/budget.mjs +0 -153
- package/kernel/verify/build.mjs +0 -428
- package/kernel/verify/dispatch.mjs +0 -114
- package/kernel/verify/spec.mjs +0 -975
- package/kernel/verify/trace.mjs +0 -604
- package/skills/ba-pitch-analyzer/SKILL.md +0 -204
- package/skills/coach/SKILL.md +0 -397
- package/skills/hill-chart/SKILL.md +0 -154
- package/skills/hill-chart/assets/dashboard.template.html +0 -1432
- package/skills/scope-architect/SKILL.md +0 -174
- package/skills/scope-hammer/SKILL.md +0 -231
- package/skills/solution-architect/SKILL.md +0 -158
- package/skills/tech-lead/SKILL.md +0 -151
- package/skills/tech-lead/references/gates.md +0 -605
- package/skills/tech-lead/references/protocol.md +0 -889
- package/skills/tech-lead/references/tiny-lane.md +0 -53
- package/skills/tech-lead/workflows/shapeup-run.js +0 -2187
- package/skills/translator/SKILL.md +0 -258
- /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/contract-patterns.md +0 -0
- /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ddd-patterns.md +0 -0
- /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/integration-analysis.md +0 -0
- /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ux-behavior-patterns.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/api-feasibility.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/be-service.contract.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/offline-storage.contract.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/third-party-api.contract.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/context-map.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/event-choreography.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/migration-plan.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/team-handoff.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/domain-model.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/feedback.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/integration.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-board.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-spike.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task.tmpl.md +0 -0
- /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/ux-behavior.tmpl.md +0 -0
package/kernel/probe/eval.mjs
DELETED
|
@@ -1,405 +0,0 @@
|
|
|
1
|
-
// probe eval — "what did round N's EVAL WorkResult actually say?"
|
|
2
|
-
//
|
|
3
|
-
// CONTRACT. A bounded, read-only query over the evaluate WorkResult ingest already wrote. Prints
|
|
4
|
-
// `{ok, overall, bug_count, report_path, round, status, reason}` on stdout; exits 0 when the round
|
|
5
|
-
// holds a verdict the run may act on, 1 when it does not — nothing ran, ingest hasn't landed, the
|
|
6
|
-
// evaluator refused the round, or the verdict is structurally invalid; `reason` says which — and 2
|
|
7
|
-
// on a bad argv. Writes nothing.
|
|
8
|
-
//
|
|
9
|
-
// WHY THIS EXISTS. `shapeup-run.js` cannot read a file itself (a Workflow script has no filesystem
|
|
10
|
-
// of its own — see this repo's own note on why it may not call `Date.now()`), so every fact it
|
|
11
|
-
// branches on has to cross an `agent()`/`query()` boundary. Before this command existed, the EVAL
|
|
12
|
-
// round-loop branch (`verdict = e.overall === "PASS" ? "pass" : "fail"`) trusted the DISPATCHING
|
|
13
|
-
// sub-agent's own end-of-turn summary — a value composed from memory/judgment after three other
|
|
14
|
-
// steps (compile, dispatch, ingest), schema-checked for SHAPE only. Nothing re-verified that
|
|
15
|
-
// summary against the WorkResult `reduce ingest` had just written to disk. Measured live
|
|
16
|
-
// (2026-08-19, todo-cli): `results/evaluate-r1.json`'s `verdict.overall` was `"FAIL"` (2 cited
|
|
17
|
-
// bugs, `spec-evaluator` correctly failing `SC-ERR`), and the run proceeded straight to QA and
|
|
18
|
-
// GATE H anyway — the exact shape `verdict = e.overall === "PASS"` allows when `e.overall` is a
|
|
19
|
-
// self-report that drifted from the artifact it was supposed to summarize. This command closes
|
|
20
|
-
// that gap the same way `probe t0` already closes it for the T0 ratchet: a narrow, single-purpose
|
|
21
|
-
// "transcribe this JSON verbatim" query over the artifact itself, not a multi-step summary.
|
|
22
|
-
//
|
|
23
|
-
// WHY IT READS `results/evaluate-r<N>.json` AND NOT THE `.md` REPORT. The WorkResult is the
|
|
24
|
-
// machine-checked envelope `reduce ingest` validated against `EVAL`'s schema before ever touching
|
|
25
|
-
// shared state; the `.md` report is prose for a human. Reading the prose to re-derive a verdict a
|
|
26
|
-
// schema already carries structurally is the paraphrase channel this repo's hooks exist to close
|
|
27
|
-
// everywhere else.
|
|
28
|
-
//
|
|
29
|
-
// WHY "NO VERDICT" CARRIES A REASON. An evaluator that refuses a round — a structural precondition
|
|
30
|
-
// it cannot meet — still writes `evaluate-r<N>.json`, with `status: failed`, no verdict, and the
|
|
31
|
-
// cause as its first deviation. A bare `ok: false` reached the operator as a sub-agent that died
|
|
32
|
-
// after retries, while the one sentence naming the actual cause sat in a file nobody was pointed at.
|
|
33
|
-
|
|
34
|
-
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
|
|
35
|
-
import { join, resolve, resolve as resolvePath, sep } from "node:path";
|
|
36
|
-
import { createHash } from "node:crypto";
|
|
37
|
-
import { runArgs } from "../lib/argv.mjs";
|
|
38
|
-
import { resultsDir, ordersDir, scopesDir, readRunId, verdictsDir, specDir } from "../lib/paths.mjs";
|
|
39
|
-
import { readBoard } from "../compile.mjs";
|
|
40
|
-
import { coveringAcs } from "./requirements.mjs";
|
|
41
|
-
|
|
42
|
-
/** Longest `reason` reported. A deviation is prose written by a worker and can run to paragraphs. */
|
|
43
|
-
const REASON_MAX = 400;
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Bound a reason to {@link REASON_MAX} characters.
|
|
47
|
-
* @param {string} s - The reason.
|
|
48
|
-
* @returns {string} `s`, or its first REASON_MAX − 1 characters and an ellipsis.
|
|
49
|
-
*/
|
|
50
|
-
const clip = (s) => (s.length > REASON_MAX ? `${s.slice(0, REASON_MAX - 1)}…` : s);
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* Whether a feature's spec is SCOPED — has scope contracts, the case in which every verdict must
|
|
54
|
-
* cite the T0 artifacts it re-hashed.
|
|
55
|
-
*
|
|
56
|
-
* @param {string} cwd - Project root.
|
|
57
|
-
* @param {string} slug - Feature slug.
|
|
58
|
-
* @returns {boolean} True when `scopes/` holds at least one contract (`.md`, or a legacy `.json`).
|
|
59
|
-
*/
|
|
60
|
-
export function isScoped(cwd, slug) {
|
|
61
|
-
try { return readdirSync(scopesDir(cwd, slug)).some((f) => /\.(md|json)$/.test(f)); }
|
|
62
|
-
catch { return false; }
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Why one T0 citation does not resolve, or null when the kernel can positively confirm it does.
|
|
67
|
-
*
|
|
68
|
-
* Re-checks the two facts a citation can lie about and the schema promises are enforced
|
|
69
|
-
* (`T0Citation`: "the evaluator RECOMPUTES sha256 from disk — a handed hash is never trusted"):
|
|
70
|
-
* that the bytes at `path` hash to the cited `sha256`, and that what those bytes actually say is a
|
|
71
|
-
* green T0 verdict — a PASS/FAIL cannot ride on an artifact that itself recorded red. NOT checked:
|
|
72
|
-
* whether `path` is one of the order's own `payload.t0_artifacts` (see the note on
|
|
73
|
-
* {@link citationProblem} for why that is left to the operator rather than enforced here).
|
|
74
|
-
*
|
|
75
|
-
* FAILS OPEN ON "CANNOT TELL", CLOSED ON "PROVEN WRONG" — and the two are not the same fact. A read
|
|
76
|
-
* that fails for a reason that says something DEFINITE about what is (or isn't) at `path` is proof,
|
|
77
|
-
* same as a hash mismatch: `ENOENT` (nothing there) and `EISDIR` (a directory, never a file — T0
|
|
78
|
-
* verdicts are always files, per `writeArtifact`) both mean no such artifact was ever produced, so
|
|
79
|
-
* both refuse. `verify t0` writes verdict artifacts immutably and never deletes one
|
|
80
|
-
* (`writeArtifact`'s `wx` flag — see kernel/verify/t0.mjs), so that absence or shape mismatch is a
|
|
81
|
-
* positive fact about the citation, not a guess. Anything else a read can fail with — permission
|
|
82
|
-
* denied, a symlink loop, a transient I/O error — says nothing about the citation's honesty, only
|
|
83
|
-
* that THIS MACHINE could not check it just now, so it is not refused on that ground alone: the
|
|
84
|
-
* fail-open discipline this repo's guards already use elsewhere for an unproven bad state.
|
|
85
|
-
*
|
|
86
|
-
* @param {string} cwd - Project root.
|
|
87
|
-
* @param {object} citation - One `T0Citation` (`scope_id`, `path`, `sha256`). Read defensively:
|
|
88
|
-
* `reduce ingest` only reaches this after the enclosing WorkResult passed schema validation, but
|
|
89
|
-
* `probe eval` reaches it over a result file it merely `JSON.parse`s, so a malformed citation must
|
|
90
|
-
* fail this check rather than throw.
|
|
91
|
-
* @returns {(string|null)} A reason phrased for an operator, or null when the file at `path` exists,
|
|
92
|
-
* hashes to the cited `sha256`, and its own `overall` reads "green".
|
|
93
|
-
*/
|
|
94
|
-
function unresolvedCitation(cwd, citation, { round = null, runId = null, verdicts = null } = {}) {
|
|
95
|
-
const rel = typeof citation?.path === "string" ? citation.path : "";
|
|
96
|
-
if (!rel) return "names no artifact path";
|
|
97
|
-
let text;
|
|
98
|
-
try {
|
|
99
|
-
text = readFileSync(resolve(cwd, rel), "utf8");
|
|
100
|
-
} catch (e) {
|
|
101
|
-
if (e.code === "ENOENT") return `cites ${rel}, which does not exist on disk`;
|
|
102
|
-
if (e.code === "EISDIR") return `cites ${rel}, which is a directory, not a T0 verdict file`;
|
|
103
|
-
// EACCES, ELOOP, EIO, EMFILE… — this machine failing to look, not evidence against the
|
|
104
|
-
// citation, so it is not refused on that ground.
|
|
105
|
-
return null;
|
|
106
|
-
}
|
|
107
|
-
const actual = createHash("sha256").update(text).digest("hex");
|
|
108
|
-
const claimed = typeof citation.sha256 === "string" ? citation.sha256.toLowerCase() : "";
|
|
109
|
-
if (actual !== claimed) return `cites ${rel} with sha256 ${citation.sha256}, but the file on disk hashes to ${actual}`;
|
|
110
|
-
let body;
|
|
111
|
-
try { body = JSON.parse(text); }
|
|
112
|
-
catch { return `cites ${rel}, whose bytes match the hash but do not read as a T0 verdict`; }
|
|
113
|
-
if (body?.overall !== "green") return `cites ${rel}, whose own verdict is "${body?.overall ?? "unknown"}", not green`;
|
|
114
|
-
// INSIDE THIS RUN'S OWN VERDICTS DIRECTORY, resolved — asked of a file that exists, so "does not
|
|
115
|
-
// exist" and "resolves somewhere else" stay different answers. Accepted before this check: a
|
|
116
|
-
// green artifact in the source tree, one outside the project reached by `../`, one by absolute
|
|
117
|
-
// path, and a symlink in the verdicts directory pointing at a forged file. Every one hashed
|
|
118
|
-
// correctly, because a digest says the bytes are the file's and nothing about which file it
|
|
119
|
-
// should have been.
|
|
120
|
-
if (verdicts) {
|
|
121
|
-
const real = (() => { try { return realpathSync.native(resolvePath(cwd, rel)); } catch { return resolvePath(cwd, rel); } })();
|
|
122
|
-
const home = (() => { try { return realpathSync.native(verdicts); } catch { return verdicts; } })();
|
|
123
|
-
if (!real.startsWith(home.endsWith(sep) ? home : home + sep)) {
|
|
124
|
-
return `cites ${rel}, which resolves outside this run's own verdicts directory (${home}) — a verdict cites what this run's own verifier wrote, not a file the judge can reach`;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
// THE ARTIFACT HAS TO BE THE ONE THE CITATION SAYS IT IS. A re-hash proves the bytes are the
|
|
128
|
-
// file's; it says nothing about whose verdict the file holds. A PASS citing scope alpha's green
|
|
129
|
-
// artifact while declaring scope beta, or a prior round's, or a prior run's over the same slug,
|
|
130
|
-
// passed the digest check unremarked. The citation's own required `scope_id`, the round being
|
|
131
|
-
// judged and the run's key are compared to what the artifact records about itself.
|
|
132
|
-
if (typeof citation.scope_id === "string" && body?.scope_id && body.scope_id !== citation.scope_id) {
|
|
133
|
-
return `cites ${rel} for scope "${citation.scope_id}", but the artifact records scope "${body.scope_id}"`;
|
|
134
|
-
}
|
|
135
|
-
if (round != null && typeof body?.round === "number" && body.round !== round) {
|
|
136
|
-
return `cites ${rel}, a round ${body.round} artifact, as evidence for round ${round}`;
|
|
137
|
-
}
|
|
138
|
-
if (runId && body?.run_id && body.run_id !== runId) {
|
|
139
|
-
return `cites ${rel}, an artifact of run ${body.run_id}, as evidence for run ${runId}`;
|
|
140
|
-
}
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* Why a verdict cannot stand as its round's judgement on T0 grounds, or null when it can.
|
|
146
|
-
*
|
|
147
|
-
* A PASS or FAIL on a scoped spec that cites no T0 artifact is structurally invalid — the
|
|
148
|
-
* evaluator's own contract says so, because T0 is the machine fact a generator cannot fabricate.
|
|
149
|
-
* That rule used to live only in the contract, so a verdict citing nothing was ingested, ledgered
|
|
150
|
-
* and branched on like any other. It is checked here so the round loop, the resume derivation, the
|
|
151
|
-
* hill and ingest all refuse the same verdict for the same reason.
|
|
152
|
-
*
|
|
153
|
-
* RESOLVES, NOT JUST PRESENT. A citation naming a path and a hash used to be taken on faith: any
|
|
154
|
-
* non-empty `t0_citations[]` passed, whatever it pointed at. Measured live: a PASS citing a path
|
|
155
|
-
* that does not exist on disk, and a PASS citing a real artifact whose own verdict was red, both
|
|
156
|
-
* ingested clean — presence stood in for a re-hash the schema had already promised. Each citation is
|
|
157
|
-
* now re-checked through {@link unresolvedCitation}.
|
|
158
|
-
*
|
|
159
|
-
* WHAT THIS DOES NOT CHECK: whether a citation is drawn from the order's own `payload.t0_artifacts`
|
|
160
|
-
* list. That would need the compiled order, which this function's callers do not equally have —
|
|
161
|
-
* `reduce ingest` holds it, but `probe eval` (and the round-loop/resume/hill readers behind it)
|
|
162
|
-
* knows only (cwd, slug, round), and a scope's T0 attempt can legitimately go green again LATER than
|
|
163
|
-
* whatever list was frozen at compile time (`greenVerdict` already treats "newest green" as
|
|
164
|
-
* authoritative for exactly this reason — see kernel/probe/t0.mjs). Enforcing membership only where
|
|
165
|
-
* the order happens to be on hand would let one channel refuse a citation the other accepts, for
|
|
166
|
-
* evidence that may simply be fresher than the order — worse than leaving it unenforced.
|
|
167
|
-
*
|
|
168
|
-
* @param {string} cwd - Project root.
|
|
169
|
-
* @param {string} slug - Feature slug.
|
|
170
|
-
* @param {object} verdict - The WorkResult's `verdict` block.
|
|
171
|
-
* @returns {(string|null)} The problem, phrased for an operator; null for a verdict whose every
|
|
172
|
-
* citation resolves, an unscoped spec, or a block with no PASS/FAIL in it (there is no judgement
|
|
173
|
-
* to invalidate).
|
|
174
|
-
*/
|
|
175
|
-
/**
|
|
176
|
-
* Why a verdict cannot stand on its own criteria, or null when it can.
|
|
177
|
-
*
|
|
178
|
-
* `overall` is the judge's field, and nothing recomputed it from the criteria the judge graded: a
|
|
179
|
-
* PASS over a failing criterion, or over no criterion at all, validated and ingested, and the
|
|
180
|
-
* round loop branched on it. The evaluator's own first rule is that absence of evidence is a FAIL,
|
|
181
|
-
* so a PASS criterion with no evidence is no evidence either. Recomputed here, on ingest and on
|
|
182
|
-
* read alike: PASS means every graded criterion passed with evidence and at least one was graded;
|
|
183
|
-
* FAIL means at least one graded criterion failed.
|
|
184
|
-
*
|
|
185
|
-
* @param {object} verdict - The WorkResult's `verdict`.
|
|
186
|
-
* @returns {(string|null)} A reason phrased for an operator, or null.
|
|
187
|
-
*/
|
|
188
|
-
export function verdictProblem(verdict) {
|
|
189
|
-
const overall = verdict?.overall;
|
|
190
|
-
if (overall !== "PASS" && overall !== "FAIL") return null;
|
|
191
|
-
const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
|
|
192
|
-
const fails = criteria.filter((c) => c?.verdict === "FAIL");
|
|
193
|
-
const passes = criteria.filter((c) => c?.verdict === "PASS");
|
|
194
|
-
const other = criteria.length - fails.length - passes.length;
|
|
195
|
-
if (overall === "PASS") {
|
|
196
|
-
if (criteria.length === 0) return "the PASS verdict grades no criterion at all — a PASS with no evidence is a claim";
|
|
197
|
-
if (fails.length) return `the verdict says PASS while ${fails.length} of its ${criteria.length} criteria read FAIL — overall is derived from the criteria, never declared over them`;
|
|
198
|
-
if (other) return `the verdict says PASS while ${other} of its criteria carry no PASS/FAIL verdict`;
|
|
199
|
-
const bare = passes.filter((c) => !(typeof c?.evidence === "string" && c.evidence.trim()));
|
|
200
|
-
if (bare.length) return `the PASS verdict has ${bare.length} criterion(s) marked PASS with no evidence — absence of evidence is a FAIL by the evaluator's own first rule`;
|
|
201
|
-
return null;
|
|
202
|
-
}
|
|
203
|
-
if (criteria.length === 0) return "the FAIL verdict grades no criterion at all — a FAIL must name what failed";
|
|
204
|
-
if (!fails.length) return `the verdict says FAIL while every one of its ${criteria.length} graded criteria reads PASS — a FAIL must cite the criterion it failed`;
|
|
205
|
-
return null;
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
/**
|
|
209
|
-
* The Test Surface row ids the spec declares, in spec order.
|
|
210
|
-
*
|
|
211
|
-
* @param {string} cwd - Project root.
|
|
212
|
-
* @param {string} slug - Feature slug.
|
|
213
|
-
* @returns {string[]} Every `TS-…` id that opens a table row in a use-case file; [] with no spec tree.
|
|
214
|
-
*/
|
|
215
|
-
export function surfaceRows(cwd, slug) {
|
|
216
|
-
const dir = join(specDir(cwd, slug), "usecases");
|
|
217
|
-
let files = [];
|
|
218
|
-
try { files = readdirSync(dir).filter((f) => /^UC-.*\.md$/.test(f)).sort(); } catch { return []; }
|
|
219
|
-
const rows = [];
|
|
220
|
-
for (const f of files) {
|
|
221
|
-
let text = "";
|
|
222
|
-
try { text = readFileSync(join(dir, f), "utf8"); } catch { continue; }
|
|
223
|
-
for (const m of text.matchAll(/^\|\s*(TS-[A-Za-z0-9_-]+)\s*\|/gm)) if (!rows.includes(m[1])) rows.push(m[1]);
|
|
224
|
-
}
|
|
225
|
-
return rows;
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* Whether a PASS grades every Test Surface row by name.
|
|
230
|
-
*
|
|
231
|
-
* A PASS that grades the surface as a whole — "device rows", "local rows" — validated like any other
|
|
232
|
-
* and switched off everything that reads a verdict row by row: the requirements matrix read no
|
|
233
|
-
* evidence under every covered requirement, a failed row had no criterion to become a bug, and a row
|
|
234
|
-
* nobody looked at passed with the rest. A row counts as graded when its id appears in a criterion's
|
|
235
|
-
* text or its `traces_to`. Only a PASS is held to it: a FAIL sends the round back either way.
|
|
236
|
-
*
|
|
237
|
-
* @param {string} cwd - Project root.
|
|
238
|
-
* @param {string} slug - Feature slug.
|
|
239
|
-
* @param {object} verdict - The WorkResult's `verdict`.
|
|
240
|
-
* @returns {(string|null)} A reason naming the rows no criterion graded, or null.
|
|
241
|
-
*/
|
|
242
|
-
export function coverageProblem(cwd, slug, verdict) {
|
|
243
|
-
if (verdict?.overall !== "PASS") return null;
|
|
244
|
-
const rows = surfaceRows(cwd, slug);
|
|
245
|
-
if (!rows.length) return null;
|
|
246
|
-
const named = new Set();
|
|
247
|
-
for (const c of Array.isArray(verdict.criteria) ? verdict.criteria : []) {
|
|
248
|
-
const text = [c?.criterion, ...(Array.isArray(c?.traces_to) ? c.traces_to : [])].map((x) => String(x ?? "")).join(" ");
|
|
249
|
-
for (const m of text.matchAll(/\bTS-[A-Za-z0-9_]+(?:-[A-Za-z0-9_]+)*/g)) named.add(m[0]);
|
|
250
|
-
}
|
|
251
|
-
const missing = rows.filter((r) => !named.has(r));
|
|
252
|
-
if (!missing.length) return null;
|
|
253
|
-
const shown = missing.slice(0, 8).join(", ") + (missing.length > 8 ? ` and ${missing.length - 8} more` : "");
|
|
254
|
-
return `the PASS verdict grades ${rows.length - missing.length} of ${rows.length} Test Surface rows by name — ` +
|
|
255
|
-
`each row is its own criterion, its id in the criterion or its traces_to; ungraded: ${shown}`;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/**
|
|
259
|
-
* Whether a verdict anchors any criterion to a requirement when the board says which ones it could.
|
|
260
|
-
*
|
|
261
|
-
* `traces_to` is copied from the `(covers: REQ-…)` clauses of the acceptance criteria a criterion
|
|
262
|
-
* grades, and the requirements matrix is joined along it. Two judges on the same spec and the same
|
|
263
|
-
* instruction differed: one anchored every row and the matrix read 5/5, the next anchored none and it
|
|
264
|
-
* read 0/5 over the same passes. Held only to the all-empty case: which requirement a criterion
|
|
265
|
-
* traces to is the judge's reading, but none at all, beside a board that covers some, is a copy
|
|
266
|
-
* step skipped.
|
|
267
|
-
*
|
|
268
|
-
* @param {string} cwd - Project root.
|
|
269
|
-
* @param {string} slug - Feature slug.
|
|
270
|
-
* @param {object} verdict - The WorkResult's `verdict`.
|
|
271
|
-
* @returns {(string|null)} A reason phrased for the judge, or null.
|
|
272
|
-
*/
|
|
273
|
-
export function tracesProblem(cwd, slug, verdict) {
|
|
274
|
-
if (verdict?.overall !== "PASS" && verdict?.overall !== "FAIL") return null;
|
|
275
|
-
const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
|
|
276
|
-
if (!criteria.length) return null;
|
|
277
|
-
if (criteria.some((c) => Array.isArray(c?.traces_to) && c.traces_to.length)) return null;
|
|
278
|
-
let covered = 0;
|
|
279
|
-
try { covered = coveringAcs(readBoard(cwd, slug)).size; } catch { return null; }
|
|
280
|
-
if (!covered) return null;
|
|
281
|
-
return `the verdict anchors none of its ${criteria.length} criteria to a requirement while the board's acceptance ` +
|
|
282
|
-
`criteria cover ${covered} — copy each graded AC's (covers: REQ-…) clause into that criterion's traces_to`;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/**
|
|
286
|
-
* A PASS that grades nothing under a dimension its order named.
|
|
287
|
-
*
|
|
288
|
-
* The order names the dimensions the judge grades, and GATE L4 prints them back as what "PASS"
|
|
289
|
-
* covers. Measured: an order named four, the PASS graded criteria under three, and L4 still listed
|
|
290
|
-
* the fourth as evaluated. A PASS must grade at least one criterion under every dimension its order
|
|
291
|
-
* named; a FAIL already stops the ship, so it is not held to this.
|
|
292
|
-
*
|
|
293
|
-
* @param {string} cwd - Project root.
|
|
294
|
-
* @param {string} slug - Feature slug.
|
|
295
|
-
* @param {object} verdict - The WorkResult's `verdict`.
|
|
296
|
-
* @param {{round:(number|null)}} [opts] - The EVAL round whose order names the set; null → no check.
|
|
297
|
-
* @returns {(string|null)} Why the verdict is refused, or null.
|
|
298
|
-
*/
|
|
299
|
-
export function dimensionsProblem(cwd, slug, verdict, { round = null } = {}) {
|
|
300
|
-
if (verdict?.overall !== "PASS" || round == null) return null;
|
|
301
|
-
const named = orderDimensions(cwd, slug, round);
|
|
302
|
-
if (!named?.length) return null;
|
|
303
|
-
const criteria = Array.isArray(verdict.criteria) ? verdict.criteria : [];
|
|
304
|
-
const graded = new Set(criteria.map((c) => c?.dimension).filter(Boolean));
|
|
305
|
-
const missing = named.filter((d) => !graded.has(d));
|
|
306
|
-
if (!missing.length) return null;
|
|
307
|
-
return `the PASS grades no criterion under ${missing.join(", ")}, which the order names — grade each named ` +
|
|
308
|
-
"dimension's criteria, or FAIL the one you cannot grade with the reason";
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
export function citationProblem(cwd, slug, verdict, { round = null } = {}) {
|
|
312
|
-
if (verdict?.overall !== "PASS" && verdict?.overall !== "FAIL") return null;
|
|
313
|
-
if (!isScoped(cwd, slug)) return null;
|
|
314
|
-
if (!Array.isArray(verdict.t0_citations) || !verdict.t0_citations.length) {
|
|
315
|
-
return `the ${verdict.overall} verdict cites no T0 artifact, and a verdict on a scoped spec must ` +
|
|
316
|
-
"cite the T0 verdict it re-hashed (the order lists them under payload.t0_artifacts)";
|
|
317
|
-
}
|
|
318
|
-
const runId = readRunId(cwd, slug);
|
|
319
|
-
const verdicts = verdictsDir(cwd, slug);
|
|
320
|
-
for (const citation of verdict.t0_citations) {
|
|
321
|
-
const reason = unresolvedCitation(cwd, citation, { round, runId, verdicts });
|
|
322
|
-
if (reason) return `the ${verdict.overall} verdict ${reason} — a T0 citation is re-hashed from disk, never taken on the handed word`;
|
|
323
|
-
}
|
|
324
|
-
return null;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
/**
|
|
328
|
-
* Read one round's EVAL verdict straight from the WorkResult `reduce ingest` wrote.
|
|
329
|
-
*
|
|
330
|
-
* @param {string} cwd - Project root.
|
|
331
|
-
* @param {string} slug - Feature slug.
|
|
332
|
-
* @param {number} round - The EVAL round (`evaluate-r<N>.json`).
|
|
333
|
-
* @returns {{found: boolean, overall: (string|null), status: (string|null), reason: (string|null),
|
|
334
|
-
* bug_count: (number|null), report_path: (string|null)}} `found` is true only for a PASS/FAIL the
|
|
335
|
-
* round may act on; otherwise `reason` says why not — a fact, not a guess.
|
|
336
|
-
*/
|
|
337
|
-
export function evalVerdict(cwd, slug, round) {
|
|
338
|
-
const path = join(resultsDir(cwd, slug), `evaluate-r${round}.json`);
|
|
339
|
-
const unfit = (reason, status = null, overall = null) =>
|
|
340
|
-
({ found: false, overall, status, reason, bug_count: null, report_path: null });
|
|
341
|
-
if (!existsSync(path)) return unfit("no evaluate result for this round yet");
|
|
342
|
-
let doc;
|
|
343
|
-
try { doc = JSON.parse(readFileSync(path, "utf8")); }
|
|
344
|
-
catch { return unfit("the evaluate result is not readable JSON"); }
|
|
345
|
-
const status = typeof doc?.status === "string" ? doc.status : null;
|
|
346
|
-
const v = doc?.verdict || {};
|
|
347
|
-
const overall = v.overall === "PASS" || v.overall === "FAIL" ? v.overall : null;
|
|
348
|
-
if (!overall) {
|
|
349
|
-
// A worker that refused to grade says why in its FIRST deviation — the only channel it has.
|
|
350
|
-
const first = Array.isArray(doc?.deviations) && typeof doc.deviations[0] === "string" ? doc.deviations[0] : "";
|
|
351
|
-
return unfit(clip(first
|
|
352
|
-
? `the evaluator returned ${status || "no status"}: ${first}`
|
|
353
|
-
: `status ${status || "unknown"} with no PASS/FAIL verdict`), status);
|
|
354
|
-
}
|
|
355
|
-
const problem = verdictProblem(v) || coverageProblem(cwd, slug, v) || tracesProblem(cwd, slug, v)
|
|
356
|
-
|| dimensionsProblem(cwd, slug, v, { round }) || citationProblem(cwd, slug, v, { round });
|
|
357
|
-
if (problem) return unfit(problem, status, overall);
|
|
358
|
-
return {
|
|
359
|
-
found: true,
|
|
360
|
-
overall,
|
|
361
|
-
status,
|
|
362
|
-
reason: null,
|
|
363
|
-
bug_count: Array.isArray(v.bugs) ? v.bugs.length : null,
|
|
364
|
-
report_path: typeof v.report_path === "string" ? v.report_path : null,
|
|
365
|
-
};
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
/**
|
|
369
|
-
* The dimensions round N's evaluate order named — the set the judge was asked to grade.
|
|
370
|
-
*
|
|
371
|
-
* @param {string} cwd - Project root.
|
|
372
|
-
* @param {string} slug - Feature slug.
|
|
373
|
-
* @param {number} round - The EVAL round (`evaluate-r<N>.json`).
|
|
374
|
-
* @returns {(string[]|null)} The order's `payload.dimensions`; null when the order or the list is absent.
|
|
375
|
-
*/
|
|
376
|
-
export function orderDimensions(cwd, slug, round) {
|
|
377
|
-
try {
|
|
378
|
-
const o = JSON.parse(readFileSync(join(ordersDir(cwd, slug), `evaluate-r${round}.json`), "utf8"));
|
|
379
|
-
const d = o?.payload?.dimensions;
|
|
380
|
-
return Array.isArray(d) && d.every((x) => typeof x === "string") ? d : null;
|
|
381
|
-
} catch { return null; }
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
export const ARGV_SPEC = {
|
|
385
|
-
usage: "harness.mjs probe eval --slug <slug> --round N [--cwd <dir>]",
|
|
386
|
-
_: { arity: 0, max: 0, name: "(no positional operands)" },
|
|
387
|
-
slug: { type: "str", required: true },
|
|
388
|
-
round: { type: "int", min: 1, required: true },
|
|
389
|
-
cwd: { type: "path" },
|
|
390
|
-
};
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* Report round N's EVAL verdict, mechanically, from the WorkResult on disk.
|
|
394
|
-
*
|
|
395
|
-
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
396
|
-
* @returns {void} Exits 0 when the round holds a verdict the run may act on, 1 when it does not.
|
|
397
|
-
*/
|
|
398
|
-
export function cli(rawArgv) {
|
|
399
|
-
const args = runArgs(ARGV_SPEC, rawArgv);
|
|
400
|
-
const cwd = resolve(args.cwd || process.cwd());
|
|
401
|
-
const { found, overall, status, reason, bug_count, report_path } = evalVerdict(cwd, args.slug, args.round);
|
|
402
|
-
const dimensions = orderDimensions(cwd, args.slug, args.round);
|
|
403
|
-
console.log(JSON.stringify({ ok: found, overall, bug_count, report_path, round: args.round, status, reason, dimensions }));
|
|
404
|
-
process.exit(found ? 0 : 1);
|
|
405
|
-
}
|
package/kernel/probe/hunt.mjs
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// probe hunt — "did the QA hunt hunt anything?"
|
|
3
|
-
//
|
|
4
|
-
// CONTRACT. A bounded, read-only query over the hunt's WorkResult, its order and its report.
|
|
5
|
-
// Prints `{ok, status, qa, charters_run, findings, reason}` on stdout; exits 0 when the hunt's
|
|
6
|
-
// result is one the run may accept, 1 when it is not, and 2 on a bad argv. Writes nothing.
|
|
7
|
-
//
|
|
8
|
-
// WHY THIS EXISTS. A hunter that reached the app, swept the fault log and then drafted no charter
|
|
9
|
-
// still returned `done`, and a `done` with no findings reads as an app with nothing wrong. The hunt
|
|
10
|
-
// report's charter count says what actually happened. A `done` hunt over an app its order could
|
|
11
|
-
// reach, with no charter run, is refused here and on ingest, and the run sends the hunter back once
|
|
12
|
-
// — the same shape as a verdict that grades nothing by name.
|
|
13
|
-
|
|
14
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
15
|
-
import { join, resolve } from "node:path";
|
|
16
|
-
import { runArgs } from "../lib/argv.mjs";
|
|
17
|
-
import { resultsDir, ordersDir, qaDir } from "../lib/paths.mjs";
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* The number of charters a hunt report says it ran.
|
|
21
|
-
*
|
|
22
|
-
* @param {(string|null)} text - The hunt report's text.
|
|
23
|
-
* @returns {(number|null)} The run count from its `charters: <run>/<approved>` line, or null when
|
|
24
|
-
* there is no report or no such line.
|
|
25
|
-
*/
|
|
26
|
-
export function chartersRun(text) {
|
|
27
|
-
const m = String(text ?? "").match(/^charters:\s*(\d+)/m);
|
|
28
|
-
return m ? Number(m[1]) : null;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Whether a hunt result may be accepted as it stands.
|
|
33
|
-
*
|
|
34
|
-
* Held to it only when the result says `done` and the order gave the hunter a way into the app
|
|
35
|
-
* (`launch_cmd` or `app_url`): a hunt that returns `failed` says it could not hunt, which is honest,
|
|
36
|
-
* and a hunt with no way in cannot be asked to drive anything.
|
|
37
|
-
*
|
|
38
|
-
* @param {object} result - The hunt WorkResult.
|
|
39
|
-
* @param {object} payload - The hunt order's payload.
|
|
40
|
-
* @param {(string|null)} report - The hunt report's text, or null.
|
|
41
|
-
* @returns {(string|null)} A reason phrased for the hunter, or null.
|
|
42
|
-
*/
|
|
43
|
-
export function huntProblem(result, payload, report) {
|
|
44
|
-
if (result?.status !== "done") return null;
|
|
45
|
-
if (!payload?.launch_cmd && !payload?.app_url) return null;
|
|
46
|
-
const n = chartersRun(report);
|
|
47
|
-
if (n === null) return "the hunt returned done with no hunt report carrying a `charters:` line";
|
|
48
|
-
if (n > 0) return null;
|
|
49
|
-
return "the hunt returned done over an app its order could reach, having run 0 charters — " +
|
|
50
|
-
"draft at least one charter where the EVAL left territory uncovered and hunt it, or return failed saying why the app could not be driven";
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Read a JSON file, or null.
|
|
55
|
-
* @param {string} p - Path.
|
|
56
|
-
* @returns {(object|null)} The parsed document, or null when absent or unreadable.
|
|
57
|
-
*/
|
|
58
|
-
const readJson = (p) => { try { return JSON.parse(readFileSync(p, "utf8")); } catch { return null; } };
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* What the hunt on disk amounts to.
|
|
62
|
-
*
|
|
63
|
-
* @param {string} cwd - Project root.
|
|
64
|
-
* @param {string} slug - Feature slug.
|
|
65
|
-
* @returns {{ok:boolean, status:(string|null), qa:string, charters_run:(number|null), findings:number, reason:(string|null)}}
|
|
66
|
-
* `qa` is `run` (charters ran), `not-hunted` (the hunt returned, no charter ran), or `missing`.
|
|
67
|
-
*/
|
|
68
|
-
export function huntOutcome(cwd, slug) {
|
|
69
|
-
const result = readJson(join(resultsDir(cwd, slug), "hunt.json"));
|
|
70
|
-
if (!result) return { ok: false, status: null, qa: "missing", charters_run: null, findings: 0, reason: "no hunt result on disk" };
|
|
71
|
-
const payload = readJson(join(ordersDir(cwd, slug), "hunt.json"))?.payload || {};
|
|
72
|
-
const reportPath = join(qaDir(cwd, slug), "hunt-report.md");
|
|
73
|
-
const report = existsSync(reportPath) ? readFileSync(reportPath, "utf8") : null;
|
|
74
|
-
const n = chartersRun(report);
|
|
75
|
-
const reason = huntProblem(result, payload, report);
|
|
76
|
-
return {
|
|
77
|
-
ok: !reason,
|
|
78
|
-
status: typeof result.status === "string" ? result.status : null,
|
|
79
|
-
qa: n ? "run" : "not-hunted",
|
|
80
|
-
charters_run: n,
|
|
81
|
-
findings: Array.isArray(result.discoveries) ? result.discoveries.length : 0,
|
|
82
|
-
reason,
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export const ARGV_SPEC = {
|
|
87
|
-
usage: "harness.mjs probe hunt --slug <slug> [--cwd <dir>]",
|
|
88
|
-
_: { arity: 0, max: 0, name: "(no positional operands)" },
|
|
89
|
-
slug: { type: "str", required: true },
|
|
90
|
-
cwd: { type: "path" },
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Report what the hunt on disk amounts to.
|
|
95
|
-
*
|
|
96
|
-
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
97
|
-
* @returns {void} Exits 0 when the hunt result may be accepted, 1 when it may not.
|
|
98
|
-
*/
|
|
99
|
-
export function cli(rawArgv) {
|
|
100
|
-
const args = runArgs(ARGV_SPEC, rawArgv);
|
|
101
|
-
const out = huntOutcome(resolve(args.cwd || process.cwd()), args.slug);
|
|
102
|
-
console.log(JSON.stringify(out));
|
|
103
|
-
process.exit(out.ok ? 0 : 1);
|
|
104
|
-
}
|