shapeup-sdlc 3.18.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude/settings.local.example.json +2 -3
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/AGENTS.md +115 -56
  4. package/README.md +101 -143
  5. package/SECURITY.md +16 -25
  6. package/agents/orient.md +14 -0
  7. package/agents/qa-edge-hunter.md +13 -0
  8. package/agents/spec-evaluator.md +14 -0
  9. package/agents/task-executor.md +14 -0
  10. package/bin/init.mjs +13 -16
  11. package/bin/lib/grant.mjs +8 -19
  12. package/commands/build.md +9 -14
  13. package/commands/eval.md +3 -3
  14. package/commands/hammer.md +9 -5
  15. package/commands/retro.md +12 -11
  16. package/commands/scopes.md +1 -12
  17. package/commands/ship.md +45 -88
  18. package/commands/wire.md +1 -8
  19. package/hooks/dispatch-receipt.mjs +94 -8
  20. package/hooks/hooks.json +0 -41
  21. package/hooks/lib/decision.mjs +6 -9
  22. package/hooks/lib/tier.mjs +50 -0
  23. package/hooks/sandbox-guard.mjs +140 -64
  24. package/kernel/gate.mjs +203 -22
  25. package/kernel/harness.mjs +20 -67
  26. package/kernel/init/run.mjs +175 -523
  27. package/kernel/lib/contract.mjs +9 -93
  28. package/kernel/lib/floor.mjs +64 -0
  29. package/kernel/lib/paths.mjs +39 -112
  30. package/kernel/order.mjs +157 -0
  31. package/kernel/probe/digest.mjs +2 -38
  32. package/kernel/schemas/domain.schema.json +146 -1125
  33. package/kernel/schemas/work-order.schema.json +8 -4
  34. package/kernel/schemas/work-result.schema.json +1 -1
  35. package/kernel/verify/envelope.mjs +2 -2
  36. package/kernel/verify/skills.mjs +27 -8
  37. package/kernel/verify/t0.mjs +2 -2
  38. package/package.json +4 -2
  39. package/skills/building/SKILL.md +144 -0
  40. package/skills/building/resources/gates.md +76 -0
  41. package/skills/building/resources/guidance.md +23 -0
  42. package/skills/{translator/references/preservation-rules.md → building/resources/intake-preservation.md} +1 -1
  43. package/skills/building/resources/intake.md +157 -0
  44. package/skills/building/resources/orders.md +69 -0
  45. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/doc-schemas.md +1 -1
  46. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/task-generation.md +13 -13
  47. package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/test-surface.md +6 -7
  48. package/skills/building/resources/plan-scopes.md +95 -0
  49. package/skills/building/resources/plan-spec.md +102 -0
  50. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/_index.tmpl.md +1 -1
  51. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/assess-report.tmpl.md +10 -11
  52. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/run-state.tmpl.md +4 -4
  53. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/scope-summary.tmpl.md +2 -2
  54. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/synthesis.tmpl.md +4 -4
  55. package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/usecase.tmpl.md +2 -2
  56. package/skills/building/resources/plan-wire.md +82 -0
  57. package/skills/building/resources/plan.md +98 -0
  58. package/skills/building/resources/retro-research.md +60 -0
  59. package/skills/building/resources/retro-scan.md +49 -0
  60. package/skills/building/resources/retro.md +225 -0
  61. package/skills/building/resources/rounds.md +78 -0
  62. package/skills/building/resources/stop.md +127 -0
  63. package/skills/orient/SKILL.md +37 -37
  64. package/skills/qa-edge-hunter/SKILL.md +29 -14
  65. package/skills/shapeup/SKILL.md +5 -5
  66. package/skills/shapeup/resources/breadboarding.md +1 -1
  67. package/skills/shapeup/resources/context-compaction.md +5 -5
  68. package/skills/shapeup/resources/shaping.md +1 -1
  69. package/skills/spec-evaluator/SKILL.md +17 -21
  70. package/skills/spec-evaluator/references/dimension-contract.md +1 -1
  71. package/skills/spec-evaluator/references/dimensions/completeness.md +3 -3
  72. package/skills/spec-evaluator/references/dimensions/spec-conformance.md +1 -1
  73. package/skills/spec-evaluator/references/dimensions/test-surface-conformance.md +5 -5
  74. package/skills/task-executor/SKILL.md +11 -15
  75. package/commands/hill.md +0 -10
  76. package/hooks/gate-intake.mjs +0 -109
  77. package/hooks/gate-zerowork.mjs +0 -349
  78. package/hooks/tier-guard.mjs +0 -162
  79. package/kernel/compile.mjs +0 -1419
  80. package/kernel/init/fit.mjs +0 -199
  81. package/kernel/init/run-args.mjs +0 -206
  82. package/kernel/lib/breadboard.mjs +0 -165
  83. package/kernel/probe/attempts.mjs +0 -161
  84. package/kernel/probe/concurrency.mjs +0 -535
  85. package/kernel/probe/eval.mjs +0 -405
  86. package/kernel/probe/hunt.mjs +0 -104
  87. package/kernel/probe/leg.mjs +0 -229
  88. package/kernel/probe/owner.mjs +0 -142
  89. package/kernel/probe/requirements.mjs +0 -328
  90. package/kernel/probe/resume.mjs +0 -1110
  91. package/kernel/probe/rounds.mjs +0 -115
  92. package/kernel/probe/staged.mjs +0 -83
  93. package/kernel/probe/stats.mjs +0 -477
  94. package/kernel/probe/t0.mjs +0 -115
  95. package/kernel/reduce/board.mjs +0 -350
  96. package/kernel/reduce/graph.mjs +0 -482
  97. package/kernel/reduce/hill.mjs +0 -322
  98. package/kernel/reduce/ingest.mjs +0 -767
  99. package/kernel/reduce/leftovers.mjs +0 -159
  100. package/kernel/reduce/ship.mjs +0 -529
  101. package/kernel/reduce/snapshot.mjs +0 -291
  102. package/kernel/reduce/verdict.mjs +0 -172
  103. package/kernel/report/export.mjs +0 -384
  104. package/kernel/report/facts.mjs +0 -234
  105. package/kernel/verify/budget.mjs +0 -153
  106. package/kernel/verify/build.mjs +0 -428
  107. package/kernel/verify/dispatch.mjs +0 -114
  108. package/kernel/verify/spec.mjs +0 -975
  109. package/kernel/verify/trace.mjs +0 -604
  110. package/skills/ba-pitch-analyzer/SKILL.md +0 -204
  111. package/skills/coach/SKILL.md +0 -397
  112. package/skills/hill-chart/SKILL.md +0 -154
  113. package/skills/hill-chart/assets/dashboard.template.html +0 -1432
  114. package/skills/scope-architect/SKILL.md +0 -174
  115. package/skills/scope-hammer/SKILL.md +0 -231
  116. package/skills/solution-architect/SKILL.md +0 -158
  117. package/skills/tech-lead/SKILL.md +0 -151
  118. package/skills/tech-lead/references/gates.md +0 -605
  119. package/skills/tech-lead/references/protocol.md +0 -889
  120. package/skills/tech-lead/references/tiny-lane.md +0 -53
  121. package/skills/tech-lead/workflows/shapeup-run.js +0 -2187
  122. package/skills/translator/SKILL.md +0 -258
  123. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/contract-patterns.md +0 -0
  124. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ddd-patterns.md +0 -0
  125. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/integration-analysis.md +0 -0
  126. /package/skills/{ba-pitch-analyzer/references → building/resources/plan-references}/ux-behavior-patterns.md +0 -0
  127. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/api-feasibility.tmpl.md +0 -0
  128. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/be-service.contract.tmpl.md +0 -0
  129. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/offline-storage.contract.tmpl.md +0 -0
  130. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/contracts/third-party-api.contract.tmpl.md +0 -0
  131. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/context-map.tmpl.md +0 -0
  132. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/event-choreography.tmpl.md +0 -0
  133. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/migration-plan.tmpl.md +0 -0
  134. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/cross-context/team-handoff.tmpl.md +0 -0
  135. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/domain-model.tmpl.md +0 -0
  136. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/feedback.tmpl.md +0 -0
  137. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/integration.tmpl.md +0 -0
  138. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-board.tmpl.md +0 -0
  139. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task-spike.tmpl.md +0 -0
  140. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/task.tmpl.md +0 -0
  141. /package/skills/{ba-pitch-analyzer/assets/templates → building/resources/plan-templates}/ux-behavior.tmpl.md +0 -0
@@ -1,115 +0,0 @@
1
- // rounds — how many BUILD/EVAL rounds a run actually reached, derived from disk.
2
- //
3
- // WHY THIS EXISTS (measured, not theorized).
4
- //
5
- // `harness-run.md`'s `rounds_used` frontmatter is written ONCE, at GATE L0.1 (`init run`), as `0`,
6
- // and nothing in the round loop ever rewrites it — the orchestrator's own RunReturn carries the
7
- // real count, but that value lives in a JS variable a relaunch loses, and it never reached the
8
- // ledger. `reduce ship`'s own derivation partly compensated by counting the highest
9
- // `evaluate-r<N>.json` result on disk, which is right for "rounds the judge saw" and wrong for
10
- // "rounds the run built": measured after two full BUILD rounds with neither reaching EVAL,
11
- // `round_count: 0` and `rounds_used: 0` both — a run that did real work reported having done none.
12
- //
13
- // TWO NUMBERS, NOT ONE. A round can be built and die before EVAL ever sees it (a circuit breaker,
14
- // a kill mid-round), so "rounds built" and "rounds judged" are different facts and collapsing them
15
- // into a single field is exactly what made the fallback silently wrong. Both are returned here, and
16
- // both are meant to survive to wherever a run's numbers are read — the ship report and the export.
17
- //
18
- // PURE. Reads the run's own trace, writes nothing — the same discipline as `reduce ship`'s other
19
- // derivations (`t0Summary`, `boardCensus`, …). A run before any of these artifacts existed, or a
20
- // lane with no round concept at all (`--tiny`), falls back to the caller-supplied ledger value,
21
- // non-regression with the earlier, EVAL-only behaviour.
22
-
23
- import { existsSync, readdirSync, readFileSync } from "node:fs";
24
- import { join } from "node:path";
25
- import { ordersDir, verdictsDir, roundBuildDir, resultsDir, readRunId } from "../lib/paths.mjs";
26
-
27
- /** Parse a JSON file, returning null rather than throwing — every reader here is best-effort. */
28
- function readJson(p) {
29
- try { return JSON.parse(readFileSync(p, "utf8")); } catch { return null; }
30
- }
31
-
32
- /**
33
- * The round a build-addressed order id encodes, e.g. `checkout/sc-01-r2-a1` → 2.
34
- * @param {*} orderId - The order's own id, whatever shape it happens to be.
35
- * @returns {(number|null)} The round, or null when the id carries none (`orient`, `wire`, `hammer`, …).
36
- */
37
- export function orderRound(orderId) {
38
- const suffix = String(orderId ?? "").split("/").slice(1).join("/");
39
- const m = suffix.match(/-r(\d+)(?:-a\d+)?$/);
40
- return m ? Number(m[1]) : null;
41
- }
42
-
43
- const maxOf = (nums) => (nums.length ? Math.max(...nums) : null);
44
-
45
- /**
46
- * How many rounds this run actually reached — built, and separately, judged.
47
- *
48
- * `rounds_used` is the highest round carrying ANY build evidence: a compiled order, a T0 verdict
49
- * artifact, a round build-gate artifact, or an EVAL result — the brief's own list, plus EVAL results
50
- * because a judged round is, by construction, a round that was also built. `rounds_judged` is the
51
- * highest round EVAL actually returned a verdict for (`evaluate-r<N>.json` on disk), kept as its own
52
- * field rather than folded into the only number a reader can see.
53
- *
54
- * @param {string} cwd - Project root.
55
- * @param {string} slug - Feature slug.
56
- * @param {*} [fallback] - What to report for `rounds_used` when nothing on disk is derivable — the
57
- * caller's own ledger value, passed through untouched (some callers hand this on already coerced
58
- * to a number, some as the raw frontmatter string; this function does not care which).
59
- * @returns {{rounds_used:*, rounds_judged:(number|null)}} Both counts.
60
- */
61
- export function deriveRounds(cwd, slug, fallback) {
62
- // THIS RUN'S RECORDS ONLY. Orders, verdicts and build gates over one slug accumulate across runs
63
- // and carry their run key; walking the directories unfiltered handed a run that had dispatched
64
- // nothing a prior run's round count — into the committed report. A record with no key at all was
65
- // written before the key existed and is kept; one with a different key is another run's.
66
- const runId = readRunId(cwd, slug);
67
- const mine = (rec) => !runId || !rec?.run_id || rec.run_id === runId;
68
- const orderRounds = [];
69
- const oDir = ordersDir(cwd, slug);
70
- const orderOf = {};
71
- if (existsSync(oDir)) {
72
- for (const f of readdirSync(oDir)) {
73
- if (!f.endsWith(".json")) continue;
74
- const o = readJson(join(oDir, f));
75
- orderOf[f] = o;
76
- if (!mine(o)) continue;
77
- const r = orderRound(o?.order_id);
78
- if (r !== null) orderRounds.push(r);
79
- }
80
- }
81
-
82
- const verdictRounds = [];
83
- const vDir = verdictsDir(cwd, slug);
84
- if (existsSync(vDir)) {
85
- for (const f of readdirSync(vDir).filter((x) => x.endsWith(".json"))) {
86
- const v = readJson(join(vDir, f));
87
- if (mine(v) && typeof v?.round === "number") verdictRounds.push(v.round);
88
- }
89
- }
90
-
91
- const buildGateRounds = [];
92
- const bDir = roundBuildDir(cwd, slug);
93
- if (existsSync(bDir)) {
94
- for (const f of readdirSync(bDir)) {
95
- const m = f.match(/^r(\d+)-t\d+\.json$/);
96
- if (m && mine(readJson(join(bDir, f)))) buildGateRounds.push(Number(m[1]));
97
- }
98
- }
99
-
100
- const evalRounds = [];
101
- const rDir = resultsDir(cwd, slug);
102
- if (existsSync(rDir)) {
103
- for (const f of readdirSync(rDir)) {
104
- const m = f.match(/^evaluate-r(\d+)\.json$/);
105
- // A WorkResult carries no run key; it reaches one through its order of the same name.
106
- if (m && mine(orderOf[f] ?? readJson(join(oDir, f)))) evalRounds.push(Number(m[1]));
107
- }
108
- }
109
-
110
- const built = maxOf([...orderRounds, ...verdictRounds, ...buildGateRounds, ...evalRounds]);
111
- return {
112
- rounds_used: built !== null ? built : fallback,
113
- rounds_judged: maxOf(evalRounds),
114
- };
115
- }
@@ -1,83 +0,0 @@
1
- // staged — is the orchestrator this launch will execute the one that is installed?
2
- //
3
- // A run stages its own copy of the workflow scripts into the local tier when it is OPENED, and
4
- // keeps that copy for the run's life. That is deliberate and right: an upgrade must not swap the
5
- // orchestrator under a run in flight. The consequence is not obvious from anywhere a person about
6
- // to soak an upgrade would look — relaunching an existing run after installing a new version
7
- // executes the OLD orchestrator, reports normally, closes normally, and every observation made of
8
- // it is an observation of the previous release. That is worse than a failed soak: it is confident
9
- // evidence about the wrong artifact.
10
- //
11
- // So the launch compares, and says so. A warning, never a block — the run keeping its copy is the
12
- // correct behaviour, and the operator is the one who decides whether this run is the one they
13
- // meant to measure.
14
- import { existsSync, readFileSync, readdirSync } from "node:fs";
15
- import { join } from "node:path";
16
- import { createHash } from "node:crypto";
17
- import { workflowsStage, receipt, readReceipt } from "../lib/paths.mjs";
18
-
19
- const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
20
-
21
- /** A file's digest, or null when it cannot be read. */
22
- function digestOf(path) {
23
- try { return sha256(readFileSync(path)); } catch { return null; }
24
- }
25
-
26
- /** The version the plugin at `pluginRoot` declares, or null. */
27
- export function installedPluginVersion(pluginRoot) {
28
- if (!pluginRoot) return null;
29
- try { return JSON.parse(readFileSync(join(pluginRoot, ".claude-plugin", "plugin.json"), "utf8")).version ?? null; }
30
- catch { return null; }
31
- }
32
-
33
- /**
34
- * How the orchestrator this launch will run compares with the one installed.
35
- *
36
- * @param {string} cwd - Project root.
37
- * @param {(string|null)} pluginRoot - The installed plugin's root, as the launch knows it.
38
- * @param {(string|null)} [slug] - The run, when the caller knows it — for the version the run opened under.
39
- * @returns {object} `{checked, drift, staged_dir, plugin_root, installed_version, run_version}`.
40
- * `drift` lists each script whose staged copy differs from the installed one; `checked: false`
41
- * means the comparison could not be made (no plugin root, or nothing staged), which is not the
42
- * same fact as "they agree" and is reported as itself.
43
- */
44
- export function stagedWorkflowDrift(cwd, pluginRoot, slug = null) {
45
- const stagedDir = workflowsStage(cwd);
46
- const srcDir = pluginRoot ? join(pluginRoot, "skills", "tech-lead", "workflows") : null;
47
- const out = {
48
- checked: false,
49
- drift: [],
50
- staged_dir: stagedDir,
51
- plugin_root: pluginRoot ?? null,
52
- installed_version: installedPluginVersion(pluginRoot),
53
- run_version: slug ? (readReceipt(receipt(cwd, slug))?.plugin?.version ?? null) : null,
54
- };
55
- if (!srcDir || !existsSync(srcDir) || !existsSync(stagedDir)) return out;
56
- let names;
57
- try { names = readdirSync(stagedDir).filter((f) => f.endsWith(".js")).sort(); } catch { return out; }
58
- if (!names.length) return out;
59
- out.checked = true;
60
- for (const f of names) {
61
- const staged = digestOf(join(stagedDir, f));
62
- const installed = digestOf(join(srcDir, f));
63
- // A script the installed plugin no longer carries is drift too — the staged copy is running
64
- // something that has no counterpart in what is installed.
65
- if (staged !== installed) out.drift.push({ file: f, staged_sha256: staged, installed_sha256: installed });
66
- }
67
- return out;
68
- }
69
-
70
- /**
71
- * The one-line warning a launch prints, or null when there is nothing to say.
72
- * @param {object} d - A {@link stagedWorkflowDrift} result.
73
- * @returns {(string|null)} Operator-facing text naming both versions.
74
- */
75
- export function driftWarning(d) {
76
- if (!d?.checked || !d.drift.length) return null;
77
- const versions = d.run_version && d.installed_version && d.run_version !== d.installed_version
78
- ? ` The run opened under plugin ${d.run_version}; ${d.installed_version} is installed.`
79
- : d.installed_version ? ` Installed plugin: ${d.installed_version}.` : "";
80
- return `the orchestrator this launch runs is NOT the installed one — ${d.drift.length} staged script(s) differ `
81
- + `(${d.drift.map((x) => x.file).join(", ")}).${versions} A run keeps the copy it opened with, by design, so this `
82
- + `launch measures the release the run was opened under. Open a NEW run to soak an upgrade.`;
83
- }
@@ -1,477 +0,0 @@
1
- #!/usr/bin/env node
2
- // Stats — the telemetry read-plane (v1.2).
3
- //
4
- // SHIP S.6 has been writing fact rows to .shapeup/metrics/<machine-id>.jsonl since
5
- // v0.x with exactly one documented reader: `cat *.jsonl`. This script is the missing
6
- // projection: rounds per pitch, hammer-cut rate, attempt-budget exhaustions, QA promotion
7
- // rate, and the round_count trend — the "is the KB flywheel actually working?" chart.
8
- //
9
- // Hard rules, inherited from the MetricsRow contract:
10
- // • READ-ONLY. Only read APIs are imported; the shards stay the single source of truth.
11
- // • FACTS ONLY. Aggregates copied fields (counts, rates, trends); it never grades — a
12
- // computed score here would be a second judge behind spec-evaluator.
13
- // • Fail-open per row: a malformed line is skipped and counted, never fatal. Pathology
14
- // rows (sandbox-guard / safety-spine denials share the shards) are partitioned into a
15
- // histogram, not treated as runs or errors.
16
- //
17
- // Output is a StatsReport (domain.schema.json#/$defs/StatsReport), self-validated before it
18
- // is emitted; --format table renders the human view from the already-validated object.
19
- //
20
- // Usage: node `harness probe stats` [--cwd <dir>] [--metrics-dir <dir>] [--slug <slug>] [--format json|table]
21
-
22
- import { readFileSync, readdirSync, existsSync } from "node:fs";
23
- import { resolve, join, relative, sep } from "node:path";
24
- import { validate } from "../verify/envelope.mjs";
25
- import { runArgs } from "../lib/argv.mjs";
26
- import { localDir, decisions as decisionsPath, metricsDir as metricsDirPath, SHARED } from "../lib/paths.mjs";
27
-
28
- /**
29
- * Read every metrics shard, partitioning valid rows, pathology rows, and malformed lines.
30
- * @param {string} metricsDir - Directory of `*.jsonl` metric shards (absent → empty result).
31
- * @returns {{rows:Array<object>, pathologies:Array<object>, sources:string[], rows_malformed:number}}
32
- * Harvest rows (each tagged with a per-shard `_seq`), pathology rows, the shard filenames read,
33
- * and the count of unparseable/incomplete lines skipped.
34
- */
35
- export function readShards(metricsDir) {
36
- const out = { rows: [], pathologies: [], sources: [], rows_malformed: 0 };
37
- if (!existsSync(metricsDir)) return out;
38
- for (const shard of readdirSync(metricsDir).filter((f) => f.endsWith(".jsonl")).sort()) {
39
- out.sources.push(shard);
40
- const lines = readFileSync(join(metricsDir, shard), "utf8").split("\n");
41
- let seq = 0;
42
- for (const line of lines) {
43
- if (!line.trim()) continue;
44
- let row;
45
- try { row = JSON.parse(line); } catch { out.rows_malformed++; continue; }
46
- if (row?.kind === "pathology") { out.pathologies.push(row); continue; }
47
- if (!row?.feature_slug || !row?.terminal_state) { out.rows_malformed++; continue; }
48
- out.rows.push({ ...row, _seq: seq++ });
49
- }
50
- }
51
- return out;
52
- }
53
-
54
- /**
55
- * @param {number} n - A number.
56
- * @returns {number} `n` rounded to two decimal places.
57
- */
58
- const round2 = (n) => Math.round(n * 100) / 100;
59
-
60
- /**
61
- * Aggregate harvest rows into the StatsReport body (pure — no I/O, no grading).
62
- * @param {{rows:Array<object>, pathologies:Array<object>, sources:string[], rows_malformed:number}}
63
- * shards - Output of {@link readShards}.
64
- * @param {{metricsDir?:string, slugFilter?:(string|null)}} [opts] - metricsDir echoed into the
65
- * report; slugFilter restricts to one feature slug.
66
- * @returns {object} A StatsReport (domain.schema.json#/$defs/StatsReport): per-slug run counts,
67
- * terminal-state histograms, hammer-cut rate, attempt exhaustions, QA promotion rates, a
68
- * pathology histogram, and the round_count trend.
69
- */
70
- export function aggregate({ rows, pathologies, sources, rows_malformed }, { metricsDir = "", slugFilter = null } = {}) {
71
- const filtered = slugFilter ? rows.filter((r) => r.feature_slug === slugFilter) : rows;
72
-
73
- const bySlug = new Map();
74
- for (const row of filtered) {
75
- if (!bySlug.has(row.feature_slug)) bySlug.set(row.feature_slug, []);
76
- bySlug.get(row.feature_slug).push(row);
77
- }
78
-
79
- const per_slug = [...bySlug.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([slug, runs]) => {
80
- const rounds = runs.map((r) => r.round_count).filter((n) => Number.isInteger(n));
81
- const terminal_states = {};
82
- for (const r of runs) terminal_states[r.terminal_state] = (terminal_states[r.terminal_state] || 0) + 1;
83
- const cuts = runs.filter((r) => (r.scope_cut_count || 0) > 0).length;
84
- const qaRows = runs.filter((r) => r.qa_findings);
85
- const qa = {
86
- total: qaRows.reduce((s, r) => s + (r.qa_findings.total || 0), 0),
87
- promoted: qaRows.reduce((s, r) => s + (r.qa_findings.promoted || 0), 0),
88
- held: qaRows.reduce((s, r) => s + (r.qa_findings.held || 0), 0),
89
- };
90
- qa.promotion_rate = qa.total > 0 ? round2(qa.promoted / qa.total) : 0;
91
- const surprises = runs.map((r) => r.surprise_count).filter((n) => Number.isInteger(n));
92
- const perSlice = runs
93
- .filter((r) => Number.isInteger(r.round_count) && Number.isInteger(r.slice_count) && r.slice_count > 0)
94
- .map((r) => r.round_count / r.slice_count);
95
- const entry = {
96
- feature_slug: slug,
97
- runs: runs.length,
98
- terminal_states,
99
- hammer_cut_rate: round2(cuts / runs.length),
100
- scope_cuts_total: runs.reduce((s, r) => s + (r.scope_cut_count || 0), 0),
101
- attempt_exhaustions: runs.reduce((s, r) => s + (Number.isInteger(r.attempt_exhaustions) ? r.attempt_exhaustions : 0), 0),
102
- qa,
103
- };
104
- if (rounds.length > 0) {
105
- entry.rounds = {
106
- min: Math.min(...rounds),
107
- max: Math.max(...rounds),
108
- avg: round2(rounds.reduce((s, n) => s + n, 0) / rounds.length),
109
- };
110
- }
111
- if (surprises.length > 0) entry.surprise_avg = round2(surprises.reduce((s, n) => s + n, 0) / surprises.length);
112
- if (perSlice.length > 0) entry.rounds_per_slice_avg = round2(perSlice.reduce((s, n) => s + n, 0) / perSlice.length);
113
- return entry;
114
- });
115
-
116
- const pathologyHistogram = {};
117
- for (const p of pathologies) {
118
- const key = p.pathology || "unknown";
119
- pathologyHistogram[key] = (pathologyHistogram[key] || 0) + 1;
120
- }
121
-
122
- const trend = filtered
123
- .filter((r) => Number.isInteger(r.round_count))
124
- .sort((a, b) => (a.at && b.at) ? a.at.localeCompare(b.at) : a._seq - b._seq)
125
- .map((r) => {
126
- const t = { feature_slug: r.feature_slug, seq: r._seq, round_count: r.round_count };
127
- if (r.at) t.at = r.at;
128
- return t;
129
- });
130
-
131
- return {
132
- schema_version: 1,
133
- generated_at: new Date().toISOString(),
134
- metrics_dir: metricsDir,
135
- sources,
136
- rows_total: filtered.length,
137
- rows_malformed,
138
- rows_pathology: pathologies.length,
139
- pathologies: pathologyHistogram,
140
- per_slug,
141
- trend,
142
- };
143
- }
144
-
145
- // --- the two exit measurements (v1.5) ------------------------------------------------------
146
- //
147
- // Both answer questions this project has never been able to ask, and both cost ZERO model tokens:
148
- // they reduce over artifacts the harness already writes while doing its ordinary work.
149
-
150
- /**
151
- * `--ratchet` — does the build loop actually ratchet?
152
- *
153
- * The question harness-versus-bare-agent is already answered elsewhere. This one is THE LOOP
154
- * VERSUS ITS OWN FIRST ATTEMPT, and it cannot be
155
- * won by a one-sentence control, because a one-sentence control has no second attempt to compare.
156
- *
157
- * A monotone series is a ratchet working. A flat or sawtooth series says the loop is still a
158
- * budgeted retry loop, and that widening `inspect()` is load-bearing rather than tidy.
159
- *
160
- * @param {Array<object>} trials - TrialRow records (from one run's `trials.jsonl`, or many).
161
- * @returns {{trials:number, scopes:number, scopes_multi_trial:number, improvement_rate:number,
162
- * monotone_rate:number, sawtooth_count:number, mean_trials_to_green:(number|null),
163
- * status_histogram:Object<string,number>, per_scope:Array<object>}} The ratchet report.
164
- */
165
- export function ratchetReport(trials) {
166
- const byScope = new Map();
167
- const status_histogram = {};
168
- for (const t of trials || []) {
169
- status_histogram[t.status] = (status_histogram[t.status] || 0) + 1;
170
- if (!byScope.has(t.scope_id)) byScope.set(t.scope_id, []);
171
- byScope.get(t.scope_id).push(t);
172
- }
173
-
174
- let afterFirst = 0, kept = 0, sawtooth_count = 0, monotone = 0, multi = 0;
175
- const toGreen = [];
176
- const per_scope = [];
177
-
178
- for (const [scope_id, rows] of [...byScope.entries()].sort(([a], [b]) => String(a).localeCompare(String(b)))) {
179
- const seq = rows.slice().sort((a, b) => a.trial - b.trial);
180
- // "Improvement rate" is measured over trials AFTER the first: the first trial is the baseline
181
- // by construction and counting it as an improvement would inflate every run to ≥ 1/1.
182
- const later = seq.slice(1);
183
- afterFirst += later.length;
184
- kept += later.filter((t) => t.status === "kept").length;
185
- for (let i = 1; i < seq.length; i++) {
186
- if (seq[i].status === "reverted" && seq[i - 1].status === "kept") sawtooth_count++;
187
- }
188
- let scopeMonotone = true;
189
- if (seq.length >= 2) {
190
- multi++;
191
- for (let i = 1; i < seq.length; i++) {
192
- // A `rebased` step changed the denominator, so it is not a decrease — it is a new series.
193
- if (seq[i].status === "reverted" || seq[i].status === "crash") { scopeMonotone = false; break; }
194
- }
195
- if (scopeMonotone) monotone++;
196
- }
197
- // A score with no `regressions` field was not measured for regressions — T0 writes one only when
198
- // it has a baseline to regress against — which is not the same as having some. Requiring an
199
- // explicit 0 reported "no scope reached green" over a run whose every scope was green.
200
- const greenAt = seq.findIndex((t) => t.score && t.score.fixtures_total > 0 && t.score.fixtures_passed === t.score.fixtures_total && (t.score.regressions ?? 0) === 0);
201
- if (greenAt !== -1) toGreen.push(greenAt + 1);
202
- per_scope.push({
203
- scope_id, trials: seq.length,
204
- first_score: seq[0]?.score ?? null,
205
- last_score: seq[seq.length - 1]?.score ?? null,
206
- monotone: seq.length >= 2 ? scopeMonotone : null,
207
- reached_green_at_trial: greenAt === -1 ? null : greenAt + 1,
208
- });
209
- }
210
-
211
- return {
212
- trials: (trials || []).length,
213
- scopes: byScope.size,
214
- scopes_multi_trial: multi,
215
- improvement_rate: afterFirst > 0 ? round2(kept / afterFirst) : 0,
216
- monotone_rate: multi > 0 ? round2(monotone / multi) : 0,
217
- sawtooth_count,
218
- mean_trials_to_green: toGreen.length ? round2(toGreen.reduce((s, n) => s + n, 0) / toGreen.length) : null,
219
- status_histogram,
220
- per_scope,
221
- };
222
- }
223
-
224
- /**
225
- * `--hooks` — do the enforcement points actually fire?
226
- *
227
- * For a hook, "never had to fire" and "never ran" used to produce the same evidence: exit 0,
228
- * empty stdout. With a decision row per evaluation, a hook that sat inert and a hook that
229
- * inspected-and-permitted become SEPARABLE FACTS rather than the same blank.
230
- *
231
- * @param {Array<object>} decisions - Rows from `.shapeup/decisions.jsonl`.
232
- * @returns {{evaluations:number, hooks:number, per_hook:Array<object>}} Per-hook fire, allow, deny,
233
- * block and error counts, plus the rules that fired.
234
- */
235
- export function hooksReport(decisions) {
236
- const byHook = new Map();
237
- for (const d of decisions || []) {
238
- const name = d.hook || "unknown";
239
- if (!byHook.has(name)) byHook.set(name, { hook: name, evaluations: 0, allow: 0, deny: 0, block: 0, error: 0, rules: {} });
240
- const h = byHook.get(name);
241
- h.evaluations++;
242
- if (h[d.verdict] !== undefined) h[d.verdict]++;
243
- if (d.rule) h.rules[d.rule] = (h.rules[d.rule] || 0) + 1;
244
- }
245
- const per_hook = [...byHook.values()].sort((a, b) => a.hook.localeCompare(b.hook));
246
- return { evaluations: (decisions || []).length, hooks: per_hook.length, per_hook };
247
- }
248
-
249
- /**
250
- * Read every trial ledger under a run root (or all runs).
251
- * @param {string} cwd - Project root.
252
- * @param {(string|null)} [slug] - Restrict to one run; null reads every run's ledger.
253
- * @returns {Array<object>} TrialRow records, in file order per run.
254
- */
255
- export function readAllTrials(cwd, slug = null) {
256
- const root = localDir(cwd);
257
- if (!existsSync(root)) return [];
258
- let slugs;
259
- try { slugs = slug ? [slug] : readdirSync(root); } catch { return []; }
260
- const out = [];
261
- for (const s of slugs) {
262
- const p = join(root, s, "t0", "trials.jsonl");
263
- if (!existsSync(p)) continue;
264
- try {
265
- for (const line of readFileSync(p, "utf8").split("\n")) {
266
- if (!line.trim()) continue;
267
- try { out.push(JSON.parse(line)); } catch { /* a torn row is not a trial */ }
268
- }
269
- } catch { /* unreadable ledger → skip */ }
270
- }
271
- return out;
272
- }
273
-
274
- /**
275
- * Read the checkout-wide decision ledger.
276
- * @param {string} cwd - Project root.
277
- * @returns {Array<object>} Decision rows; [] when no hook has ever recorded one.
278
- */
279
- export function readDecisions(cwd) {
280
- const p = decisionsPath(cwd);
281
- if (!existsSync(p)) return [];
282
- try {
283
- return readFileSync(p, "utf8").split("\n").filter((l) => l.trim())
284
- .map((l) => { try { return JSON.parse(l); } catch { return null; } })
285
- .filter(Boolean);
286
- } catch { return []; }
287
- }
288
-
289
- /**
290
- * Decision ledgers that landed OUTSIDE the project root — the trace a hook left when it filed
291
- * under the shell's working directory instead of the project's.
292
- *
293
- * The hooks now resolve the root themselves, so a stray is either history from before that fix or
294
- * evidence the resolver missed a layout; either way the rows in it are missing from the ledger
295
- * `--hooks` reads, and a count that silently omits them is the inert-layer signature this report
296
- * exists to expose. Bounded walk: depth-limited, skipping dependency and VCS directories.
297
- *
298
- * @param {string} cwd - Project root.
299
- * @param {number} [maxDepth=8] - How deep to look.
300
- * @returns {Array<{path:string, rows:number}>} Each stray ledger and its row count.
301
- */
302
- export function strayLedgers(cwd, maxDepth = 8) {
303
- const SKIP = new Set(["node_modules", ".git", ".hg", ".svn", ".hvigor", ".idea", "dist", "build", "target", ".next", ".cache"]);
304
- const out = [];
305
- const home = decisionsPath(cwd);
306
- /**
307
- * Visit one directory level.
308
- * @param {string} dir - Directory to scan.
309
- * @param {number} depth - Its depth below `cwd`.
310
- * @returns {void}
311
- */
312
- const walk = (dir, depth) => {
313
- if (depth > maxDepth) return;
314
- let entries;
315
- try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
316
- for (const e of entries) {
317
- if (!e.isDirectory() || SKIP.has(e.name)) continue;
318
- const sub = join(dir, e.name);
319
- const candidate = decisionsPath(sub);
320
- if (candidate !== home && existsSync(candidate)) {
321
- let rows = 0;
322
- try { rows = readFileSync(candidate, "utf8").split("\n").filter((l) => l.trim()).length; } catch { /* unreadable */ }
323
- out.push({ path: relative(cwd, candidate).split(sep).join("/"), rows });
324
- }
325
- walk(sub, depth + 1);
326
- }
327
- };
328
- walk(cwd, 1);
329
- return out;
330
- }
331
-
332
- /**
333
- * Render a StatsReport as a human-readable fixed-width table.
334
- * @param {object} report - A validated StatsReport (see {@link aggregate}).
335
- * @returns {string} The multi-line table text (header + one row per slug + optional trend line).
336
- */
337
- function renderTable(report) {
338
- const lines = [];
339
- lines.push(`metrics: ${report.metrics_dir || "(none)"} — ${report.rows_total} run row(s), ` +
340
- `${report.rows_pathology} pathology, ${report.rows_malformed} malformed (skipped)`);
341
- if (Object.keys(report.pathologies).length > 0) {
342
- lines.push(`pathologies: ${Object.entries(report.pathologies).map(([k, n]) => `${k}=${n}`).join(" ")}`);
343
- }
344
- if (report.per_slug.length === 0) {
345
- lines.push("(no harvest rows yet — rows appear at SHIP S.6)");
346
- return lines.join("\n");
347
- }
348
- lines.push("");
349
- lines.push("slug runs rounds(min/avg/max) cut-rate exhaust qa(prom/total)");
350
- for (const s of report.per_slug) {
351
- const r = s.rounds ? `${s.rounds.min}/${s.rounds.avg}/${s.rounds.max}` : "—";
352
- lines.push(
353
- `${s.feature_slug.padEnd(28).slice(0, 28)}${String(s.runs).padEnd(6)}` +
354
- `${r.padEnd(21)}${String(s.hammer_cut_rate).padEnd(10)}${String(s.attempt_exhaustions).padEnd(9)}` +
355
- `${s.qa.promoted}/${s.qa.total}`
356
- );
357
- }
358
- if (report.trend.length > 1) {
359
- lines.push("");
360
- lines.push("round_count trend: " + report.trend.map((t) => `${t.feature_slug}:${t.round_count}`).join(" → "));
361
- }
362
- return lines.join("\n");
363
- }
364
-
365
- // --- CLI -----------------------------------------------------------------------
366
-
367
- /** The typed argv contract (see `./lib/argv.mjs`). */
368
- export const ARGV_SPEC = {
369
- usage: "harness.mjs probe stats [--cwd <dir>] [--metrics-dir <dir>] [--slug <slug>] [--format json|table] " +
370
- "[--ratchet] [--hooks]",
371
- _: { arity: 0, max: 0, name: "(no positional operands)" },
372
- cwd: { type: "path" },
373
- "metrics-dir": { type: "path" },
374
- slug: { type: "str" },
375
- format: { type: "enum", values: ["json", "table"], default: "json" },
376
- ratchet: { type: "flag" },
377
- hooks: { type: "flag" },
378
- };
379
-
380
- /**
381
- * Format one score vector for the ratchet table.
382
- * @param {(object|null)} x - A T0Score, or null.
383
- * @returns {string} e.g. "4/5" or "2/5 +1reg", or "—" when absent.
384
- */
385
- function fmtScore(x) {
386
- return x ? `${x.fixtures_passed}/${x.fixtures_total}${x.regressions ? ` +${x.regressions}reg` : ""}` : "—";
387
- }
388
-
389
- /**
390
- * Render the ratchet report as text.
391
- * @param {object} r - Output of {@link ratchetReport}.
392
- * @returns {string} The multi-line report.
393
- */
394
- function renderRatchet(r) {
395
- const lines = [
396
- `ratchet: ${r.trials} trial(s) across ${r.scopes} scope(s); ${r.scopes_multi_trial} with ≥2 trials`,
397
- ` improvement_rate ${r.improvement_rate} (kept ÷ trials after the first)`,
398
- ` monotone_rate ${r.monotone_rate} (scopes whose score never decreased)`,
399
- ` sawtooth_count ${r.sawtooth_count} (a revert immediately after a keep)`,
400
- ` mean_trials_to_green ${r.mean_trials_to_green ?? "—"}`,
401
- ];
402
- const hist = Object.entries(r.status_histogram);
403
- if (hist.length) lines.push(` status: ${hist.map(([k, n]) => `${k}=${n}`).join(" ")}`);
404
- if (r.scopes_multi_trial === 0) {
405
- lines.push("", "(no scope has a second trial yet — the Day-1 question needs at least one retry to answer)");
406
- }
407
- for (const s of r.per_scope) {
408
- lines.push(` ${String(s.scope_id).padEnd(16)} ${String(s.trials).padStart(2)} trial(s) ` +
409
- `${fmtScore(s.first_score)} → ${fmtScore(s.last_score)} ` +
410
- `${s.monotone === null ? "" : s.monotone ? "monotone" : "sawtooth"}${s.reached_green_at_trial ? ` green@t${s.reached_green_at_trial}` : ""}`);
411
- }
412
- return lines.join("\n");
413
- }
414
-
415
- /**
416
- * Render the hook report as text.
417
- * @param {object} r - Output of {@link hooksReport}.
418
- * @returns {string} The multi-line report.
419
- */
420
- function renderHooks(r) {
421
- const lines = [`hooks: ${r.evaluations} evaluation(s) across ${r.hooks} hook(s)`, ""];
422
- lines.push("hook evals allow deny block error");
423
- for (const h of r.per_hook) {
424
- lines.push(`${h.hook.padEnd(26)}${String(h.evaluations).padEnd(7)}${String(h.allow).padEnd(7)}` +
425
- `${String(h.deny).padEnd(6)}${String(h.block).padEnd(7)}${h.error}`);
426
- }
427
- if (r.evaluations === 0) {
428
- lines.push("", "(zero rows: either no hook has run in this checkout, or the enforcement layer is inert —");
429
- lines.push(" and that distinction is exactly what this ledger exists to make.)");
430
- }
431
- if (Array.isArray(r.stray_ledgers) && r.stray_ledgers.length) {
432
- lines.push("", `stray ledgers: ${r.stray_ledgers.length} decisions.jsonl outside the project root — rows not counted above:`);
433
- for (const s of r.stray_ledgers) lines.push(` ${s.path} (${s.rows} row(s))`);
434
- }
435
- return lines.join("\n");
436
- }
437
-
438
- /**
439
- * Aggregate the metric shards, or report the ratchet and hook ledgers.
440
- *
441
- * @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
442
- * @returns {(Promise<void>|void)} Settles when the subcommand has written its output; most paths
443
- * call `process.exit()` with the subcommand's documented code rather than returning.
444
- */
445
- export async function cli(rawArgv) {
446
- const args = runArgs(ARGV_SPEC, rawArgv);
447
- const cwd = resolve(args.cwd || process.cwd());
448
- const metricsDir = args.metricsDir ? resolve(cwd, args.metricsDir) : metricsDirPath(cwd);
449
- const format = args.format;
450
-
451
- // The two exit measurements are separate modes: each reads a different ledger, and neither is a
452
- // StatsReport (which is schema-locked to the harvest shards).
453
- if (args.ratchet || args.hooks) {
454
- const out = {};
455
- if (args.ratchet) out.ratchet = ratchetReport(readAllTrials(cwd, args.slug ?? null));
456
- if (args.hooks) out.hooks = { ...hooksReport(readDecisions(cwd)), stray_ledgers: strayLedgers(cwd) };
457
- if (format === "table") {
458
- const parts = [];
459
- if (out.ratchet) parts.push(renderRatchet(out.ratchet));
460
- if (out.hooks) parts.push(renderHooks(out.hooks));
461
- console.log(parts.join("\n\n"));
462
- } else {
463
- console.log(JSON.stringify(out, null, 2));
464
- }
465
- process.exit(0);
466
- }
467
-
468
- const report = aggregate(readShards(metricsDir), { metricsDir, slugFilter: args.slug ?? null });
469
-
470
- const { valid, errors } = validate(report, { $ref: "domain.schema.json#/$defs/StatsReport" });
471
- if (!valid) {
472
- console.error(` ✗ stats report drifted from domain.schema.json#/$defs/StatsReport:`);
473
- for (const e of errors) console.error(` ✗ ${e}`);
474
- process.exit(1);
475
- }
476
- console.log(format === "table" ? renderTable(report) : JSON.stringify(report, null, 2));
477
- }