rungate 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +209 -0
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-diff.ts +139 -0
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +347 -0
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +18 -1
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +196 -20
- package/lib/config-loader.ts +51 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1272 -189
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/context-docs.ts +143 -28
- package/lib/create-brief.ts +2 -1
- package/lib/create-sc.ts +323 -4
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/doc-scan-targets.ts +9 -0
- package/lib/evidence-command-transport.ts +160 -0
- package/lib/evidence-prevalidator.ts +246 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +286 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-refusal-currency.ts +112 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/gate-timeouts.ts +82 -0
- package/lib/generators/agent-briefs.ts +4 -1
- package/lib/generators/agents-md.ts +204 -48
- package/lib/generators/code-map.ts +2 -1
- package/lib/generators/types.ts +8 -0
- package/lib/git-dates.ts +55 -0
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +536 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/local-calendar-day.ts +78 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/project-state-shape.ts +59 -0
- package/lib/promote-outputs.ts +6 -2
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/report-provenance.ts +156 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-claim-forms.ts +75 -0
- package/lib/sc-id-cross-tree.ts +323 -0
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4777 -0
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +186 -39
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/rehearsal.ts +205 -71
- package/lib/scaffold/steps.ts +1517 -293
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +10 -4
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +80 -62
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/spec-listing.ts +148 -0
- package/lib/spec-requirements.ts +209 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +411 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-command.ts +133 -0
- package/lib/test-suite-lock.ts +325 -26
- package/lib/tool-probe.ts +142 -0
- package/lib/top-level-cd.ts +393 -0
- package/lib/tracked-vs-ignored.ts +234 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/validators/spec-validators.ts +3 -2
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/audit-specs.ts +247 -12
- package/scripts/audit-transcript.ts +2 -1
- package/scripts/convert-spec.ts +2 -1
- package/scripts/create-sc.ts +37 -7
- package/scripts/create-spec.ts +2 -1
- package/scripts/create-success.ts +2 -1
- package/scripts/extract-constraints.ts +197 -26
- package/scripts/generate-code-map.ts +62 -28
- package/scripts/scaffold-project.ts +12 -2
- package/scripts/split-spec.ts +2 -1
- package/scripts/sync-sc-status.ts +127 -14
- package/scripts/sync-spec-tests.ts +2 -1
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #617 / #619 — the one place that decides what a top-level `docs/*.md`
|
|
3
|
+
* contributes to the Documentation Routing table.
|
|
4
|
+
*
|
|
5
|
+
* ## What was wrong
|
|
6
|
+
*
|
|
7
|
+
* Both doc-routing sites ended with a "filter to non-obvious mappings":
|
|
8
|
+
*
|
|
9
|
+
* return docRouting.filter(d => {
|
|
10
|
+
* const slug = d.file.replace(/\.md$/, "").replace(/.*\//, "")
|
|
11
|
+
* .toLowerCase().replace(/-/g, " ");
|
|
12
|
+
* return !d.need.toLowerCase().startsWith(slug)
|
|
13
|
+
* || d.need.includes("—") || d.need.includes("(");
|
|
14
|
+
* });
|
|
15
|
+
*
|
|
16
|
+
* `need` falls back to the filename with dashes turned into spaces when a
|
|
17
|
+
* document carries no `governs:`, so `need === slug` and the row was dropped.
|
|
18
|
+
* Measured on one consumer: **0 of 38 top-level `docs/*.md`
|
|
19
|
+
* carried the field, so all 38 were dropped** — `ENVIRONMENTS.md` among them,
|
|
20
|
+
* which cost a DA a false P1 on 2026-10-17 after it inferred the production
|
|
21
|
+
* host rather than reading the document that names it in three places.
|
|
22
|
+
*
|
|
23
|
+
* The filter's instinct is right — `| environments | docs/ENVIRONMENTS.md |`
|
|
24
|
+
* teaches nothing about *when* to read the file. Its action is wrong: dropping
|
|
25
|
+
* the row does not produce a better row, it produces no evidence the document
|
|
26
|
+
* exists. That is `.claude/rules/config-access.md`'s *Silent Default
|
|
27
|
+
* Substitution* — **a document with no `governs:` is UNMEASURED, not
|
|
28
|
+
* "governs nothing"**, and a dropped row is indistinguishable from a document
|
|
29
|
+
* that was never written.
|
|
30
|
+
*
|
|
31
|
+
* ## What it does instead
|
|
32
|
+
*
|
|
33
|
+
* - Never drops a document for being quiet. The row is emitted with
|
|
34
|
+
* `NO_GOVERNS_MARKER` in front of the filename-derived description, so the
|
|
35
|
+
* gap reads as a to-do rather than as an answer — the same treatment
|
|
36
|
+
* `specs-routing.md` already gives a spec with no criteria.
|
|
37
|
+
* - Exclusion is something a document declares **about itself**
|
|
38
|
+
* (`status: archived`, `routing: exclude`), not a string-similarity accident.
|
|
39
|
+
* Without a deliberate way out, every consumer gets rows it does not want
|
|
40
|
+
* and the next person widens the filter again.
|
|
41
|
+
*
|
|
42
|
+
* ## Why this is a module and not two edits
|
|
43
|
+
*
|
|
44
|
+
* `lib/scanner.ts` and `lib/scaffold/steps.ts` each held a complete copy of
|
|
45
|
+
* this scan, each with its own inline `governs:` regex. #91 is what that costs:
|
|
46
|
+
* the scaffold calls the steps.ts copy, so a fix to scanner.ts alone changed
|
|
47
|
+
* no generated output at all while the scanner tests went green over the copy
|
|
48
|
+
* nobody calls. Two deciders answering one question are behaviourally one —
|
|
49
|
+
* a mutation of either leaves the other holding every case up, and both read
|
|
50
|
+
* as covered (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape).
|
|
51
|
+
*
|
|
52
|
+
* So there is exactly ONE decider here, `topLevelDocRoute`, and exactly one
|
|
53
|
+
* frontmatter parser underneath it, `lib/frontmatter.ts` `parseFrontmatter`.
|
|
54
|
+
* The obvious-mapping filter is deleted rather than kept-but-defeated: the
|
|
55
|
+
* marker already contains an em dash, so leaving the filter in place would
|
|
56
|
+
* make it dead code that no mutation could distinguish from live code.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import { existsSync, readdirSync, readFileSync } from "fs";
|
|
60
|
+
import { join } from "path";
|
|
61
|
+
import { parseFrontmatter } from "./frontmatter";
|
|
62
|
+
|
|
63
|
+
export interface DocRoutingRow {
|
|
64
|
+
need: string;
|
|
65
|
+
file: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The marker that makes an undiscoverable document visible. It leads the row
|
|
70
|
+
* so a reader scanning the first column sees the gap without reading on.
|
|
71
|
+
*/
|
|
72
|
+
export const NO_GOVERNS_MARKER = "NO GOVERNS FIELD —";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The frontmatter declarations by which a document takes itself out of the
|
|
76
|
+
* routing table. One table, so there is one place to add a spelling and one
|
|
77
|
+
* place to mutate.
|
|
78
|
+
*/
|
|
79
|
+
export const ROUTING_EXCLUSIONS: ReadonlyArray<{ key: string; value: string }> = [
|
|
80
|
+
{ key: "status", value: "archived" },
|
|
81
|
+
{ key: "routing", value: "exclude" },
|
|
82
|
+
];
|
|
83
|
+
|
|
84
|
+
/** A `governs:` holding this is a placeholder, not a statement. */
|
|
85
|
+
const GOVERNS_PLACEHOLDER = "TODO";
|
|
86
|
+
|
|
87
|
+
export type DocRoutingReading = "GOVERNS" | "UNMEASURED";
|
|
88
|
+
|
|
89
|
+
export type TopLevelDocDecision =
|
|
90
|
+
| { routed: true; need: string; reading: DocRoutingReading }
|
|
91
|
+
| { routed: false; excludedBy: string };
|
|
92
|
+
|
|
93
|
+
/** `MAC-MINI-DEMO-SETUP.md` → `mac mini demo setup`. */
|
|
94
|
+
export function filenameDescription(filename: string): string {
|
|
95
|
+
return filename.replace(/\.md$/, "").replace(/.*\//, "").replace(/-/g, " ").toLowerCase();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The single decider: what, if anything, this document contributes.
|
|
100
|
+
*
|
|
101
|
+
* A frontmatter block `parseFrontmatter` refuses is UNMEASURED, never
|
|
102
|
+
* excluded — silence from the parser must not read as a document opting out.
|
|
103
|
+
*/
|
|
104
|
+
export function topLevelDocRoute(filename: string, content: string): TopLevelDocDecision {
|
|
105
|
+
const parsed = parseFrontmatter(content);
|
|
106
|
+
const entries = parsed?.entries ?? [];
|
|
107
|
+
const valueOf = (key: string): string | null => {
|
|
108
|
+
const e = entries.find(x => x.key === key);
|
|
109
|
+
return e ? e.inline.trim() : null;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
for (const ex of ROUTING_EXCLUSIONS) {
|
|
113
|
+
if (valueOf(ex.key)?.toLowerCase() === ex.value) {
|
|
114
|
+
return { routed: false, excludedBy: `${ex.key}: ${ex.value}` };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const governs = valueOf("governs");
|
|
119
|
+
if (governs && governs !== GOVERNS_PLACEHOLDER) {
|
|
120
|
+
return { routed: true, need: governs, reading: "GOVERNS" };
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
routed: true,
|
|
124
|
+
need: `${NO_GOVERNS_MARKER} ${filenameDescription(filename)}`,
|
|
125
|
+
reading: "UNMEASURED",
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Every row `docs/` contributes: top-level documents, then subdirectories the
|
|
131
|
+
* caller has not already listed as a category.
|
|
132
|
+
*
|
|
133
|
+
* The caller owns the categories because the two sites build that list from
|
|
134
|
+
* their own configuration; everything else about the scan lives here.
|
|
135
|
+
*/
|
|
136
|
+
export function scanDocsDirectory(docsDir: string, categoryDirs: readonly string[]): DocRoutingRow[] {
|
|
137
|
+
if (!existsSync(docsDir)) return [];
|
|
138
|
+
const rows: DocRoutingRow[] = [];
|
|
139
|
+
|
|
140
|
+
for (const f of readdirSync(docsDir).filter(f => f.endsWith(".md"))) {
|
|
141
|
+
const decision = topLevelDocRoute(f, readFileSync(join(docsDir, f), "utf-8"));
|
|
142
|
+
if (!decision.routed) continue;
|
|
143
|
+
rows.push({ need: decision.need, file: `docs/${f}` });
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const subDirs = readdirSync(docsDir, { withFileTypes: true })
|
|
147
|
+
.filter(d => d.isDirectory() && !categoryDirs.includes(`docs/${d.name}`))
|
|
148
|
+
.map(d => d.name);
|
|
149
|
+
for (const sub of subDirs) {
|
|
150
|
+
const subFiles = readdirSync(join(docsDir, sub)).filter(f => f.endsWith(".md"));
|
|
151
|
+
if (subFiles.length > 0) {
|
|
152
|
+
rows.push({ need: `${sub} (${subFiles.length} files)`, file: `docs/${sub}/` });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return rows;
|
|
157
|
+
}
|
package/lib/doc-scan-targets.ts
CHANGED
|
@@ -393,6 +393,15 @@ export function planDocScan(projectPath: string): DocScanPlan {
|
|
|
393
393
|
if (real === null) { st.refused.push(path); continue; }
|
|
394
394
|
if (st.seen.has(real)) continue;
|
|
395
395
|
st.seen.add(real);
|
|
396
|
+
// A name is not a file type. `walkMarkdown` asks `statSync` before it
|
|
397
|
+
// decides; this loop asked only the suffix, so a DIRECTORY called
|
|
398
|
+
// `NOTES.md` at the project root was planned as a document and handed
|
|
399
|
+
// to `readFileSync`, which throws EISDIR — the plan claimed a file it
|
|
400
|
+
// could never read (#508). Nothing reports it either way: a directory
|
|
401
|
+
// with a markdown name carries no prose.
|
|
402
|
+
let isFile: boolean;
|
|
403
|
+
try { isFile = statSync(real).isFile(); } catch { continue; }
|
|
404
|
+
if (!isFile) continue;
|
|
396
405
|
if (st.files.length >= MAX_SCAN_FILES) { st.truncatedFiles++; continue; }
|
|
397
406
|
st.files.push(anchored(projectPath, st.rootReal, real));
|
|
398
407
|
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Moving an agent-authored evidence command from discovery to the state file,
|
|
3
|
+
* and recording every legitimate change made to one on the way (#614).
|
|
4
|
+
*
|
|
5
|
+
* ## The defect this module exists for
|
|
6
|
+
*
|
|
7
|
+
* `workflows/ship.js` persisted discovery's ACs by interpolating
|
|
8
|
+
* `JSON.stringify(discovery.acs…)` into a DOUBLE-QUOTED `bun -e "…"` shell
|
|
9
|
+
* string in the setup agent's prompt. Inside double quotes a shell expands
|
|
10
|
+
* `$1` to the empty string, `${VAR}` to the environment, and `$(cmd)` by
|
|
11
|
+
* RUNNING it. On run `wf_c92dd0a7-fe6` the command
|
|
12
|
+
*
|
|
13
|
+
* grep -rhcE '^export function localCalendarDay' lib | awk '{s+=$1} END {print s+0}'
|
|
14
|
+
*
|
|
15
|
+
* arrived in `workflow-state.json` as `awk '{s+=} END {print s+0}'` — an awk
|
|
16
|
+
* syntax error. The AC could never produce evidence, stayed PENDING, and took
|
|
17
|
+
* the verify gate down with four failures. The blast radius was every evidence
|
|
18
|
+
* command containing a `$`.
|
|
19
|
+
*
|
|
20
|
+
* The fix is to stop putting agent-authored text where a shell looks. The
|
|
21
|
+
* payload leaves the generator as a SINGLE-QUOTED argument — the one shell
|
|
22
|
+
* construct with no expansion inside it at all — and the `bun -e` program
|
|
23
|
+
* reads it out of `process.argv` instead of carrying it as program text.
|
|
24
|
+
*
|
|
25
|
+
* ## Why the provenance half is in the same module
|
|
26
|
+
*
|
|
27
|
+
* Three places legitimately rewrite an evidence command: discovery's path
|
|
28
|
+
* normaliser, discovery's evidence-type-ratio upgrade, and the git-ref
|
|
29
|
+
* qualification at the execution boundary (#118). Each used to overwrite
|
|
30
|
+
* `evidenceMethod.command` in place, so a reader of a broken command could not
|
|
31
|
+
* tell a repair from a corruption — which is exactly the question #614 took a
|
|
32
|
+
* run and a post-mortem to answer. `recordEvidenceRewrite` is the single site
|
|
33
|
+
* that changes a command, and it keeps the FIRST authored form.
|
|
34
|
+
*
|
|
35
|
+
* It belongs here because it is the same claim from the other side: the first
|
|
36
|
+
* half says nothing may change the command by accident, the second says
|
|
37
|
+
* everything that changes it on purpose must say so.
|
|
38
|
+
*
|
|
39
|
+
* `workflows/ship.js` cannot import (#69). It carries an inlined copy inside
|
|
40
|
+
* the `AC-PAYLOAD-TRANSPORT` markers, and `test/evidence-command-roundtrip.test.ts`
|
|
41
|
+
* runs both over one matrix so drift is red.
|
|
42
|
+
*/
|
|
43
|
+
import { shellSingleQuote } from "./phase2-decomposition";
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Where the payload arrives inside the `bun -e` program.
|
|
47
|
+
*
|
|
48
|
+
* `bun -e '<program>' a b` gives the program `process.argv[1] === "a"`.
|
|
49
|
+
*/
|
|
50
|
+
export const AC_PAYLOAD_ARGV_INDEX = 1;
|
|
51
|
+
|
|
52
|
+
/** One recorded change to an evidence command. */
|
|
53
|
+
export interface EvidenceRewrite {
|
|
54
|
+
/** The site that made the change, named so a reader can go and read it. */
|
|
55
|
+
by: string;
|
|
56
|
+
/** The command as it stood before this change. */
|
|
57
|
+
from: string;
|
|
58
|
+
/** The command as it stands after it. */
|
|
59
|
+
to: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The AC fields the setup step persists into workflow-state.json. */
|
|
63
|
+
export interface AcPayloadRecord {
|
|
64
|
+
id: unknown;
|
|
65
|
+
type: unknown;
|
|
66
|
+
statement: unknown;
|
|
67
|
+
threshold: unknown;
|
|
68
|
+
evidenceMethod: unknown;
|
|
69
|
+
specElement: unknown;
|
|
70
|
+
coversCriteria: unknown[];
|
|
71
|
+
contextFiles: unknown[];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Project discovery's ACs onto the fields `writeACs` persists.
|
|
76
|
+
*
|
|
77
|
+
* Kept as a function rather than inlined at the two call sites because the
|
|
78
|
+
* two used to be two copies of the same object literal, and a field added to
|
|
79
|
+
* one of them would have been silently missing from the fix-acs fallback.
|
|
80
|
+
*/
|
|
81
|
+
export function acPayloadRecords(acs: unknown): AcPayloadRecord[] {
|
|
82
|
+
return (Array.isArray(acs) ? acs : []).map((ac: any) => ({
|
|
83
|
+
id: ac.id,
|
|
84
|
+
type: ac.type,
|
|
85
|
+
statement: ac.statement,
|
|
86
|
+
threshold: ac.threshold,
|
|
87
|
+
evidenceMethod: ac.evidenceMethod,
|
|
88
|
+
specElement: ac.specElement,
|
|
89
|
+
coversCriteria: ac.coversCriteria || [],
|
|
90
|
+
contextFiles: ac.contextFiles || [],
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The payload exactly as it will be written to disk. */
|
|
95
|
+
export function acPayloadJson(acs: unknown): string {
|
|
96
|
+
return JSON.stringify(acPayloadRecords(acs));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The payload as a shell WORD, to sit after the closing quote of the
|
|
101
|
+
* `bun -e "…"` program.
|
|
102
|
+
*
|
|
103
|
+
* Single quotes are the one shell construct with no expansion inside them at
|
|
104
|
+
* all, so this is the only position in the command where agent-authored text
|
|
105
|
+
* is safe. `shellSingleQuote` closes, escapes and reopens on an apostrophe,
|
|
106
|
+
* which is the single character the construct cannot carry verbatim — and
|
|
107
|
+
* evidence commands are full of apostrophes, because that is how an awk or
|
|
108
|
+
* grep expression is quoted.
|
|
109
|
+
*
|
|
110
|
+
* ## Why an argument and not a file
|
|
111
|
+
*
|
|
112
|
+
* The obvious alternative — write the payload to `${WORK_DIR}/…json` and have
|
|
113
|
+
* the `bun -e` read it — was built first and does not work, for a reason
|
|
114
|
+
* worth recording rather than rediscovering. `initWorkflow` calls `beginRun`,
|
|
115
|
+
* whose whole job (#288) is to sweep the work directory into
|
|
116
|
+
* `previous-run-<id>/` so a dead run's artefacts cannot be read as this
|
|
117
|
+
* run's. A payload written before `initWorkflow` is archived by it, and the
|
|
118
|
+
* read that follows is an ENOENT; a payload written after it needs a third
|
|
119
|
+
* command in the agent's prompt. The argument has neither problem: one
|
|
120
|
+
* process, no new file in a swept directory, and no ordering to get wrong.
|
|
121
|
+
*/
|
|
122
|
+
export function acPayloadShellArgument(acs: unknown): string {
|
|
123
|
+
return shellSingleQuote(acPayloadJson(acs));
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The JavaScript expression that reads the payload back, for embedding in the
|
|
128
|
+
* `bun -e` program that calls `writeACs`.
|
|
129
|
+
*
|
|
130
|
+
* It contains no agent-authored text and no `$`, so it is safe inside the
|
|
131
|
+
* double-quoted shell string the setup step still is.
|
|
132
|
+
*/
|
|
133
|
+
export function acPayloadReadExpression(): string {
|
|
134
|
+
return `JSON.parse(process.argv[${AC_PAYLOAD_ARGV_INDEX}])`;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Change an evidence command, on the record.
|
|
139
|
+
*
|
|
140
|
+
* Returns true when the command actually changed. A no-op records nothing: a
|
|
141
|
+
* recorder that logged every call would leave `rewrites` non-empty on every
|
|
142
|
+
* AC and the field would stop meaning "this command was changed".
|
|
143
|
+
*
|
|
144
|
+
* `authoredCommand` is written once and never overwritten. A field holding
|
|
145
|
+
* "whatever it was one rewrite ago" cannot answer the question #614 asks,
|
|
146
|
+
* which is what the agent wrote.
|
|
147
|
+
*/
|
|
148
|
+
export function recordEvidenceRewrite(
|
|
149
|
+
evidenceMethod: { command?: string; authoredCommand?: string; rewrites?: EvidenceRewrite[] } | undefined | null,
|
|
150
|
+
by: string,
|
|
151
|
+
next: string,
|
|
152
|
+
): boolean {
|
|
153
|
+
if (!evidenceMethod || typeof evidenceMethod.command !== "string") return false;
|
|
154
|
+
const from = evidenceMethod.command;
|
|
155
|
+
if (from === next) return false;
|
|
156
|
+
if (evidenceMethod.authoredCommand === undefined) evidenceMethod.authoredCommand = from;
|
|
157
|
+
evidenceMethod.command = next;
|
|
158
|
+
evidenceMethod.rewrites = [...(evidenceMethod.rewrites || []), { by, from, to: next }];
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
@@ -26,9 +26,57 @@ export interface ACInput {
|
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
/** The AC's own bar, recorded verbatim beside the output it will be judged against. */
|
|
30
|
+
export interface ACThresholdInput {
|
|
31
|
+
op?: string;
|
|
32
|
+
value?: string | number;
|
|
33
|
+
unit?: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Whether the baseline output is a value a threshold can be compared against.
|
|
38
|
+
*
|
|
39
|
+
* MEASURED means the command printed something that stands for a reading of
|
|
40
|
+
* the tree. UNMEASURED means it did not — it printed nothing, or it printed
|
|
41
|
+
* only the tool announcing itself. Those two states are the whole point: `0`,
|
|
42
|
+
* `{}`, an empty string and `bun test v1.3.14 (d1632b29)` are all
|
|
43
|
+
* indistinguishable from success, and UNMEASURED is not.
|
|
44
|
+
*/
|
|
45
|
+
export type BaselineVerdict = "MEASURED" | "UNMEASURED";
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* What an AC's evidence command printed on the PRE-CHANGE tree, beside the
|
|
49
|
+
* threshold it will later be judged against (#669).
|
|
50
|
+
*
|
|
51
|
+
* Both halves are persisted so neither has to be re-derived: the pre-change
|
|
52
|
+
* tree stops existing the moment Marcus writes a line, so a reader asking
|
|
53
|
+
* "what did this print before the work" had no way to find out that did not
|
|
54
|
+
* involve a checkout.
|
|
55
|
+
*/
|
|
56
|
+
export interface ACBaseline {
|
|
57
|
+
verdict: BaselineVerdict;
|
|
58
|
+
/** The command's own output, verbatim and trimmed. Empty only when UNMEASURED. */
|
|
59
|
+
output: string;
|
|
60
|
+
/** Why the baseline is UNMEASURED. Null — and only null — when it is MEASURED. */
|
|
61
|
+
reason: string | null;
|
|
62
|
+
/** The AC's threshold as written. Null when the AC declares none. */
|
|
63
|
+
threshold: ACThresholdInput | null;
|
|
64
|
+
/**
|
|
65
|
+
* Whether the PRE-CHANGE output already clears the bar.
|
|
66
|
+
*
|
|
67
|
+
* Null — and only null — when no comparison was made: an UNMEASURED
|
|
68
|
+
* baseline is never reduced to one, and neither is a bar nothing can be
|
|
69
|
+
* compared against. `false` would claim the comparison happened and missed,
|
|
70
|
+
* which is a different fact and the one that hides a banner.
|
|
71
|
+
*/
|
|
72
|
+
satisfiesThreshold: boolean | null;
|
|
73
|
+
}
|
|
74
|
+
|
|
29
75
|
export interface PrevalidationResult {
|
|
30
76
|
id: string;
|
|
31
77
|
status: "ok" | "broken" | "empty" | "skipped";
|
|
78
|
+
/** The pre-change reading (#669). */
|
|
79
|
+
baseline?: ACBaseline;
|
|
32
80
|
autoFixed?: boolean;
|
|
33
81
|
/**
|
|
34
82
|
* The command as the AC was written, present only when `autoFixed` is.
|
|
@@ -57,10 +105,173 @@ export interface EvidencePrevalidationReading {
|
|
|
57
105
|
checkedAt: string;
|
|
58
106
|
/** Per-AC results. Empty — and only empty — when the verdict is UNMEASURED. */
|
|
59
107
|
acs: PrevalidationResult[];
|
|
108
|
+
/**
|
|
109
|
+
* The ids of the ACs classified `broken`, in the order they were measured.
|
|
110
|
+
*
|
|
111
|
+
* Derived from `acs`, never supplied — `EvidencePrevalidationSchema` refuses
|
|
112
|
+
* a reading whose `broken` disagrees with its own results, so this cannot
|
|
113
|
+
* drift into an independent claim.
|
|
114
|
+
*
|
|
115
|
+
* It exists because the gate's single refusal site,
|
|
116
|
+
* `prevalidationViolations` in gates/workflow.test.ts, names the ACs to fix
|
|
117
|
+
* out of exactly this key — and until #274 no writer produced it. Every
|
|
118
|
+
* genuine FAIL therefore refused with "no broken AC ids were recorded", which
|
|
119
|
+
* is true of the key and false of the reading: the per-AC results were right
|
|
120
|
+
* there under `acs`. Fail-closed, so nothing shipped that should not have,
|
|
121
|
+
* but a refusal the heal loop cannot act on costs a round and teaches
|
|
122
|
+
* nothing, which is part of how #261 cost two.
|
|
123
|
+
*/
|
|
124
|
+
broken: string[];
|
|
125
|
+
/**
|
|
126
|
+
* The ids whose PRE-CHANGE output already satisfied their own threshold
|
|
127
|
+
* (#669).
|
|
128
|
+
*
|
|
129
|
+
* Derived from `acs` by `alreadySatisfiedIds`, the same binding the gate's
|
|
130
|
+
* refusal reads, so the list the reading carries and the list the refusal
|
|
131
|
+
* names cannot be two different answers to one question.
|
|
132
|
+
*/
|
|
133
|
+
alreadySatisfied: string[];
|
|
60
134
|
/** Why the reading is UNMEASURED. Null — and only null — when it is not. */
|
|
61
135
|
reason: string | null;
|
|
62
136
|
}
|
|
63
137
|
|
|
138
|
+
// ── The pre-change baseline (#669) ──────────────────────────────────────
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* A line that is a tool announcing itself rather than reporting anything.
|
|
142
|
+
*
|
|
143
|
+
* Deliberately narrow. The direction this is allowed to be wrong in is toward
|
|
144
|
+
* MEASURED: a pattern that swallowed ordinary output would convert real
|
|
145
|
+
* evidence into an absence, which is strictly worse than the state it
|
|
146
|
+
* replaces. So the digits must be introduced by a literal `v` or the word
|
|
147
|
+
* `version` — `3.14` on its own is a number somebody measured, and
|
|
148
|
+
* `bun test v1.3.14 (d1632b29)` is not.
|
|
149
|
+
*/
|
|
150
|
+
const VERSION_BANNER =
|
|
151
|
+
/^(?:[A-Za-z][\w.+-]*(?:[ \t]+[A-Za-z][\w.+-]*){0,2}[ \t]+)?(?:v|[Vv]ersion[ \t]+)\d+\.\d+(?:\.\d+)?[\w.+-]*(?:[ \t]*\([0-9a-f]{6,}\))?$/;
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Why this output is not a measurement, or null if it is.
|
|
155
|
+
*
|
|
156
|
+
* THE single binding that decides it, because two deciders answering one
|
|
157
|
+
* question are behaviourally one and a mutation of either leaves the other
|
|
158
|
+
* holding the case up (`.claude/rules/checks-must-be-able-to-fail.md`).
|
|
159
|
+
*
|
|
160
|
+
* The two unmeasured states carry DIFFERENT sentences on purpose. A command
|
|
161
|
+
* that printed nothing and a command that printed only a banner are fixed by
|
|
162
|
+
* different edits, so one shared reason would be a warning nobody can act on.
|
|
163
|
+
*/
|
|
164
|
+
export function baselineUnmeasuredReason(output: string | null): string | null {
|
|
165
|
+
if (output === null) {
|
|
166
|
+
return "the evidence command was not run, so there is no baseline output to read";
|
|
167
|
+
}
|
|
168
|
+
const trimmed = output.trim();
|
|
169
|
+
if (trimmed.length === 0) {
|
|
170
|
+
return "the command ran and printed nothing, so there is no value to compare against the threshold";
|
|
171
|
+
}
|
|
172
|
+
const lines = trimmed.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
|
|
173
|
+
if (lines.every((l) => VERSION_BANNER.test(l))) {
|
|
174
|
+
return (
|
|
175
|
+
`the only output is a tool version banner (${lines.join(" / ")}), which is the tool ` +
|
|
176
|
+
"announcing itself and not a reading of the tree"
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Does this output clear this bar? Null when no comparison could be made.
|
|
184
|
+
*
|
|
185
|
+
* Null rather than false for an uncomparable pair, because `false` says the
|
|
186
|
+
* bar was tested and missed — a claim about the tree — while null says nobody
|
|
187
|
+
* could test it. Collapsing the two is how an AC with a nonsense threshold
|
|
188
|
+
* reads as an AC that simply has not landed yet.
|
|
189
|
+
*/
|
|
190
|
+
export function thresholdSatisfiedBy(
|
|
191
|
+
output: string,
|
|
192
|
+
threshold: ACThresholdInput | null | undefined,
|
|
193
|
+
): boolean | null {
|
|
194
|
+
const op = threshold?.op;
|
|
195
|
+
if (!op) return null;
|
|
196
|
+
const text = output.trim();
|
|
197
|
+
if (op === "exists") return text.length > 0;
|
|
198
|
+
if (op === "contains") {
|
|
199
|
+
if (threshold?.value === undefined || threshold.value === null) return null;
|
|
200
|
+
return text.includes(String(threshold.value));
|
|
201
|
+
}
|
|
202
|
+
const expected = parseFloat(String(threshold?.value));
|
|
203
|
+
const actual = parseFloat(text);
|
|
204
|
+
if (Number.isNaN(expected) || Number.isNaN(actual) || !/^-?\d+(?:\.\d+)?$/.test(text)) return null;
|
|
205
|
+
switch (op) {
|
|
206
|
+
case ">=": return actual >= expected;
|
|
207
|
+
case "<=": return actual <= expected;
|
|
208
|
+
case ">": return actual > expected;
|
|
209
|
+
case "<": return actual < expected;
|
|
210
|
+
case "==": return actual === expected;
|
|
211
|
+
case "!=": return actual !== expected;
|
|
212
|
+
default: return null;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Compose one AC's pre-change baseline from what its command printed.
|
|
218
|
+
*
|
|
219
|
+
* One site, so the ordering is not re-decided anywhere: the output is
|
|
220
|
+
* classified FIRST, and only a MEASURED output is ever handed to the
|
|
221
|
+
* comparison. That ordering is the whole of #669's third criterion — an
|
|
222
|
+
* UNMEASURED baseline is never reduced to a threshold comparison, and the
|
|
223
|
+
* only way to guarantee that is to make the comparison unreachable from the
|
|
224
|
+
* unmeasured branch rather than to check afterwards that it did not happen.
|
|
225
|
+
*
|
|
226
|
+
* `output` is null when the command never ran at all.
|
|
227
|
+
*/
|
|
228
|
+
export function acBaseline(
|
|
229
|
+
output: string | null,
|
|
230
|
+
threshold: ACThresholdInput | null | undefined,
|
|
231
|
+
): ACBaseline {
|
|
232
|
+
const recorded = threshold ? { ...threshold } : null;
|
|
233
|
+
const reason = baselineUnmeasuredReason(output);
|
|
234
|
+
if (reason !== null) {
|
|
235
|
+
return {
|
|
236
|
+
verdict: "UNMEASURED",
|
|
237
|
+
output: (output ?? "").trim(),
|
|
238
|
+
reason,
|
|
239
|
+
threshold: recorded,
|
|
240
|
+
satisfiesThreshold: null,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
const text = (output as string).trim();
|
|
244
|
+
return {
|
|
245
|
+
verdict: "MEASURED",
|
|
246
|
+
output: text,
|
|
247
|
+
reason: null,
|
|
248
|
+
threshold: recorded,
|
|
249
|
+
satisfiesThreshold: recorded ? thresholdSatisfiedBy(text, recorded) : null,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* The ids whose PRE-CHANGE output already clears their own threshold.
|
|
255
|
+
*
|
|
256
|
+
* An AC in this list cannot observe the work: its command printed a
|
|
257
|
+
* satisfying value before a line was written, so it will print one afterwards
|
|
258
|
+
* whatever Marcus does. That is a check that cannot fail, arriving in the
|
|
259
|
+
* harness's own acceptance criteria.
|
|
260
|
+
*
|
|
261
|
+
* Exported and called by BOTH the writer (`measureEvidencePrevalidation`) and
|
|
262
|
+
* the reader (`prevalidationViolations` in gates/workflow.test.ts), so there
|
|
263
|
+
* is one binding to mutate. `broken` taught this at the cost of a heal round:
|
|
264
|
+
* until #295 the reader derived its own answer, disagreed with the writer, and
|
|
265
|
+
* the refusal named nothing.
|
|
266
|
+
*/
|
|
267
|
+
export function alreadySatisfiedIds(acs: unknown): string[] {
|
|
268
|
+
if (!Array.isArray(acs)) return [];
|
|
269
|
+
return acs
|
|
270
|
+
.filter((ac: unknown) => (ac as { baseline?: ACBaseline })?.baseline?.satisfiesThreshold === true)
|
|
271
|
+
.map((ac: unknown) => String((ac as { id?: unknown })?.id ?? ""))
|
|
272
|
+
.filter((id: string) => id.length > 0);
|
|
273
|
+
}
|
|
274
|
+
|
|
64
275
|
// ── Auto-fix patterns ───────────────────────────────────────────────────
|
|
65
276
|
|
|
66
277
|
/**
|
|
@@ -143,11 +354,17 @@ export async function prevalidateEvidence(
|
|
|
143
354
|
for (const ac of acs) {
|
|
144
355
|
const rawCommand = ac.evidenceMethod?.command;
|
|
145
356
|
|
|
146
|
-
// Skip ACs without a command
|
|
357
|
+
// Skip ACs without a command.
|
|
358
|
+
//
|
|
359
|
+
// It still carries a baseline (#669), and the baseline still says why
|
|
360
|
+
// there is nothing in it. A missing `baseline` key and a command that
|
|
361
|
+
// produced no value read identically to every consumer, which is the
|
|
362
|
+
// confusion this field exists to remove.
|
|
147
363
|
if (!rawCommand) {
|
|
148
364
|
results.push({
|
|
149
365
|
id: ac.id,
|
|
150
366
|
status: "skipped",
|
|
367
|
+
baseline: acBaseline(null, ac.threshold),
|
|
151
368
|
});
|
|
152
369
|
continue;
|
|
153
370
|
}
|
|
@@ -186,8 +403,22 @@ export async function prevalidateEvidence(
|
|
|
186
403
|
* single push site rather than at each branch is what keeps a later
|
|
187
404
|
* auto-fix pattern from quietly landing with only half the pair.
|
|
188
405
|
*/
|
|
406
|
+
/**
|
|
407
|
+
* What the command printed on this tree, or null if it never ran (#669).
|
|
408
|
+
*
|
|
409
|
+
* Captured per AC and read by `record` so the baseline is composed at the
|
|
410
|
+
* SINGLE push site. Stamping it at each branch instead would let a later
|
|
411
|
+
* classification land with the pre-change reading quietly missing, which
|
|
412
|
+
* is the half-pair problem `originalCommand` already has a comment about.
|
|
413
|
+
*/
|
|
414
|
+
let printed: string | null = null;
|
|
415
|
+
|
|
189
416
|
const record = (r: PrevalidationResult): void => {
|
|
190
|
-
|
|
417
|
+
const withBaseline: PrevalidationResult = {
|
|
418
|
+
...r,
|
|
419
|
+
baseline: acBaseline(printed, ac.threshold),
|
|
420
|
+
};
|
|
421
|
+
results.push(withBaseline.autoFixed ? { ...withBaseline, originalCommand: rawCommand } : withBaseline);
|
|
191
422
|
};
|
|
192
423
|
|
|
193
424
|
// Check for bun test pipe pattern BEFORE running — always auto-fix
|
|
@@ -245,6 +476,7 @@ export async function prevalidateEvidence(
|
|
|
245
476
|
cwd: projectRoot,
|
|
246
477
|
stdio: ["pipe", "pipe", "pipe"],
|
|
247
478
|
}).trim();
|
|
479
|
+
printed = output;
|
|
248
480
|
|
|
249
481
|
if (output.length === 0) {
|
|
250
482
|
record(withRefFix({
|
|
@@ -267,6 +499,10 @@ export async function prevalidateEvidence(
|
|
|
267
499
|
const exitCode: number = e.status ?? 1;
|
|
268
500
|
const stdout = (e.stdout?.toString?.() || "").trim();
|
|
269
501
|
const stderr = (e.stderr?.toString?.() || "").trim();
|
|
502
|
+
// A command that exited non-zero still ran, so its stdout is the
|
|
503
|
+
// baseline reading. `grep` exit 1 is the case that matters: it is a
|
|
504
|
+
// RESULT (#271), and the value it measured is in stdout.
|
|
505
|
+
printed = stdout;
|
|
270
506
|
|
|
271
507
|
// A grep that selected nothing has not failed — it has measured zero
|
|
272
508
|
// (#271). Classified here, before any repair is attempted, so the
|
|
@@ -348,6 +584,8 @@ export async function measureEvidencePrevalidation(
|
|
|
348
584
|
verdict: "UNMEASURED",
|
|
349
585
|
checkedAt,
|
|
350
586
|
acs: [],
|
|
587
|
+
broken: [],
|
|
588
|
+
alreadySatisfied: [],
|
|
351
589
|
reason,
|
|
352
590
|
});
|
|
353
591
|
|
|
@@ -385,6 +623,12 @@ export async function measureEvidencePrevalidation(
|
|
|
385
623
|
verdict: broken.length > 0 ? "FAIL" : "PASS",
|
|
386
624
|
checkedAt,
|
|
387
625
|
acs: results,
|
|
626
|
+
// Mapped from the same filter the verdict is derived from, at the single
|
|
627
|
+
// site that derives it, so the verdict and the names it refuses under
|
|
628
|
+
// cannot disagree.
|
|
629
|
+
broken: broken.map((r) => r.id),
|
|
630
|
+
// Same discipline as `broken`, through the binding the gate reads (#669).
|
|
631
|
+
alreadySatisfied: alreadySatisfiedIds(results),
|
|
388
632
|
reason: null,
|
|
389
633
|
};
|
|
390
634
|
}
|