rungate 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +70 -0
- package/lib/aes-calculator.ts +66 -0
- package/lib/agent-audit.ts +139 -0
- package/lib/agent-brief-validation.ts +359 -0
- package/lib/agents-md-refs.ts +90 -0
- package/lib/bash-file-read.ts +197 -0
- package/lib/behavioral-cache.ts +78 -0
- package/lib/branch-cleanup.ts +234 -0
- package/lib/brief-context-parser.ts +35 -0
- package/lib/canary.ts +117 -0
- package/lib/check-registry.ts +226 -0
- package/lib/ci-checkout.ts +400 -0
- package/lib/cli-commands.ts +137 -0
- package/lib/compliance-backfill.ts +220 -0
- package/lib/compliance-report.ts +504 -0
- package/lib/compliance.ts +361 -0
- package/lib/config-loader.ts +255 -0
- package/lib/conformity.ts +1818 -131
- package/lib/container-provenance.ts +230 -0
- package/lib/create-brief.ts +150 -0
- package/lib/create-sc.ts +272 -0
- package/lib/deep-merge.ts +69 -0
- package/lib/dev-liveness.ts +214 -0
- package/lib/directive-extractor.ts +210 -0
- package/lib/distribution-standing.ts +113 -0
- package/lib/doc-hygiene.ts +238 -0
- package/lib/eval-criteria.ts +6 -0
- package/lib/evidence-portability.ts +123 -0
- package/lib/evidence-prevalidator.ts +390 -0
- package/lib/evidence-required.ts +150 -0
- package/lib/evidence-satisfiability.ts +288 -0
- package/lib/frontmatter.ts +163 -0
- package/lib/gap-scanner.ts +348 -0
- package/lib/gate-enforcement.ts +366 -0
- package/lib/generators/agent-briefs.ts +136 -0
- package/lib/generators/agents-md.ts +506 -0
- package/lib/generators/code-map.ts +60 -0
- package/lib/generators/types.ts +184 -0
- package/lib/git-ref-normalizer.ts +107 -0
- package/lib/github.ts +539 -0
- package/lib/harness-identity.ts +63 -0
- package/lib/hill-climb.ts +164 -0
- package/lib/instruction-effectiveness.ts +131 -0
- package/lib/issue-closure-audit.ts +225 -0
- package/lib/issue-create-override.ts +343 -0
- package/lib/js-top-level-code.ts +250 -0
- package/lib/liveness-contradiction.ts +149 -0
- package/lib/organize.ts +505 -0
- package/lib/paths.ts +197 -6
- package/lib/post-fix-verify.ts +157 -0
- package/lib/prior-branch.ts +489 -0
- package/lib/project-state-shape.ts +166 -0
- package/lib/promote-outputs.ts +156 -0
- package/lib/prove-label-gate.ts +35 -0
- package/lib/reachability.ts +302 -0
- package/lib/release-preflight.ts +186 -0
- package/lib/rule-health-pipeline.ts +264 -0
- package/lib/rule-health-trend.ts +161 -0
- package/lib/rule-provenance.ts +149 -0
- package/lib/rule-registry.ts +128 -0
- package/lib/rungate-schema.ts +114 -4
- package/lib/sc-claim-forms.ts +179 -0
- package/lib/sc-cover.ts +150 -0
- package/lib/sc-guard.ts +157 -0
- package/lib/sc-ids.ts +344 -0
- package/lib/scaffold/defaults.ts +165 -0
- package/lib/scaffold/managed-workflow.ts +370 -0
- package/lib/scaffold/owned-files.ts +265 -0
- package/lib/scaffold/steps.ts +2680 -0
- package/lib/scanner.ts +701 -0
- package/lib/screenshot-upload.ts +73 -0
- package/lib/secret-patterns.ts +112 -0
- package/lib/security-verdict.ts +616 -0
- package/lib/signal-phrases.ts +1 -0
- package/lib/spec-change-conformity.ts +80 -0
- package/lib/spec-files.ts +62 -0
- package/lib/spec-registry.ts +133 -0
- package/lib/spec-updater.ts +96 -0
- package/lib/stale-cleanup.ts +224 -0
- package/lib/stale-issue-scanner.ts +36 -0
- package/lib/strip-js-comments.ts +269 -0
- package/lib/suite-measurement.ts +228 -0
- package/lib/task-completion-checks.ts +177 -0
- package/lib/test-suite-lock.ts +1136 -0
- package/lib/transcript-checker.ts +1140 -0
- package/lib/validators/spec-validators.ts +171 -0
- package/lib/verdict-capture.ts +73 -0
- package/lib/verified-push.ts +358 -0
- package/lib/workflow-security.ts +174 -0
- package/lib/worktree-cleanup.ts +250 -0
- package/lib/worktree-collect.ts +283 -0
- package/lib/worktree-isolation.ts +82 -0
- package/package.json +24 -2
- package/prompts/ac-adversary.md +14 -2
- package/prompts/aditi.md +5 -0
- package/prompts/blast-radius.md +5 -0
- package/prompts/container-verify.md +5 -0
- package/prompts/discovery.md +4 -0
- package/prompts/environment.md +5 -0
- package/prompts/evidence-hierarchy.md +5 -0
- package/prompts/evidence-validator.md +11 -0
- package/prompts/marcus.md +5 -0
- package/prompts/prevention.md +5 -0
- package/prompts/prove-reproducer.md +12 -0
- package/prompts/quinn-decision-tree.md +5 -0
- package/prompts/quinn-ui-brief.md +5 -0
- package/prompts/quinn.md +5 -0
- package/prompts/rca.md +5 -0
- package/prompts/read-before-write.md +4 -0
- package/prompts/regression.md +5 -0
- package/prompts/serena.md +5 -0
- package/scripts/audit-specs.ts +464 -0
- package/scripts/convert-spec.ts +293 -0
- package/scripts/create-sc.ts +92 -0
- package/scripts/create-spec.ts +166 -0
- package/scripts/extract-constraints.ts +4 -3
- package/scripts/generate-code-map.ts +1 -1
- package/scripts/scaffold-project.ts +138 -1426
- package/scripts/sync-spec-tests.ts +211 -126
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evidence pre-validator — dry-runs AC evidence commands at SCOPE phase
|
|
3
|
+
* to catch broken commands BEFORE Marcus runs.
|
|
4
|
+
*
|
|
5
|
+
* 70% of pipeline failures (14/20 non-first-pass runs) are caused by evidence
|
|
6
|
+
* commands that fail at VERIFY time. This module catches them early.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { execSync } from "child_process";
|
|
10
|
+
|
|
11
|
+
import { normalizeGitRefs } from "./git-ref-normalizer";
|
|
12
|
+
import { satisfiabilityRefusal } from "./evidence-satisfiability";
|
|
13
|
+
|
|
14
|
+
// ── Types ────────────────────────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
export interface ACInput {
|
|
17
|
+
id: string;
|
|
18
|
+
evidenceMethod?: {
|
|
19
|
+
type?: string;
|
|
20
|
+
command?: string;
|
|
21
|
+
};
|
|
22
|
+
threshold?: {
|
|
23
|
+
op?: string;
|
|
24
|
+
value?: string | number;
|
|
25
|
+
unit?: string;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface PrevalidationResult {
|
|
30
|
+
id: string;
|
|
31
|
+
status: "ok" | "broken" | "empty" | "skipped";
|
|
32
|
+
autoFixed?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* The command as the AC was written, present only when `autoFixed` is.
|
|
35
|
+
*
|
|
36
|
+
* `fixedCommand` alone says what to run next and nothing about what was
|
|
37
|
+
* wrong with what was there. Once the reading is persisted (#235), the
|
|
38
|
+
* original is the only way a reader can tell a repaired command from one
|
|
39
|
+
* that was always written that way.
|
|
40
|
+
*/
|
|
41
|
+
originalCommand?: string;
|
|
42
|
+
fixedCommand?: string;
|
|
43
|
+
needsRewrite?: boolean;
|
|
44
|
+
diagnostic?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The verdicts a pre-validation reading can carry. UNMEASURED is one of them. */
|
|
48
|
+
export type PrevalidationVerdict = "PASS" | "FAIL" | "UNMEASURED";
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* One pre-validation of one AC set, in the shape persisted at
|
|
52
|
+
* `evidencePrevalidation` in workflow-state.json.
|
|
53
|
+
*/
|
|
54
|
+
export interface EvidencePrevalidationReading {
|
|
55
|
+
verdict: PrevalidationVerdict;
|
|
56
|
+
/** ISO timestamp of the measurement. */
|
|
57
|
+
checkedAt: string;
|
|
58
|
+
/** Per-AC results. Empty — and only empty — when the verdict is UNMEASURED. */
|
|
59
|
+
acs: PrevalidationResult[];
|
|
60
|
+
/** Why the reading is UNMEASURED. Null — and only null — when it is not. */
|
|
61
|
+
reason: string | null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ── Auto-fix patterns ───────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Does this command's non-zero exit mean "grep selected no lines" (#271)?
|
|
68
|
+
*
|
|
69
|
+
* `grep` documents three outcomes: 0 if a line was selected, 1 if none was,
|
|
70
|
+
* and greater than 1 if an error occurred. Exit 1 is a RESULT — and for a
|
|
71
|
+
* zero-count acceptance criterion it is the result that satisfies the
|
|
72
|
+
* threshold. Run wf_ea3e428f-64f was refused because its AC printed `0`, the
|
|
73
|
+
* exact value its `== 0` threshold required, and exited 1.
|
|
74
|
+
*
|
|
75
|
+
* This replaces an auto-fix that appended `|| true`. That wrapper swallowed
|
|
76
|
+
* exit 2 as well, turning a missing file or a bad pattern into a clean
|
|
77
|
+
* verdict, so reading grep's own convention is strictly narrower than what it
|
|
78
|
+
* replaces rather than a wider amnesty.
|
|
79
|
+
*
|
|
80
|
+
* Only the LAST pipeline stage is consulted, because that is the one whose
|
|
81
|
+
* status the shell reports. A grep earlier in the chain must not excuse a
|
|
82
|
+
* later command's failure, or any command could buy amnesty by mentioning
|
|
83
|
+
* grep.
|
|
84
|
+
*/
|
|
85
|
+
export function grepNoMatchIsResult(command: string, exitCode: number): boolean {
|
|
86
|
+
if (exitCode !== 1) return false;
|
|
87
|
+
const lastStage = command.split("|").pop()?.trim() ?? "";
|
|
88
|
+
return /^(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)*(?:[a-z]*grep)\b/.test(lastStage);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Detect if a command pipes bun test output through grep.
|
|
93
|
+
* This loses exit codes and is fragile. Rewrite to direct bun test.
|
|
94
|
+
*/
|
|
95
|
+
function tryFixBunTestPipe(command: string): string | null {
|
|
96
|
+
const match = command.match(/^(bun test\s+\S+)\s*\|.*$/);
|
|
97
|
+
if (match) {
|
|
98
|
+
// Strip the pipe and use direct bun test
|
|
99
|
+
return match[1].trim();
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Attempt auto-fix on a command. Returns the fixed command or null.
|
|
106
|
+
*/
|
|
107
|
+
function attemptAutoFix(
|
|
108
|
+
command: string,
|
|
109
|
+
evidenceType: string | undefined,
|
|
110
|
+
exitCode: number,
|
|
111
|
+
stdout: string,
|
|
112
|
+
): { fixedCommand: string } | null {
|
|
113
|
+
// The grep exit-1 pattern used to live here, rewriting the command to
|
|
114
|
+
// `... || true`. It is gone (#271): a zero-match grep is now CLASSIFIED as a
|
|
115
|
+
// result rather than repaired into one that cannot fail. Rewriting it hid
|
|
116
|
+
// exit 2 — a missing file or a bad pattern — behind the same clean verdict.
|
|
117
|
+
// See grepNoMatchIsResult.
|
|
118
|
+
|
|
119
|
+
// Pattern 2: bun test piped through grep — rewrite to direct bun test
|
|
120
|
+
const pipeFix = tryFixBunTestPipe(command);
|
|
121
|
+
if (pipeFix) return { fixedCommand: pipeFix };
|
|
122
|
+
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ── Core function ────────────────────────────────────────────────────────
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Dry-run every AC evidence command and return per-AC validation results.
|
|
130
|
+
*
|
|
131
|
+
* For each AC with an evidenceMethod.command:
|
|
132
|
+
* - Runs the command with a short timeout
|
|
133
|
+
* - Classifies result as ok, broken, empty, or skipped
|
|
134
|
+
* - Attempts auto-fix for known broken patterns
|
|
135
|
+
* - Sets needsRewrite when auto-fix cannot repair
|
|
136
|
+
*/
|
|
137
|
+
export async function prevalidateEvidence(
|
|
138
|
+
acs: ACInput[],
|
|
139
|
+
projectRoot: string,
|
|
140
|
+
): Promise<PrevalidationResult[]> {
|
|
141
|
+
const results: PrevalidationResult[] = [];
|
|
142
|
+
|
|
143
|
+
for (const ac of acs) {
|
|
144
|
+
const rawCommand = ac.evidenceMethod?.command;
|
|
145
|
+
|
|
146
|
+
// Skip ACs without a command
|
|
147
|
+
if (!rawCommand) {
|
|
148
|
+
results.push({
|
|
149
|
+
id: ac.id,
|
|
150
|
+
status: "skipped",
|
|
151
|
+
});
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Qualify bare `main`/`master` refs to `origin/...` BEFORE anything else,
|
|
156
|
+
// including before the dry-run (#118).
|
|
157
|
+
//
|
|
158
|
+
// This is deliberately NOT an entry in attemptAutoFix, which only fires on
|
|
159
|
+
// a FAILING dry-run. A stale-`main` command often succeeds here — the local
|
|
160
|
+
// ref is only sometimes behind — and then fails at Verify, which is exactly
|
|
161
|
+
// what happened on #103: a false FAIL on an untouched file spawned a second
|
|
162
|
+
// Marcus to "fix" it and cost roughly 300k tokens. The defect is in the
|
|
163
|
+
// text of the command, not in its exit code, so the repair has to be too,
|
|
164
|
+
// and it has to happen before the command is dry-run or persisted.
|
|
165
|
+
const refFix = normalizeGitRefs(rawCommand);
|
|
166
|
+
const command = refFix.command;
|
|
167
|
+
const refRewritten = refFix.changed;
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Carry the ref rewrite onto whatever verdict this AC ends up with.
|
|
171
|
+
*
|
|
172
|
+
* Every exit path has to go through here. If the rewrite were only
|
|
173
|
+
* reported on the paths that already set `fixedCommand`, a command that
|
|
174
|
+
* dry-runs clean would be reported "ok" with no fixed form, the caller
|
|
175
|
+
* would persist the original, and the stale ref would reach Verify intact
|
|
176
|
+
* — the bug surviving its own fix.
|
|
177
|
+
*/
|
|
178
|
+
const withRefFix = (r: PrevalidationResult): PrevalidationResult =>
|
|
179
|
+
refRewritten && !r.fixedCommand
|
|
180
|
+
? { ...r, autoFixed: true, fixedCommand: command }
|
|
181
|
+
: r;
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Every exit path for this AC goes through here, so an auto-fix cannot be
|
|
185
|
+
* reported without the command it replaced (#235). Stamping it at the
|
|
186
|
+
* single push site rather than at each branch is what keeps a later
|
|
187
|
+
* auto-fix pattern from quietly landing with only half the pair.
|
|
188
|
+
*/
|
|
189
|
+
const record = (r: PrevalidationResult): void => {
|
|
190
|
+
results.push(r.autoFixed ? { ...r, originalCommand: rawCommand } : r);
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// Check for bun test pipe pattern BEFORE running — always auto-fix
|
|
194
|
+
const pipeFix = tryFixBunTestPipe(command);
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Refuse an AC whose command could not satisfy its threshold under ANY
|
|
198
|
+
* implementation, BEFORE dry-running it (#217).
|
|
199
|
+
*
|
|
200
|
+
* Before the dry-run because the dry-run cannot see this. Measured on
|
|
201
|
+
* `wf_5e3ed885-98c`: `grep -n 'SC-\w+' …` is a basic regular expression
|
|
202
|
+
* whose `+` is a literal, so it selects nothing and dry-runs as `empty` —
|
|
203
|
+
* a WARNING, and the same reading a correct AC produces before the work
|
|
204
|
+
* is done. `grep -c … | wc -l` is worse: it prints `1`, dry-runs `ok`,
|
|
205
|
+
* and the threshold it satisfies is then a constant.
|
|
206
|
+
*
|
|
207
|
+
* This is the inverse of the defect `.claude/rules/checks-must-be-able-to-fail.md`
|
|
208
|
+
* is about, and it is the same defect: a verdict independent of the code.
|
|
209
|
+
*
|
|
210
|
+
* Judged on the command that will actually RUN, not the one as written —
|
|
211
|
+
* after the ref rewrite and after the bun-test-pipe fix. Both change the
|
|
212
|
+
* text, and the pipe fix changes whether the threshold is coherent at
|
|
213
|
+
* all: `bun test … | grep -c pass` with `contains` is an incoherent pair
|
|
214
|
+
* that the fix resolves by deleting the `grep -c`. Refusing the written
|
|
215
|
+
* form would refuse a command the pre-validator was about to repair.
|
|
216
|
+
*/
|
|
217
|
+
const unsatisfiable = satisfiabilityRefusal({
|
|
218
|
+
...ac,
|
|
219
|
+
evidenceMethod: { ...ac.evidenceMethod, command: pipeFix ?? command },
|
|
220
|
+
});
|
|
221
|
+
if (unsatisfiable) {
|
|
222
|
+
record(withRefFix({
|
|
223
|
+
id: ac.id,
|
|
224
|
+
status: "broken",
|
|
225
|
+
needsRewrite: true,
|
|
226
|
+
diagnostic: `The evidence cannot satisfy its own threshold: ${unsatisfiable}`,
|
|
227
|
+
}));
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (pipeFix) {
|
|
232
|
+
record({
|
|
233
|
+
id: ac.id,
|
|
234
|
+
status: "ok",
|
|
235
|
+
autoFixed: true,
|
|
236
|
+
fixedCommand: pipeFix,
|
|
237
|
+
});
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
try {
|
|
242
|
+
const output = execSync(command, {
|
|
243
|
+
encoding: "utf-8",
|
|
244
|
+
timeout: 10000,
|
|
245
|
+
cwd: projectRoot,
|
|
246
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
247
|
+
}).trim();
|
|
248
|
+
|
|
249
|
+
if (output.length === 0) {
|
|
250
|
+
record(withRefFix({
|
|
251
|
+
id: ac.id,
|
|
252
|
+
status: "empty",
|
|
253
|
+
diagnostic: `Command succeeded but produced no output: ${command}`,
|
|
254
|
+
}));
|
|
255
|
+
} else {
|
|
256
|
+
// The ok path is the one that MATTERS for #118: a stale-`main` command
|
|
257
|
+
// usually dry-runs clean here and only diverges later, in a worktree
|
|
258
|
+
// whose local ref has drifted. Without withRefFix the caller would
|
|
259
|
+
// persist the original and the rewrite would be a no-op in exactly the
|
|
260
|
+
// case it exists for.
|
|
261
|
+
record(withRefFix({
|
|
262
|
+
id: ac.id,
|
|
263
|
+
status: "ok",
|
|
264
|
+
}));
|
|
265
|
+
}
|
|
266
|
+
} catch (e: any) {
|
|
267
|
+
const exitCode: number = e.status ?? 1;
|
|
268
|
+
const stdout = (e.stdout?.toString?.() || "").trim();
|
|
269
|
+
const stderr = (e.stderr?.toString?.() || "").trim();
|
|
270
|
+
|
|
271
|
+
// A grep that selected nothing has not failed — it has measured zero
|
|
272
|
+
// (#271). Classified here, before any repair is attempted, so the
|
|
273
|
+
// verdict comes from grep's documented contract rather than from a
|
|
274
|
+
// rewrite. `grep -c` prints "0", a real value; plain `grep` prints
|
|
275
|
+
// nothing, which is `empty` — the status that already exists for a
|
|
276
|
+
// command that ran and produced no value. Collapsing that into "ok"
|
|
277
|
+
// would hand the gate an AC with no evidence, which is #277.
|
|
278
|
+
if (grepNoMatchIsResult(command, exitCode)) {
|
|
279
|
+
record(withRefFix({
|
|
280
|
+
id: ac.id,
|
|
281
|
+
...(stdout.length === 0
|
|
282
|
+
? {
|
|
283
|
+
status: "empty" as const,
|
|
284
|
+
diagnostic: `Command selected no lines and printed nothing: ${command}`,
|
|
285
|
+
}
|
|
286
|
+
: { status: "ok" as const }),
|
|
287
|
+
}));
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Try auto-fix first
|
|
292
|
+
const fix = attemptAutoFix(command, ac.evidenceMethod?.type, exitCode, stdout);
|
|
293
|
+
|
|
294
|
+
if (fix) {
|
|
295
|
+
// fix.fixedCommand is derived from `command`, which is already
|
|
296
|
+
// ref-normalized, so the rewrite is carried without needing withRefFix.
|
|
297
|
+
record({
|
|
298
|
+
id: ac.id,
|
|
299
|
+
status: "ok",
|
|
300
|
+
autoFixed: true,
|
|
301
|
+
fixedCommand: fix.fixedCommand,
|
|
302
|
+
});
|
|
303
|
+
} else {
|
|
304
|
+
// Cannot auto-fix — mark as broken with diagnostic
|
|
305
|
+
const diagParts: string[] = [];
|
|
306
|
+
diagParts.push(`Command failed with exit code ${exitCode}`);
|
|
307
|
+
if (stderr) diagParts.push(stderr.slice(0, 200));
|
|
308
|
+
if (!stdout && !stderr) diagParts.push(`No output from: ${command}`);
|
|
309
|
+
|
|
310
|
+
record(withRefFix({
|
|
311
|
+
id: ac.id,
|
|
312
|
+
status: "broken",
|
|
313
|
+
needsRewrite: true,
|
|
314
|
+
diagnostic: diagParts.join(". "),
|
|
315
|
+
}));
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return results;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// ── The persisted reading (#235) ─────────────────────────────────────────
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Pre-validate an AC set and reduce it to one reading for workflow-state.json.
|
|
327
|
+
*
|
|
328
|
+
* NEVER THROWS, for the reason the rest of this path fails closed: it runs
|
|
329
|
+
* inside the scope gate, and a throw there is a refusal the caller's error
|
|
330
|
+
* handling turns back into "nothing to report". Before #235 the gate called
|
|
331
|
+
* `prevalidateEvidence(...).then(...)` and swallowed rejections into a
|
|
332
|
+
* `console.warn`, so a pre-validation that blew up and a pre-validation that
|
|
333
|
+
* found nothing wrong left the state file in exactly the same condition:
|
|
334
|
+
* empty. UNMEASURED is the third verdict that makes those two distinguishable.
|
|
335
|
+
*
|
|
336
|
+
* The runner is injectable so the UNMEASURED path can be driven in a test. The
|
|
337
|
+
* default is the production one; nothing but a test should pass the third
|
|
338
|
+
* argument.
|
|
339
|
+
*/
|
|
340
|
+
export async function measureEvidencePrevalidation(
|
|
341
|
+
acs: ACInput[],
|
|
342
|
+
projectRoot: string,
|
|
343
|
+
run: (acs: ACInput[], projectRoot: string) => Promise<PrevalidationResult[]> = prevalidateEvidence,
|
|
344
|
+
): Promise<EvidencePrevalidationReading> {
|
|
345
|
+
const checkedAt = new Date().toISOString();
|
|
346
|
+
|
|
347
|
+
const unmeasured = (reason: string): EvidencePrevalidationReading => ({
|
|
348
|
+
verdict: "UNMEASURED",
|
|
349
|
+
checkedAt,
|
|
350
|
+
acs: [],
|
|
351
|
+
reason,
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
if (!Array.isArray(acs) || acs.length === 0) {
|
|
355
|
+
return unmeasured(
|
|
356
|
+
"the evidence commands were not pre-validated: this run has no ACs to measure, " +
|
|
357
|
+
"so there is no verdict to read",
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
let results: PrevalidationResult[];
|
|
362
|
+
try {
|
|
363
|
+
results = await run(acs, projectRoot);
|
|
364
|
+
} catch (e: any) {
|
|
365
|
+
return unmeasured(
|
|
366
|
+
`the evidence commands were not pre-validated: the pre-validator threw — ${
|
|
367
|
+
e?.message ? String(e.message).slice(0, 300) : String(e)
|
|
368
|
+
}`,
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
if (!Array.isArray(results)) {
|
|
373
|
+
return unmeasured(
|
|
374
|
+
`the evidence commands were not pre-validated: the pre-validator returned ${typeof results}, not a list of results`,
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// `empty` is a warning rather than a refusal — the command runs, it just
|
|
379
|
+
// prints nothing — so only `broken` moves the verdict. Both still reach the
|
|
380
|
+
// file, which is the point: a warning that lives in stdout is a warning no
|
|
381
|
+
// later check can see.
|
|
382
|
+
const broken = results.filter((r) => r.status === "broken");
|
|
383
|
+
|
|
384
|
+
return {
|
|
385
|
+
verdict: broken.length > 0 ? "FAIL" : "PASS",
|
|
386
|
+
checkedAt,
|
|
387
|
+
acs: results,
|
|
388
|
+
reason: null,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #277 — the verify gate's evidence-existence rule, as a library.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `gates/workflow.test.ts` rather than left beside the checks
|
|
5
|
+
* that call it. That file only executes when a gate spawns it with
|
|
6
|
+
* `TEST_WORK_DIR`; importing it from an ordinary test to reach one function
|
|
7
|
+
* drags sixteen of its gate checks into the normal suite, where `sf()` reads
|
|
8
|
+
* nothing and every one of them passes vacuously — sixteen checks that cannot
|
|
9
|
+
* fail, added by a test written to close a hole of exactly that shape.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* #277 — the evidence chain had no mandatory link for eleven of thirteen types.
|
|
14
|
+
*
|
|
15
|
+
* `evidence-method-executed` opened with `if (t !== "COMMAND" && t !== "command")
|
|
16
|
+
* return false`, so every other `EVIDENCE_METHOD_TYPES` member exited before
|
|
17
|
+
* anything was looked at. `threshold-crosscheck` opened with
|
|
18
|
+
* `if (!ac.threshold || !ac.evidence) continue`, so an AC with no evidence was
|
|
19
|
+
* the one case it had no opinion about — and that is precisely the state an AC
|
|
20
|
+
* is in when nobody recorded any. `EvidenceSchema` is `.nullable().optional()`,
|
|
21
|
+
* so absent is schema-valid.
|
|
22
|
+
*
|
|
23
|
+
* An AC declaring `BUN_TEST` and `threshold: >= 5`, with nothing ever run and
|
|
24
|
+
* nothing ever recorded, passed both checks. So did `PLAYWRIGHT`, `SCREENSHOT`
|
|
25
|
+
* and `GREP_CHECK`. The lower-case aliases do not rescue it; `grep` is a
|
|
26
|
+
* distinct enum member from `command`, so they widen the hole.
|
|
27
|
+
*
|
|
28
|
+
* Two independent links now, so neither going quiet reopens it: this function
|
|
29
|
+
* requires the evidence to EXIST, `thresholdCrosscheckViolations` requires its
|
|
30
|
+
* value to clear the bar.
|
|
31
|
+
*/
|
|
32
|
+
export const EVIDENCE_EXEMPT_TYPES: ReadonlyMap<string, string> = new Map([
|
|
33
|
+
[
|
|
34
|
+
"MANUAL",
|
|
35
|
+
"a human attests; there is no command to produce a value, so demanding a " +
|
|
36
|
+
"recorded one would refuse every honest manual AC. The attestation is the " +
|
|
37
|
+
"evidence, and evidence-tier is what grades it.",
|
|
38
|
+
],
|
|
39
|
+
["manual", "lower-case alias of MANUAL, carried by states written before the enum was cased."],
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Enumerated, not fallen through (#277 SC-2). An exemption produced by a type
|
|
44
|
+
* comparison missing a branch is indistinguishable from one somebody decided
|
|
45
|
+
* on, which is how eleven types came to be exempt without anyone choosing it.
|
|
46
|
+
*/
|
|
47
|
+
export function evidenceExemption(ac: any): string | null {
|
|
48
|
+
const t = ac?.evidenceMethod?.type;
|
|
49
|
+
if (typeof t === "string" && EVIDENCE_EXEMPT_TYPES.has(t)) {
|
|
50
|
+
return `evidenceMethod.type=${t}: ${EVIDENCE_EXEMPT_TYPES.get(t)}`;
|
|
51
|
+
}
|
|
52
|
+
if (ac?.verdict === "SKIP") {
|
|
53
|
+
return "verdict=SKIP: the AC declares it was not evaluated, which is a statement rather than a silence.";
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** True when the AC carries something a reader could inspect. */
|
|
59
|
+
function hasRecordedEvidence(ac: any): boolean {
|
|
60
|
+
const e = ac?.evidence;
|
|
61
|
+
if (!e) return false;
|
|
62
|
+
if (typeof e !== "object") return true;
|
|
63
|
+
return Boolean(e.content) || e.value != null || Boolean(e.url);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function missingEvidenceViolations(acs: unknown): string[] {
|
|
67
|
+
if (!Array.isArray(acs)) {
|
|
68
|
+
// UNMEASURED, not PASS. `sf("acs")` returning a non-array means the state
|
|
69
|
+
// is not the shape this check reads, and a check that cannot find its
|
|
70
|
+
// subject has not cleared it.
|
|
71
|
+
return [
|
|
72
|
+
`evidence-method-executed: acs is ${acs === undefined ? "absent" : typeof acs}, not an array — ` +
|
|
73
|
+
`nothing could be checked. This is UNMEASURED, not PASS.`,
|
|
74
|
+
];
|
|
75
|
+
}
|
|
76
|
+
const out: string[] = [];
|
|
77
|
+
for (const ac of acs) {
|
|
78
|
+
const id = ac?.id ?? "(unnamed AC)";
|
|
79
|
+
const t = ac?.evidenceMethod?.type;
|
|
80
|
+
|
|
81
|
+
// The original rule, kept intact rather than folded in: a COMMAND AC needs
|
|
82
|
+
// evidence CONTENT specifically, which is stricter than "something was
|
|
83
|
+
// recorded" and must not be loosened by widening the check around it.
|
|
84
|
+
if (t === "COMMAND" || t === "command") {
|
|
85
|
+
if (!ac?.evidence || (typeof ac.evidence === "object" && !ac.evidence.content)) {
|
|
86
|
+
out.push(`${id}: evidenceMethod.type=${t} and no evidence content was recorded`);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// An AC with no threshold has no bar to clear; ACSchema already refuses
|
|
92
|
+
// one at the schema layer, so this is not where that is enforced.
|
|
93
|
+
if (!ac?.threshold) continue;
|
|
94
|
+
|
|
95
|
+
const exempt = evidenceExemption(ac);
|
|
96
|
+
if (exempt) continue;
|
|
97
|
+
if (hasRecordedEvidence(ac)) continue;
|
|
98
|
+
|
|
99
|
+
out.push(
|
|
100
|
+
`${id}: threshold ${ac.threshold.op} ${ac.threshold.value} with NO evidence recorded ` +
|
|
101
|
+
`(evidenceMethod.type=${t ?? "unset"}). An AC that was never measured is not an AC ` +
|
|
102
|
+
`that passed — this check covered only COMMAND before #277, so eleven of thirteen ` +
|
|
103
|
+
`evidence types reached the verdict with nothing behind them.`,
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* #277 SC-6 — each skip here is now a decision rather than a gap.
|
|
111
|
+
*
|
|
112
|
+
* `!ac.evidence` used to be the widest hole in the gate: the one state it
|
|
113
|
+
* declined to have an opinion about was the one that mattered. It is still a
|
|
114
|
+
* skip, because refusing here as well would report the same AC twice with two
|
|
115
|
+
* different explanations, but it is only safe because
|
|
116
|
+
* `missingEvidenceViolations` refuses it first. The two are a pair; neither is
|
|
117
|
+
* sufficient alone, which is why both are exported and both are mutated.
|
|
118
|
+
*/
|
|
119
|
+
export function thresholdCrosscheckViolations(acs: unknown): string[] {
|
|
120
|
+
if (!Array.isArray(acs)) {
|
|
121
|
+
return [`threshold-crosscheck: acs is not an array — nothing could be checked. UNMEASURED, not PASS.`];
|
|
122
|
+
}
|
|
123
|
+
const violations: string[] = [];
|
|
124
|
+
for (const ac of acs) {
|
|
125
|
+
// No bar, or nothing recorded. The second is caught by
|
|
126
|
+
// missingEvidenceViolations, which runs as its own check.
|
|
127
|
+
if (!ac?.threshold || !ac?.evidence) continue;
|
|
128
|
+
const val = typeof ac.evidence === "object" ? ac.evidence.value : null;
|
|
129
|
+
// A non-numeric evidence shape — a screenshot, a file citation — cannot be
|
|
130
|
+
// compared to a numeric bar. Skipped on purpose, and visible because the
|
|
131
|
+
// AC had to get past the existence check to arrive here.
|
|
132
|
+
if (val == null) continue;
|
|
133
|
+
const exp = parseFloat(String(ac.threshold.value));
|
|
134
|
+
// `contains` and `exists` are legitimate non-numeric operators in
|
|
135
|
+
// THRESHOLD_OPS; parseFloat on their value is NaN and comparing it would
|
|
136
|
+
// be meaningless, not lenient.
|
|
137
|
+
if (isNaN(exp) || isNaN(val)) continue;
|
|
138
|
+
let pass = true;
|
|
139
|
+
switch (ac.threshold.op) {
|
|
140
|
+
case ">=": pass = val >= exp; break;
|
|
141
|
+
case "<=": pass = val <= exp; break;
|
|
142
|
+
case ">": pass = val > exp; break;
|
|
143
|
+
case "<": pass = val < exp; break;
|
|
144
|
+
case "==": pass = val === exp; break;
|
|
145
|
+
case "!=": pass = val !== exp; break;
|
|
146
|
+
}
|
|
147
|
+
if (!pass) violations.push(`${ac.id}: ${val} ${ac.threshold.op} ${exp} = false`);
|
|
148
|
+
}
|
|
149
|
+
return violations;
|
|
150
|
+
}
|