rungate 0.8.23 → 0.8.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +316 -68
- package/gates/orchestrator.test.ts +108 -2
- package/gates/orchestrator.ts +25 -5
- package/gates/prompts/prove-reproducer.md +14 -1
- package/gates/prompts.sha256 +1 -1
- package/gates/prove.test.ts +6 -1
- package/gates/schema.ts +21 -1
- package/gates/workflow.test.ts +12 -0
- package/lib/ai-review-suppression.ts +26 -2
- package/lib/artifact-schema.ts +37 -0
- package/lib/scaffold/ai-review-step.ts +14 -0
- package/lib/scaffold/intake-workflow.ts +120 -0
- package/lib/scaffold/owned-files.ts +10 -0
- package/lib/scaffold/steps.ts +14 -1
- package/package.json +1 -1
- package/scripts/ai-review-findings.ts +6 -2
- package/config/prove-evidence.schema.json +0 -83
package/gates/.gate-salt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
55059bcaab0d2a5e265a824747d8d98b6a512c0b49e3b984deb3bc1db05deba3
|
package/gates/gate-executor.ts
CHANGED
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
measureEvidencePrevalidation,
|
|
39
39
|
type EvidencePrevalidationReading,
|
|
40
40
|
} from "../lib/evidence-prevalidator";
|
|
41
|
-
import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
|
|
41
|
+
import { AdversaryReportSchema, EvidencePrevalidationSchema, ProveEvidenceSchema, type PrevalidationSource } from "./schema";
|
|
42
42
|
import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
|
|
43
43
|
import { deepMerge } from "../lib/deep-merge";
|
|
44
44
|
import { livenessPlan, probeTarget } from "../lib/dev-liveness";
|
|
@@ -426,6 +426,170 @@ export function comparisonSummaryFor(
|
|
|
426
426
|
return `compared against the ${reading.type} before-state captured at ${reading.capturedAt}; ${counts}`;
|
|
427
427
|
}
|
|
428
428
|
|
|
429
|
+
// The validator moved to lib/ in #2255 so gates/orchestrator.ts can use it
|
|
430
|
+
// without importing this file back; re-exported so it has one definition.
|
|
431
|
+
import { validateArtifactSchema, type ArtifactSchema } from "../lib/artifact-schema";
|
|
432
|
+
export { validateArtifactSchema, type ArtifactSchema };
|
|
433
|
+
|
|
434
|
+
const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
435
|
+
/**
|
|
436
|
+
* The after-evidence types a reproducer may report, and the prompt's list
|
|
437
|
+
* exactly. `none` is B3's to write. `screenshot` is not the reproducer's to
|
|
438
|
+
* claim: it has Bash and Read only (gates/prompts/prove-reproducer.md), so a
|
|
439
|
+
* reported screenshot is a UI verification it could not have made.
|
|
440
|
+
*/
|
|
441
|
+
const REPORTED_AFTER_TYPES = ["text", "api", "data"];
|
|
442
|
+
const AFTER_ENVIRONMENTS = ["local", "prod"];
|
|
443
|
+
|
|
444
|
+
/** What B3 knows about the run that the reproducer does not get to say. */
|
|
445
|
+
export interface ProveEvidenceContext {
|
|
446
|
+
issueNumber: number;
|
|
447
|
+
commitSHA: string;
|
|
448
|
+
capturedAt: string;
|
|
449
|
+
beforeReading: BeforeStateReading;
|
|
450
|
+
/** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
|
|
451
|
+
source?: string;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
type AfterEvidence = {
|
|
455
|
+
type: string; path: string | null; capturedAt: string; environment: string; description: string;
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
/** The reproducer's `afterEvidence`, or why there is none. Never repaired into shape. */
|
|
459
|
+
function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
|
|
460
|
+
if (raw === undefined || raw === null) return { ok: false, reason: "the reproducer reported no afterEvidence" };
|
|
461
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
462
|
+
return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
|
|
463
|
+
}
|
|
464
|
+
const a = raw as Record<string, unknown>;
|
|
465
|
+
if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
|
|
466
|
+
return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
|
|
467
|
+
}
|
|
468
|
+
if (typeof a.type !== "string" || !REPORTED_AFTER_TYPES.includes(a.type)) {
|
|
469
|
+
return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one of ${REPORTED_AFTER_TYPES.join("|")}` };
|
|
470
|
+
}
|
|
471
|
+
if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
|
|
472
|
+
return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
|
|
473
|
+
}
|
|
474
|
+
if (typeof a.description !== "string" || a.description.trim() === "") {
|
|
475
|
+
return { ok: false, reason: "afterEvidence.description is empty — it must say what the after-state showed" };
|
|
476
|
+
}
|
|
477
|
+
return {
|
|
478
|
+
ok: true,
|
|
479
|
+
value: {
|
|
480
|
+
type: a.type, path: typeof a.path === "string" ? a.path : null,
|
|
481
|
+
capturedAt: a.capturedAt, environment: a.environment, description: a.description,
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Build `prove-evidence.json` from the reproducer's report (#1559, Jason's
|
|
488
|
+
* option (a) of 2026-11-15).
|
|
489
|
+
*
|
|
490
|
+
* B3 used to write `{issueNumber, verdict, commitSHA, capturedAt,
|
|
491
|
+
* criteriaResults, reproduced, evidence, source}`, which lacks the
|
|
492
|
+
* `beforeEvidence`, `afterEvidence` and `comparisonSummary` that
|
|
493
|
+
* `ProveEvidenceSchema` requires. The before-evidence and the comparison are
|
|
494
|
+
* B3's own readings (#1540), so a reproducer cannot overwrite them. The
|
|
495
|
+
* after-evidence is the reproducer's report of the environment it tested and
|
|
496
|
+
* what it saw. When that report is missing or malformed, the record is
|
|
497
|
+
* `type: "none"` and a PROVEN becomes INCONCLUSIVE, because a proof with no
|
|
498
|
+
* observed after-state is not a proof.
|
|
499
|
+
*
|
|
500
|
+
* The `none` record says `environment: "local"` because the schema requires
|
|
501
|
+
* a value. `local` is the one that satisfies no STANDARD+ prod requirement
|
|
502
|
+
* (gates/prove.test.ts), so a missing after-evidence can never pass that check.
|
|
503
|
+
*/
|
|
504
|
+
export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
|
|
505
|
+
const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
|
|
506
|
+
const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
|
|
507
|
+
const after = readReportedAfterEvidence(raw?.afterEvidence);
|
|
508
|
+
|
|
509
|
+
let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
|
|
510
|
+
let verdictDowngrade: { reported: unknown; reason: string } | undefined;
|
|
511
|
+
if (verdict !== raw?.verdict) {
|
|
512
|
+
verdictDowngrade = {
|
|
513
|
+
reported: raw?.verdict,
|
|
514
|
+
reason: `verdict ${JSON.stringify(raw?.verdict)} is not one of ${PROVE_VERDICTS.join("|")}`,
|
|
515
|
+
};
|
|
516
|
+
} else if (verdict === "PROVEN" && !after.ok) {
|
|
517
|
+
verdict = "INCONCLUSIVE";
|
|
518
|
+
verdictDowngrade = { reported: "PROVEN", reason: `PROVEN with no after-evidence: ${after.reason}` };
|
|
519
|
+
} else if (verdict === "PROVEN" && passed === 0) {
|
|
520
|
+
// A proof that verified no criterion is not a proof: an empty list, or
|
|
521
|
+
// nothing but SKIPs, is what a reproducer with no way to check says.
|
|
522
|
+
verdict = "INCONCLUSIVE";
|
|
523
|
+
verdictDowngrade = {
|
|
524
|
+
reported: "PROVEN",
|
|
525
|
+
reason: `PROVEN with no criterion verified: ${criteriaResults.length} criteriaResults row(s), none PASS`,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
const afterEvidence: AfterEvidence = after.ok ? after.value : {
|
|
530
|
+
type: "none", path: null, capturedAt: ctx.capturedAt, environment: "local",
|
|
531
|
+
description: `no after-evidence: ${after.reason}`,
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
return {
|
|
535
|
+
issueNumber: ctx.issueNumber,
|
|
536
|
+
verdict,
|
|
537
|
+
...(verdictDowngrade ? { verdictDowngrade } : {}),
|
|
538
|
+
commitSHA: ctx.commitSHA,
|
|
539
|
+
capturedAt: ctx.capturedAt,
|
|
540
|
+
beforeEvidence: beforeEvidenceFor(ctx.beforeReading),
|
|
541
|
+
afterEvidence,
|
|
542
|
+
comparisonSummary: comparisonSummaryFor(ctx.beforeReading, passed, criteriaResults.length),
|
|
543
|
+
reproduced: raw?.reproduced === true,
|
|
544
|
+
criteriaResults,
|
|
545
|
+
evidence: Array.isArray(raw?.evidence) ? raw.evidence : [],
|
|
546
|
+
source: ctx.source ?? "B3-prove-reproducer",
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
export type ProveEvidenceRecord =
|
|
551
|
+
| { written: true; artifact: ReturnType<typeof buildProveEvidence> }
|
|
552
|
+
| { written: false; reason: string };
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Build, validate and write `prove-evidence.json`, and say what happened
|
|
556
|
+
* (#1559). Never throws.
|
|
557
|
+
*
|
|
558
|
+
* B3 has always degraded to a warning when it could not record the artifact
|
|
559
|
+
* (`prove.test.ts` writes its own), so a failing `git rev-parse` or a write
|
|
560
|
+
* into a missing directory must not crash the prove gate. Every outcome that
|
|
561
|
+
* is not a written file says why.
|
|
562
|
+
*/
|
|
563
|
+
export function recordProveEvidence(
|
|
564
|
+
raw: Record<string, any>,
|
|
565
|
+
opts: {
|
|
566
|
+
issueNumber: number;
|
|
567
|
+
workDir: string;
|
|
568
|
+
beforeReading: BeforeStateReading;
|
|
569
|
+
readCommit: () => string;
|
|
570
|
+
schema?: ArtifactSchema;
|
|
571
|
+
source?: string;
|
|
572
|
+
},
|
|
573
|
+
): ProveEvidenceRecord {
|
|
574
|
+
try {
|
|
575
|
+
const artifact = buildProveEvidence(raw, {
|
|
576
|
+
issueNumber: opts.issueNumber,
|
|
577
|
+
commitSHA: opts.readCommit(),
|
|
578
|
+
capturedAt: new Date().toISOString(),
|
|
579
|
+
beforeReading: opts.beforeReading,
|
|
580
|
+
source: opts.source,
|
|
581
|
+
});
|
|
582
|
+
const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
|
|
583
|
+
if (violations.length > 0) {
|
|
584
|
+
return { written: false, reason: `it fails ProveEvidenceSchema (SC-2770): ${violations.join("; ")}` };
|
|
585
|
+
}
|
|
586
|
+
writeFileSync(join(opts.workDir, "prove-evidence.json"), JSON.stringify(artifact, null, 2));
|
|
587
|
+
return { written: true, artifact };
|
|
588
|
+
} catch (e: any) {
|
|
589
|
+
return { written: false, reason: `could not record it: ${String(e?.message ?? e).slice(0, 200)}` };
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
429
593
|
/** What a gate run does about the tests that existed when it last looked. */
|
|
430
594
|
export type CensusVerdict =
|
|
431
595
|
| { kind: "unmeasured"; reason: string }
|
|
@@ -1554,26 +1718,33 @@ async function runProveReproducer(state: Record<string, any>, issue: number, iss
|
|
|
1554
1718
|
}
|
|
1555
1719
|
|
|
1556
1720
|
const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
|
|
1721
|
+
let parsed: Record<string, any> | null = null;
|
|
1557
1722
|
if (jsonMatch) {
|
|
1558
1723
|
try {
|
|
1559
|
-
|
|
1560
|
-
const proveEvidence = {
|
|
1561
|
-
issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
|
|
1562
|
-
commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1563
|
-
capturedAt: new Date().toISOString(),
|
|
1564
|
-
criteriaResults: parsed.criteriaResults || [],
|
|
1565
|
-
reproduced: parsed.reproduced ?? false,
|
|
1566
|
-
evidence: parsed.evidence || [],
|
|
1567
|
-
source: "B3-prove-reproducer",
|
|
1568
|
-
};
|
|
1569
|
-
writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
|
|
1570
|
-
console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
|
|
1724
|
+
parsed = JSON.parse(jsonMatch[0]);
|
|
1571
1725
|
} catch {
|
|
1572
1726
|
console.warn("WARN: B3 reproducer JSON parse failed");
|
|
1573
1727
|
}
|
|
1574
1728
|
} else {
|
|
1575
1729
|
console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
|
|
1576
1730
|
}
|
|
1731
|
+
if (parsed && typeof parsed === "object") {
|
|
1732
|
+
// #1559: the artifact is built in one place and checked against its one
|
|
1733
|
+
// contract before it is written. An artifact the schema refuses is not
|
|
1734
|
+
// written, and the fields are named; nothing here can crash the gate.
|
|
1735
|
+
const record = recordProveEvidence(parsed, {
|
|
1736
|
+
issueNumber: issue, workDir, beforeReading,
|
|
1737
|
+
readCommit: () => execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1738
|
+
});
|
|
1739
|
+
if (!record.written) {
|
|
1740
|
+
console.warn(`WARN: B3 prove-evidence.json NOT written — ${record.reason}; prove.test.ts will generate evidence`);
|
|
1741
|
+
} else {
|
|
1742
|
+
console.log(`B3: prove-evidence.json written — verdict: ${record.artifact.verdict}`);
|
|
1743
|
+
if (record.artifact.verdictDowngrade) {
|
|
1744
|
+
console.warn(`WARN: B3 reproducer reported ${JSON.stringify(record.artifact.verdictDowngrade.reported)}, recorded INCONCLUSIVE — ${record.artifact.verdictDowngrade.reason}`);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1577
1748
|
}
|
|
1578
1749
|
}
|
|
1579
1750
|
|
|
@@ -2318,7 +2489,7 @@ function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, ha
|
|
|
2318
2489
|
writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
|
|
2319
2490
|
const { spawn } = require("child_process");
|
|
2320
2491
|
const child = spawn("sh", ["-c",
|
|
2321
|
-
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format
|
|
2492
|
+
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format text --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
|
|
2322
2493
|
], { detached: true, stdio: "ignore", cwd: harnessRoot });
|
|
2323
2494
|
child.unref();
|
|
2324
2495
|
console.log("B1: AC Adversary spawned in background — verify gate will check result");
|
|
@@ -2424,40 +2595,138 @@ function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, resu
|
|
|
2424
2595
|
results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
|
|
2425
2596
|
console.log("B1: adversary report not yet available — verify FAIL");
|
|
2426
2597
|
} else {
|
|
2598
|
+
let raw: string | null = null;
|
|
2427
2599
|
try {
|
|
2428
|
-
|
|
2429
|
-
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2430
|
-
if (jsonMatch) {
|
|
2431
|
-
const report = JSON.parse(jsonMatch[0]);
|
|
2432
|
-
console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
|
|
2433
|
-
if (report.gameable > 0) {
|
|
2434
|
-
let outcomeCount = 0;
|
|
2435
|
-
for (const exploit of report.exploits || []) {
|
|
2436
|
-
const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
|
|
2437
|
-
const isOutcome = ac?.type === "OUTCOME";
|
|
2438
|
-
if (isOutcome) {
|
|
2439
|
-
addedFails++;
|
|
2440
|
-
outcomeCount++;
|
|
2441
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2442
|
-
} else {
|
|
2443
|
-
addedWarns++;
|
|
2444
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2445
|
-
}
|
|
2446
|
-
}
|
|
2447
|
-
if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
|
|
2448
|
-
const codeCount = (report.gameable || 0) - outcomeCount;
|
|
2449
|
-
if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
|
|
2450
|
-
}
|
|
2451
|
-
} else {
|
|
2452
|
-
console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
|
|
2453
|
-
}
|
|
2600
|
+
raw = readFileSync(reportPath, "utf-8");
|
|
2454
2601
|
} catch (e: any) {
|
|
2455
|
-
|
|
2602
|
+
addedFails++;
|
|
2603
|
+
results.push({ check: "B1-adversary: report unreadable", result: "FAIL" as const, detail: `adversary-report.json could not be read: ${e.message?.slice(0, 200)}` });
|
|
2604
|
+
}
|
|
2605
|
+
if (raw !== null) {
|
|
2606
|
+
const reading = adversaryReportReading(raw, state.acs || []);
|
|
2607
|
+
addedFails += reading.fails;
|
|
2608
|
+
addedWarns += reading.warns;
|
|
2609
|
+
results.push(...reading.results);
|
|
2610
|
+
console.log(`B1: adversary report — ${reading.fails} FAIL, ${reading.warns} WARN`);
|
|
2456
2611
|
}
|
|
2457
2612
|
}
|
|
2458
2613
|
return { fails: addedFails, warns: addedWarns };
|
|
2459
2614
|
}
|
|
2460
2615
|
|
|
2616
|
+
/**
|
|
2617
|
+
* Read B1's adversary report into gate results (#2259, SC-2770).
|
|
2618
|
+
*
|
|
2619
|
+
* The report is raw output from the model B1 spawns. It was read with a
|
|
2620
|
+
* regex and `JSON.parse` and never checked against `AdversaryReportSchema`,
|
|
2621
|
+
* so `"gameable": "2"` with no exploits, or `exploits` that was not a list,
|
|
2622
|
+
* produced no result at all: the reading of a clean report. A report with no
|
|
2623
|
+
* JSON only printed a warning, so the gate passed on a report it never read.
|
|
2624
|
+
* Now an unreadable report and a report the schema refuses are each one FAIL
|
|
2625
|
+
* naming why, the same as a missing report. Exported so it runs on fixtures;
|
|
2626
|
+
* the schema is a parameter so a test can hand it a stricter copy.
|
|
2627
|
+
*/
|
|
2628
|
+
export function adversaryReportReading(
|
|
2629
|
+
raw: string,
|
|
2630
|
+
acs: ReadonlyArray<{ id?: unknown; type?: unknown }>,
|
|
2631
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2632
|
+
): { fails: number; warns: number; results: GateResult[] } {
|
|
2633
|
+
const parsed = parseAdversaryReport(raw, schema);
|
|
2634
|
+
if (parsed.status !== "read") {
|
|
2635
|
+
return { fails: 1, warns: 0, results: [{ check: `B1-adversary: ${parsed.check}`, result: "FAIL" as const, detail: parsed.detail }] };
|
|
2636
|
+
}
|
|
2637
|
+
let fails = 0;
|
|
2638
|
+
let warns = 0;
|
|
2639
|
+
const results: GateResult[] = [];
|
|
2640
|
+
// Every listed exploit is read whatever `gameable` says: one dropped because
|
|
2641
|
+
// the count was 0 is the same silent pass in the other direction.
|
|
2642
|
+
for (const exploit of parsed.report.exploits) {
|
|
2643
|
+
const ac = acs.find(a => a.id === exploit.acId);
|
|
2644
|
+
if (ac?.type === "OUTCOME") {
|
|
2645
|
+
fails++;
|
|
2646
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2647
|
+
} else {
|
|
2648
|
+
warns++;
|
|
2649
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
return { fails, warns, results };
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
type AdversaryReport = {
|
|
2656
|
+
gameable: number;
|
|
2657
|
+
approved: boolean;
|
|
2658
|
+
exploits: Array<{ acId: string; exploit: string; recommendation: string }>;
|
|
2659
|
+
};
|
|
2660
|
+
|
|
2661
|
+
/**
|
|
2662
|
+
* THE ONE READ of a B1 adversary report (#2259), for the report on disk and
|
|
2663
|
+
* for the live re-run at verify alike. Both used to spell their own regex and
|
|
2664
|
+
* `JSON.parse` with no schema, and review round 1 found the second copy still
|
|
2665
|
+
* swallowing a malformed reply after the first was fixed. Two readers of one
|
|
2666
|
+
* shape are one guard with a dead twin; test/adversary-report-schema.test.ts
|
|
2667
|
+
* asserts the extraction appears once in this file.
|
|
2668
|
+
*
|
|
2669
|
+
* Not "read" when the output holds no JSON naming `gameable`, when that JSON
|
|
2670
|
+
* does not parse, when the schema refuses it, or when it claims more gameable
|
|
2671
|
+
* criteria than it lists exploits for (`{gameable: 3, exploits: []}` is
|
|
2672
|
+
* schema-valid and used to read as clean). More exploits than the count is
|
|
2673
|
+
* not a contradiction: two ways to game one criterion.
|
|
2674
|
+
*/
|
|
2675
|
+
export function parseAdversaryReport(
|
|
2676
|
+
raw: string,
|
|
2677
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2678
|
+
): { status: "read"; report: AdversaryReport } | { status: "not-read"; check: string; detail: string } {
|
|
2679
|
+
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2680
|
+
if (!jsonMatch) {
|
|
2681
|
+
return { status: "not-read", check: "report unreadable", detail: "the output holds no JSON object naming \"gameable\"" };
|
|
2682
|
+
}
|
|
2683
|
+
let report: any;
|
|
2684
|
+
try {
|
|
2685
|
+
report = JSON.parse(jsonMatch[0]);
|
|
2686
|
+
} catch (e: any) {
|
|
2687
|
+
return { status: "not-read", check: "report unreadable", detail: `the report's JSON does not parse: ${e.message?.slice(0, 200)}` };
|
|
2688
|
+
}
|
|
2689
|
+
const violations = validateArtifactSchema(report, schema);
|
|
2690
|
+
if (violations.length > 0) {
|
|
2691
|
+
return { status: "not-read", check: "report fails AdversaryReportSchema", detail: violations.join("; ") };
|
|
2692
|
+
}
|
|
2693
|
+
if (report.gameable > report.exploits.length) {
|
|
2694
|
+
return {
|
|
2695
|
+
status: "not-read",
|
|
2696
|
+
check: "report contradicts itself",
|
|
2697
|
+
detail: `gameable ${report.gameable} but ${report.exploits.length} exploit(s) listed, so which criteria are gameable is unknown`,
|
|
2698
|
+
};
|
|
2699
|
+
}
|
|
2700
|
+
return { status: "read", report };
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
/**
|
|
2704
|
+
* The live B1 re-run at verify, read (#2259 review round 1). Advisory, as it
|
|
2705
|
+
* always was: every finding is a WARN. What changes is that a reply that could
|
|
2706
|
+
* not be read is a WARN naming why, and `agentResult` records it as
|
|
2707
|
+
* UNMEASURED, where it used to be `null` and the verify gate's record of the
|
|
2708
|
+
* adversary was never written at all.
|
|
2709
|
+
*/
|
|
2710
|
+
export function liveAdversaryReading(
|
|
2711
|
+
output: string,
|
|
2712
|
+
ts: string,
|
|
2713
|
+
): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
|
|
2714
|
+
const parsed = parseAdversaryReport(output);
|
|
2715
|
+
if (parsed.status !== "read") {
|
|
2716
|
+
return {
|
|
2717
|
+
agentResult: { ts, status: "UNMEASURED", reason: `${parsed.check}: ${parsed.detail}`, source: "B1-adversary-verify-agent" },
|
|
2718
|
+
warns: 1,
|
|
2719
|
+
results: [{ check: `B1-adversary-verify: ${parsed.check}`, result: "WARN" as const, detail: parsed.detail }],
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
const { report } = parsed;
|
|
2723
|
+
return {
|
|
2724
|
+
agentResult: { ts, gameable: report.gameable, approved: report.approved, exploits: report.exploits, source: "B1-adversary-verify-agent" },
|
|
2725
|
+
warns: report.exploits.length,
|
|
2726
|
+
results: report.exploits.map(exploit => ({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit })),
|
|
2727
|
+
};
|
|
2728
|
+
}
|
|
2729
|
+
|
|
2461
2730
|
function runB2EvidenceValidation(state: Record<string, any>): void {
|
|
2462
2731
|
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2463
2732
|
if (tier === "LIGHT") return;
|
|
@@ -2677,32 +2946,11 @@ function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, res
|
|
|
2677
2946
|
console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
|
|
2678
2947
|
}
|
|
2679
2948
|
|
|
2680
|
-
const
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
ts: new Date().toISOString(),
|
|
2686
|
-
gameable: b1Report.gameable || 0,
|
|
2687
|
-
approved: b1Report.approved ?? true,
|
|
2688
|
-
exploits: b1Report.exploits || [],
|
|
2689
|
-
source: "B1-adversary-verify-agent",
|
|
2690
|
-
};
|
|
2691
|
-
if (b1Report.gameable > 0) {
|
|
2692
|
-
for (const exploit of b1Report.exploits || []) {
|
|
2693
|
-
addedWarns++;
|
|
2694
|
-
results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2695
|
-
}
|
|
2696
|
-
console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
|
|
2697
|
-
} else {
|
|
2698
|
-
console.log(`B1: All ACs approved at verify — no exploits found`);
|
|
2699
|
-
}
|
|
2700
|
-
} catch {
|
|
2701
|
-
console.warn("WARN: B1 AC Adversary JSON parse failed");
|
|
2702
|
-
}
|
|
2703
|
-
} else {
|
|
2704
|
-
console.warn("WARN: B1 AC Adversary produced no parseable JSON");
|
|
2705
|
-
}
|
|
2949
|
+
const live = liveAdversaryReading(b1Output, new Date().toISOString());
|
|
2950
|
+
agentResult = live.agentResult;
|
|
2951
|
+
addedWarns += live.warns;
|
|
2952
|
+
results.push(...live.results);
|
|
2953
|
+
console.log(`B1: verify agent — ${live.warns} WARN${live.agentResult.status === "UNMEASURED" ? ` (UNMEASURED: ${live.agentResult.reason})` : ""}`);
|
|
2706
2954
|
|
|
2707
2955
|
return { agentResult, warns: addedWarns };
|
|
2708
2956
|
}
|
|
@@ -3198,7 +3446,7 @@ export async function executeGate(input: GateExecutorInput): Promise<GateExecuto
|
|
|
3198
3446
|
const hash = generateHmac(sf, slug, issue, state.projectRoot);
|
|
3199
3447
|
console.log(`HMAC: ${hash}`);
|
|
3200
3448
|
const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
|
|
3201
|
-
console.log(`Ship evidence: ${evidencePath}`);
|
|
3449
|
+
console.log(evidencePath ? `Ship evidence: ${evidencePath}` : "Ship evidence: NOT written — see the fields named above (#2255)");
|
|
3202
3450
|
}
|
|
3203
3451
|
|
|
3204
3452
|
// Verify-specific: convergence
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
|
2
2
|
import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync, statSync } from "fs";
|
|
3
3
|
import { join } from "path";
|
|
4
|
+
import { z } from "zod";
|
|
4
5
|
import { gateSaltPath } from "../lib/paths";
|
|
6
|
+
import { validateArtifactSchema } from "../lib/artifact-schema";
|
|
7
|
+
import { ShipEvidenceSchema } from "./schema";
|
|
5
8
|
import {
|
|
6
9
|
writeGateResult,
|
|
7
10
|
generateHmac,
|
|
@@ -419,8 +422,10 @@ describe("generateShipEvidence", () => {
|
|
|
419
422
|
|
|
420
423
|
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
421
424
|
|
|
422
|
-
|
|
423
|
-
|
|
425
|
+
// null means the contract refused the artifact (#2255).
|
|
426
|
+
expect(path).not.toBeNull();
|
|
427
|
+
expect(existsSync(path!)).toBe(true);
|
|
428
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
424
429
|
expect(evidence.contractVersion).toBe("1.0");
|
|
425
430
|
expect(evidence.issueNumber).toBe(432);
|
|
426
431
|
expect(evidence.gateOut.status).toBe("PASS");
|
|
@@ -429,6 +434,107 @@ describe("generateShipEvidence", () => {
|
|
|
429
434
|
expect(evidence.capturedAt).toBeTruthy();
|
|
430
435
|
expect(evidence.iterationCount).toBeGreaterThanOrEqual(1);
|
|
431
436
|
expect(evidence.sizing.predicted).toBe("M");
|
|
437
|
+
// #2255 / SC-2770: the artifact meets its declared contract, not only the
|
|
438
|
+
// eight fields this case happens to name.
|
|
439
|
+
expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
test("#2255 / SC-2770: a schema with one required field more than the writer emits refuses the artifact", () => {
|
|
443
|
+
writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP" }), null, 2));
|
|
444
|
+
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
445
|
+
expect(path).not.toBeNull();
|
|
446
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
447
|
+
const stricter = ShipEvidenceSchema.extend({ reviewedBy: z.string() });
|
|
448
|
+
expect(validateArtifactSchema(evidence, stricter).join("\n")).toContain("reviewedBy");
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
test("#2255: an artifact the contract refuses is not written, and the field is named", () => {
|
|
452
|
+
// A verdict outside PASS|FAIL|SKIP is the live way the writer can emit
|
|
453
|
+
// what ShipEvidenceSchema refuses: `result: ac.verdict || "SKIP"`.
|
|
454
|
+
writeFileSync(SF, JSON.stringify(makeState({
|
|
455
|
+
phase: "SHIP",
|
|
456
|
+
acs: [{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }],
|
|
457
|
+
}), null, 2));
|
|
458
|
+
const errors: string[] = [];
|
|
459
|
+
const original = console.error;
|
|
460
|
+
console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); };
|
|
461
|
+
let path: string | null;
|
|
462
|
+
try {
|
|
463
|
+
path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
464
|
+
} finally {
|
|
465
|
+
console.error = original;
|
|
466
|
+
}
|
|
467
|
+
expect(path).toBeNull();
|
|
468
|
+
expect(existsSync(join(TMP_DIR, "ship-evidence.json"))).toBe(false);
|
|
469
|
+
expect(errors.join("\n")).toContain("gateOut.evidence.0.result");
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
test("#2255 review round 1: an OUTCOME criterion still PENDING at ship is written — a legitimate ship-time state", () => {
|
|
473
|
+
// gates/workflow.test.ts: "OUTCOME ACs may be PENDING at ship time — they
|
|
474
|
+
// require Quinn/prove for verification", and enrichment writes the literal
|
|
475
|
+
// "PENDING". The contract refused it, so validation would have dropped the
|
|
476
|
+
// evidence on every normal run with an OUTCOME criterion outstanding.
|
|
477
|
+
writeFileSync(SF, JSON.stringify(makeState({
|
|
478
|
+
phase: "SHIP",
|
|
479
|
+
acs: [{ id: "SC-1", statement: "s", type: "OUTCOME", verdict: "PENDING", evidence: null }],
|
|
480
|
+
}), null, 2));
|
|
481
|
+
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
482
|
+
expect(path).not.toBeNull();
|
|
483
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
484
|
+
expect(evidence.gateOut.evidence[0].result).toBe("PENDING");
|
|
485
|
+
expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
|
|
486
|
+
});
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
describe("#2255 review round 1: a refused artifact is never reported to the issue as shipped", () => {
|
|
490
|
+
// A loopback stub stands in for the GitHub API (lib/github.ts allows exactly
|
|
491
|
+
// GitHub or loopback). `fixture/repo` makes verifiedPush refuse rather than
|
|
492
|
+
// push from this checkout (#430). The control case is what keeps the
|
|
493
|
+
// refusal's "no requests" from being vacuous: the same stub, a valid
|
|
494
|
+
// artifact, and the label and comment arrive.
|
|
495
|
+
async function requestsDuring(acs: Array<Record<string, unknown>>): Promise<string[]> {
|
|
496
|
+
const seen: string[] = [];
|
|
497
|
+
const server = Bun.serve({
|
|
498
|
+
port: 0,
|
|
499
|
+
hostname: "127.0.0.1",
|
|
500
|
+
fetch(req) {
|
|
501
|
+
seen.push(`${req.method} ${new URL(req.url).pathname}`);
|
|
502
|
+
return Response.json({});
|
|
503
|
+
},
|
|
504
|
+
});
|
|
505
|
+
try {
|
|
506
|
+
writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP", acs }), null, 2));
|
|
507
|
+
// A child process, so the credential is SUPPLIED rather than read: the
|
|
508
|
+
// tree resolves it in one place (resolveGitHubToken), and nothing here
|
|
509
|
+
// names it as a read. Async, because this process serves the stub, and
|
|
510
|
+
// a blocking spawn would stall the server the child is calling. The
|
|
511
|
+
// child's pending requests keep it alive until they have landed.
|
|
512
|
+
const child = Bun.spawn(
|
|
513
|
+
["bun", "-e", `import { generateShipEvidence } from ${JSON.stringify(join(import.meta.dir, "orchestrator.ts"))};` +
|
|
514
|
+
`generateShipEvidence(${JSON.stringify(SF)}, ${JSON.stringify(TMP_DIR)}, 432, "fixture/repo", 1, 0, 0);`],
|
|
515
|
+
{
|
|
516
|
+
cwd: join(import.meta.dir, ".."),
|
|
517
|
+
env: { ...process.env, GH_TOKEN: "test-token-not-real", GITHUB_API_URL: `http://127.0.0.1:${server.port}` },
|
|
518
|
+
stdout: "ignore",
|
|
519
|
+
stderr: "ignore",
|
|
520
|
+
},
|
|
521
|
+
);
|
|
522
|
+
await child.exited;
|
|
523
|
+
} finally {
|
|
524
|
+
server.stop(true);
|
|
525
|
+
}
|
|
526
|
+
return seen;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
test("control: a valid artifact labels the issue shipped and posts the comment", async () => {
|
|
530
|
+
const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "PASS", evidence: { content: "ok" } }]);
|
|
531
|
+
expect(seen.some(r => r.endsWith("/issues/432/labels"))).toBe(true);
|
|
532
|
+
expect(seen.some(r => r.endsWith("/issues/432/comments"))).toBe(true);
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
test("a refused artifact sends nothing: no shipped label, no Ship Complete comment", async () => {
|
|
536
|
+
const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }]);
|
|
537
|
+
expect(seen).toEqual([]);
|
|
432
538
|
});
|
|
433
539
|
});
|
|
434
540
|
|
package/gates/orchestrator.ts
CHANGED
|
@@ -2,7 +2,8 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync, statSync
|
|
|
2
2
|
import { join, dirname } from "path";
|
|
3
3
|
import { createHash, createHmac, randomBytes } from "crypto";
|
|
4
4
|
import { execSync } from "child_process";
|
|
5
|
-
import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS } from "./schema";
|
|
5
|
+
import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS, ShipEvidenceSchema } from "./schema";
|
|
6
|
+
import { validateArtifactSchema } from "../lib/artifact-schema";
|
|
6
7
|
import { ZodError } from "zod";
|
|
7
8
|
import { createGitHubClient, addLabels, addComment, type GitHubClient } from "../lib/github";
|
|
8
9
|
import { computeACHash } from "../lib/workflow-security";
|
|
@@ -384,7 +385,7 @@ export function generateShipEvidence(
|
|
|
384
385
|
passes: number,
|
|
385
386
|
fails: number,
|
|
386
387
|
warns: number,
|
|
387
|
-
): string {
|
|
388
|
+
): string | null {
|
|
388
389
|
const state = readState(sf);
|
|
389
390
|
const sha = gitSha();
|
|
390
391
|
const ts = isoNow();
|
|
@@ -483,12 +484,31 @@ export function generateShipEvidence(
|
|
|
483
484
|
...(pushOutcome ? { push: pushOutcome, pushLeftWorkLocal: leftWorkLocal(pushOutcome) } : {}),
|
|
484
485
|
};
|
|
485
486
|
|
|
487
|
+
// #2255 / SC-2770: checked against its one contract before it is written,
|
|
488
|
+
// as prove-evidence.json is (#1559). An artifact the schema refuses is not
|
|
489
|
+
// written, and the fields are named. The gate's verdict is untouched: by
|
|
490
|
+
// here the ship gate has already passed and the push has happened, and the
|
|
491
|
+
// missing artifact is the visible consequence.
|
|
486
492
|
const evidencePath = join(workDir, "ship-evidence.json");
|
|
487
|
-
|
|
493
|
+
const violations = validateArtifactSchema(evidence, ShipEvidenceSchema);
|
|
494
|
+
if (violations.length > 0) {
|
|
495
|
+
console.error("ship gate: ship-evidence.json NOT written — it fails ShipEvidenceSchema (SC-2770):");
|
|
496
|
+
for (const v of violations) console.error(` - ${v}`);
|
|
497
|
+
} else {
|
|
498
|
+
writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
|
|
499
|
+
}
|
|
488
500
|
|
|
489
501
|
// GitHub: add shipped label + post comment via Octokit (best-effort, fire-and-forget)
|
|
490
502
|
// Uses POST additive label strategy (D-5) — never PUT/replace
|
|
491
|
-
|
|
503
|
+
//
|
|
504
|
+
// Not when the contract refused the artifact (#2255): labelling the issue
|
|
505
|
+
// shipped and posting "Ship Complete" beside a refusal is a public success
|
|
506
|
+
// report the run's own record contradicts, and a label is harder to take
|
|
507
|
+
// back than to withhold.
|
|
508
|
+
if (violations.length > 0 && issueRepo && issue > 0) {
|
|
509
|
+
console.error(`ship gate: #${issue} NOT labelled shipped and no Ship Complete comment posted — the evidence was not recorded`);
|
|
510
|
+
}
|
|
511
|
+
if (violations.length === 0 && issueRepo && issue > 0) {
|
|
492
512
|
try {
|
|
493
513
|
const github = createGitHubClient();
|
|
494
514
|
addLabels(github, issueRepo, issue, ["shipped"]).catch(() => {});
|
|
@@ -512,7 +532,7 @@ export function generateShipEvidence(
|
|
|
512
532
|
} catch {}
|
|
513
533
|
}
|
|
514
534
|
|
|
515
|
-
return evidencePath;
|
|
535
|
+
return violations.length > 0 ? null : evidencePath;
|
|
516
536
|
}
|
|
517
537
|
|
|
518
538
|
const GATE_PENDING_PATH = join(
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
doc-type: reference
|
|
3
3
|
status: active
|
|
4
4
|
owner: jason
|
|
5
|
-
updated: 2026-
|
|
5
|
+
updated: 2026-11-15
|
|
6
6
|
---
|
|
7
7
|
|
|
8
8
|
You are a prove reproducer. Your objective: reproduce the bug described
|
|
@@ -44,9 +44,22 @@ Output JSON:
|
|
|
44
44
|
"evidence": [
|
|
45
45
|
{"type": "screenshot|api-response|console", "description": "what this shows"}
|
|
46
46
|
],
|
|
47
|
+
"afterEvidence": {
|
|
48
|
+
"type": "text|api|data",
|
|
49
|
+
"path": null,
|
|
50
|
+
"capturedAt": "ISO-8601 time you observed the after-state",
|
|
51
|
+
"environment": "local|prod",
|
|
52
|
+
"description": "what the after-state showed"
|
|
53
|
+
},
|
|
47
54
|
"gaps": []
|
|
48
55
|
}
|
|
49
56
|
|
|
57
|
+
`afterEvidence` is the after-state you observed yourself. `environment` is
|
|
58
|
+
where you observed it: `local` for this checkout or a dev server, `prod` for
|
|
59
|
+
the production URL. If you observed no after-state, leave `afterEvidence`
|
|
60
|
+
out. Never fill it in to make the record complete: a PROVEN without it is
|
|
61
|
+
recorded as INCONCLUSIVE.
|
|
62
|
+
|
|
50
63
|
Verdict rules:
|
|
51
64
|
- PROVEN: bug was reproduced in before-state AND fix verified in after-state
|
|
52
65
|
- UNPROVEN: fix does not resolve the issue (evidence shows problem persists)
|
package/gates/prompts.sha256
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
ba56e1fcb00c7a9bc1f2b05beca701cb264785e017cd2bf54a2dd07cdde331df ac-adversary.md
|
|
2
2
|
7d4ff0ef4b787e68b59643c11ba51e62a9411f5e24509b97855a337598c67508 evidence-validator.md
|
|
3
|
-
|
|
3
|
+
6f15ff6891118f749894f55f18fa21234211f993cf6eadfb3c90561a64b91691 prove-reproducer.md
|
package/gates/prove.test.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { test, expect, describe } from "bun:test";
|
|
2
2
|
import { readFileSync, existsSync, writeFileSync } from "fs";
|
|
3
3
|
import { execSync } from "child_process";
|
|
4
|
-
import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState } from "./gate-executor";
|
|
4
|
+
import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState, validateArtifactSchema } from "./gate-executor";
|
|
5
|
+
import { ProveEvidenceSchema } from "./schema";
|
|
5
6
|
|
|
6
7
|
const TEST_DIR = process.env.TEST_WORK_DIR || "";
|
|
7
8
|
const PROJECT_ROOT = process.env.PROJECT_ROOT || process.cwd();
|
|
@@ -145,6 +146,10 @@ describe("prove: run evidence commands from workflow-state", () => {
|
|
|
145
146
|
criteriaResults,
|
|
146
147
|
};
|
|
147
148
|
|
|
149
|
+
// #2254 / SC-2770: checked against the one contract before it is written,
|
|
150
|
+
// as B3's and prove.js's are. A required field this writer stops emitting
|
|
151
|
+
// fails here, with the field named, instead of in whatever reads it next.
|
|
152
|
+
expect(validateArtifactSchema(evidence, ProveEvidenceSchema)).toEqual([]);
|
|
148
153
|
writeFileSync(`${TEST_DIR}/prove-evidence.json`, JSON.stringify(evidence, null, 2));
|
|
149
154
|
expect(verdict).toBe("PROVEN");
|
|
150
155
|
});
|
package/gates/schema.ts
CHANGED
|
@@ -1193,7 +1193,10 @@ export const ShipEvidenceSchema = z.object({
|
|
|
1193
1193
|
status: z.enum(["PASS", "FAIL"]),
|
|
1194
1194
|
evidence: z.array(z.object({
|
|
1195
1195
|
criterion: z.string(),
|
|
1196
|
-
|
|
1196
|
+
// PENDING is a legitimate ship-time state (#2255): an OUTCOME criterion
|
|
1197
|
+
// waits for Quinn or /prove, and the verify gate passes it through
|
|
1198
|
+
// (checkVerifyPendingACs). The writer copies the criterion's verdict.
|
|
1199
|
+
result: z.enum(["PASS", "FAIL", "SKIP", "PENDING"]),
|
|
1197
1200
|
detail: z.any().optional(),
|
|
1198
1201
|
})),
|
|
1199
1202
|
}),
|
|
@@ -1252,9 +1255,26 @@ const BeforeEvidenceSchema = z.union([
|
|
|
1252
1255
|
}),
|
|
1253
1256
|
]);
|
|
1254
1257
|
|
|
1258
|
+
/**
|
|
1259
|
+
* One row of `prove-evidence.json`'s `criteriaResults` (#1559). The shape all
|
|
1260
|
+
* three writers already emit — B3's reproducer, `gates/prove.test.ts` and
|
|
1261
|
+
* `workflows/prove.js` — and the one the deleted
|
|
1262
|
+
* `config/prove-evidence.schema.json` declared. SKIP is the reproducer's
|
|
1263
|
+
* verdict for a UI criterion it has no browser to check.
|
|
1264
|
+
*/
|
|
1265
|
+
const CriterionResultSchema = z.object({
|
|
1266
|
+
scId: z.string(),
|
|
1267
|
+
verdict: z.enum(["PASS", "FAIL", "SKIP"]),
|
|
1268
|
+
evidence: z.string(),
|
|
1269
|
+
});
|
|
1270
|
+
|
|
1255
1271
|
export const ProveEvidenceSchema = z.object({
|
|
1256
1272
|
issueNumber: z.number(),
|
|
1257
1273
|
verdict: z.enum(["PROVEN", "UNPROVEN", "INCONCLUSIVE"]),
|
|
1274
|
+
// Required, but not `.min(1)`: an INCONCLUSIVE with no criteria is a record
|
|
1275
|
+
// `gates/prove.test.ts` legitimately writes. That a PROVEN needs a PASS row
|
|
1276
|
+
// is `buildProveEvidence`'s rule, because it is about two fields at once.
|
|
1277
|
+
criteriaResults: z.array(CriterionResultSchema),
|
|
1258
1278
|
commitSHA: z.string(),
|
|
1259
1279
|
capturedAt: z.string(),
|
|
1260
1280
|
beforeEvidence: BeforeEvidenceSchema,
|
package/gates/workflow.test.ts
CHANGED
|
@@ -453,6 +453,18 @@ export function verifyFanoutViolations(label: string, val: unknown, tier: unknow
|
|
|
453
453
|
`is an absence with extra steps (#126)`,
|
|
454
454
|
];
|
|
455
455
|
}
|
|
456
|
+
// #2259: the agent ran and its reply could not be read. Before, that reply
|
|
457
|
+
// left no record, which the absence branch above refused, so the ship
|
|
458
|
+
// outcome is unchanged. What changes is that the refusal names the cause
|
|
459
|
+
// instead of failing on a missing `gameable` further down.
|
|
460
|
+
if ((val as { status?: unknown }).status === "UNMEASURED") {
|
|
461
|
+
const reason = (val as { reason?: unknown }).reason;
|
|
462
|
+
return [
|
|
463
|
+
`gates.verify.${label} is UNMEASURED — the agent ran and its reply could not be read` +
|
|
464
|
+
(typeof reason === "string" && reason.trim() !== "" ? `: ${reason}` : "") +
|
|
465
|
+
`. An unread reply is not an approval (#126, #2259).`,
|
|
466
|
+
];
|
|
467
|
+
}
|
|
456
468
|
return [];
|
|
457
469
|
}
|
|
458
470
|
|
|
@@ -224,8 +224,19 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
|
|
|
224
224
|
const a = anchorOnlyKey(k);
|
|
225
225
|
if (a !== null) anchors.add(a);
|
|
226
226
|
}
|
|
227
|
+
// #2257: how many findings cite a line, and how many of those lines could
|
|
228
|
+
// be read. Read from the SAME calls the keys are built from, not a second
|
|
229
|
+
// pass, so the count cannot disagree with what the keys saw.
|
|
230
|
+
let located = 0;
|
|
231
|
+
let readable = 0;
|
|
227
232
|
const repeat = candidates.map(c => {
|
|
228
|
-
const
|
|
233
|
+
const cited = c as ReviewFinding;
|
|
234
|
+
const lineText = inputs.lineTextFor(cited);
|
|
235
|
+
if (typeof cited?.file === "string" && Number.isInteger(cited?.line) && (cited.line as number) >= 1) {
|
|
236
|
+
located++;
|
|
237
|
+
if (lineText !== undefined) readable++;
|
|
238
|
+
}
|
|
239
|
+
const key = carryKey(c, lineText);
|
|
229
240
|
// Unkeyable CARRIES. It cannot be shown to be a repeat, and the
|
|
230
241
|
// direction this is allowed to be wrong in is toward reporting.
|
|
231
242
|
if (key === null) return null;
|
|
@@ -282,7 +293,20 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
|
|
|
282
293
|
// this whole increment is a correction for.
|
|
283
294
|
return null;
|
|
284
295
|
});
|
|
285
|
-
|
|
296
|
+
// #2257. A gate that could read no cited line built every key from a line
|
|
297
|
+
// NUMBER, which never equals a key an earlier round recorded from line
|
|
298
|
+
// TEXT. "Nothing to suppress" from that gate is a reading of nothing, and
|
|
299
|
+
// it printed the same words as a gate that read every line (#1872: 0 of
|
|
300
|
+
// 420 in CI). Byte-key matches above still stand; this says the anchor
|
|
301
|
+
// half was blind.
|
|
302
|
+
const unmeasured =
|
|
303
|
+
located > 0 && readable === 0 && prior.keys.length > 0
|
|
304
|
+
? `no cited line could be read for any of this round's ${located} finding(s) that cite one, ` +
|
|
305
|
+
`so no key could match one an earlier round recorded from the line's text, and a ` +
|
|
306
|
+
`restated finding could not be recognised. The checkout the gate reads is probably not ` +
|
|
307
|
+
`the one under review (#2257).`
|
|
308
|
+
: null;
|
|
309
|
+
return { repeat, unmeasured };
|
|
286
310
|
};
|
|
287
311
|
}
|
|
288
312
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Check a gate artifact against its declared schema, before it is written
|
|
3
|
+
* (#1559, #2255, SC-2770).
|
|
4
|
+
*
|
|
5
|
+
* Lives in `lib/` rather than beside its first caller in
|
|
6
|
+
* `gates/gate-executor.ts` because `gates/orchestrator.ts` needs it too, and
|
|
7
|
+
* gate-executor already imports orchestrator. `gates/gate-executor.ts`
|
|
8
|
+
* re-exports both names, so there is one definition and every existing import
|
|
9
|
+
* keeps working.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Anything with zod's `safeParse` shape. Structural, so callers keep no
|
|
14
|
+
* direct zod import and a test can hand it a stricter copy of a schema.
|
|
15
|
+
*/
|
|
16
|
+
export interface ArtifactSchema {
|
|
17
|
+
safeParse(value: unknown): {
|
|
18
|
+
success: boolean;
|
|
19
|
+
error?: { issues: ReadonlyArray<{ path: ReadonlyArray<PropertyKey>; message: string }> };
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* One `path: message` line per violation, `[]` when it validates.
|
|
25
|
+
*
|
|
26
|
+
* `config/prove-evidence.schema.json` declared a contract nothing read, so
|
|
27
|
+
* writer and contract drifted apart: B3's artifact failed both the JSON
|
|
28
|
+
* schema and the zod one, and no check said so. A refusal that names no
|
|
29
|
+
* issue is still a refusal — an empty list there would read as a pass.
|
|
30
|
+
*/
|
|
31
|
+
export function validateArtifactSchema(artifact: unknown, schema: ArtifactSchema): string[] {
|
|
32
|
+
const result = schema.safeParse(artifact);
|
|
33
|
+
if (result.success) return [];
|
|
34
|
+
const issues = result.error?.issues ?? [];
|
|
35
|
+
if (issues.length === 0) return ["(root): the schema refused the artifact without naming a field"];
|
|
36
|
+
return issues.map((i) => `${i.path.length > 0 ? i.path.map(String).join(".") : "(root)"}: ${i.message}`);
|
|
37
|
+
}
|
|
@@ -1822,6 +1822,20 @@ export const AI_REVIEW_LEDGER_SNAPSHOT_BASENAME = "ai-review-ledger-snapshot.txt
|
|
|
1822
1822
|
*/
|
|
1823
1823
|
export const AI_REVIEW_THREADS_PATH_VAR = "AI_REVIEW_THREADS_PATH";
|
|
1824
1824
|
|
|
1825
|
+
/**
|
|
1826
|
+
* The checkout the adjudicator reads cited lines from (#2257).
|
|
1827
|
+
*
|
|
1828
|
+
* The adjudicator runs from the payload's own directory, outside the
|
|
1829
|
+
* checkout, because `bunx` resolves bunfig.toml, .npmrc and package.json by
|
|
1830
|
+
* walking up from its working directory, so the PR under review must not be
|
|
1831
|
+
* the tree it starts in. That made `process.cwd()` the payload directory,
|
|
1832
|
+
* the suppression gate could read no cited line, and the anchor rule matched
|
|
1833
|
+
* 0 of 420 round-2+ findings in CI (#1872). The root has to arrive by name.
|
|
1834
|
+
* `scripts/ai-review-findings.ts` falls back to its working directory when
|
|
1835
|
+
* this is unset, which is what a step generated before it was set does.
|
|
1836
|
+
*/
|
|
1837
|
+
export const AI_REVIEW_CHECKOUT_ROOT_VAR = "AI_REVIEW_CHECKOUT_ROOT";
|
|
1838
|
+
|
|
1825
1839
|
/** The basename inside the findings directory. One spelling. */
|
|
1826
1840
|
export const AI_REVIEW_THREADS_BASENAME = "ai-review-threads.json";
|
|
1827
1841
|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `.github/workflows/intake.yml`, generated — #2233.
|
|
3
|
+
*
|
|
4
|
+
* Jason decided on #1126 (2026-11-15) that each intake source runs on its own
|
|
5
|
+
* trigger, not on scaffold: "scaffold" reads as a one-time setup step, so
|
|
6
|
+
* nobody re-runs it to re-mine, and intake should run when its data appears.
|
|
7
|
+
* For `fix-commits` the data is the history of the default branch, and the
|
|
8
|
+
* trigger is a push to it.
|
|
9
|
+
*
|
|
10
|
+
* ## Full history, or the reading is of the clone
|
|
11
|
+
*
|
|
12
|
+
* `mineFixCommits` reports `SHALLOW_HISTORY` on a shallow clone, and
|
|
13
|
+
* `actions/checkout` clones to depth 1 unless told otherwise. That depth is
|
|
14
|
+
* the only reason the source read UNMEASURED on a runner, so the checkout
|
|
15
|
+
* sets `fetch-depth: 0`.
|
|
16
|
+
*
|
|
17
|
+
* ## Advisory, and never silent
|
|
18
|
+
*
|
|
19
|
+
* `rungate intake` exits 0 on every reading (#2248), so a reading never turns
|
|
20
|
+
* the run red: an unreadable source says nothing about the push that fired
|
|
21
|
+
* it. UNMEASURED is printed as the command prints it and raised as a warning
|
|
22
|
+
* annotation, so it is visible on the run without being mistaken for zero.
|
|
23
|
+
* A non-zero exit is the command refusing its arguments or `bunx` failing,
|
|
24
|
+
* which is the tool broken, and the step goes red on it.
|
|
25
|
+
*
|
|
26
|
+
* ## It calls the published CLI
|
|
27
|
+
*
|
|
28
|
+
* A consumer has `node_modules/rungate`, not `scripts/`, so the step runs
|
|
29
|
+
* `bunx rungate@<version> intake`, at the generating version like every other
|
|
30
|
+
* pin. `intake` first shipped in 0.8.23 (#2250).
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
/** The file this module generates, under `.github/workflows/`. */
|
|
34
|
+
export const INTAKE_WORKFLOW_FILE = "intake.yml";
|
|
35
|
+
|
|
36
|
+
/** The one job in it. */
|
|
37
|
+
export const INTAKE_JOB = "fix-commit-intake";
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The CLI command the step runs. `lib/cli-commands.ts` must register it, or
|
|
41
|
+
* the workflow calls a command `bin/rungate.ts` refuses (#675's defect).
|
|
42
|
+
*/
|
|
43
|
+
export const INTAKE_COMMAND = "intake";
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The step's `run:` body. Exported so `test/intake-workflow.test.ts` executes
|
|
47
|
+
* THIS text against a stub `bunx`, rather than a paraphrase.
|
|
48
|
+
*
|
|
49
|
+
* The output is captured so it can be both printed and inspected. Under
|
|
50
|
+
* `bash -e` a failing command substitution in an assignment stops the step,
|
|
51
|
+
* which is the red a broken tool should produce.
|
|
52
|
+
*/
|
|
53
|
+
export function buildIntakeScript(version: string): string {
|
|
54
|
+
return `out=$(bunx rungate@${version} ${INTAKE_COMMAND} --source fix-commits)
|
|
55
|
+
printf '%s\\n' "$out"
|
|
56
|
+
case "$out" in
|
|
57
|
+
*UNMEASURED*)
|
|
58
|
+
echo "::warning::UNMEASURED — the fix-commits intake reading could not be taken on this run; the reason is printed above. This is not zero candidates."
|
|
59
|
+
;;
|
|
60
|
+
esac
|
|
61
|
+
`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface IntakeWorkflowOptions {
|
|
65
|
+
/** The `runs-on:` value, already JSON-quoted by the caller. */
|
|
66
|
+
runsOn: string;
|
|
67
|
+
/** The pinned `actions/checkout` reference. */
|
|
68
|
+
checkout: string;
|
|
69
|
+
/** The pinned `oven-sh/setup-bun` reference. */
|
|
70
|
+
setupBun: string;
|
|
71
|
+
/** The `with:` body of that step, at ten spaces — the project's single bun declaration (#622). */
|
|
72
|
+
setupBunWith: string;
|
|
73
|
+
/** The step's `env:` block, from `setupBunEnv` (#2039). */
|
|
74
|
+
setupBunEnv: string;
|
|
75
|
+
/** The published harness version the step runs through `bunx`. */
|
|
76
|
+
version: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The whole `intake.yml` body. */
|
|
80
|
+
export function buildIntakeWorkflow(opts: IntakeWorkflowOptions): string {
|
|
81
|
+
const indent = (text: string, pad: string) =>
|
|
82
|
+
text
|
|
83
|
+
.trimEnd()
|
|
84
|
+
.split("\n")
|
|
85
|
+
.map(line => (line === "" ? "" : `${pad}${line}`))
|
|
86
|
+
.join("\n");
|
|
87
|
+
|
|
88
|
+
return `# Managed by rungate — do not edit. Customize the ci section of .claude/rungate/config.json.
|
|
89
|
+
#
|
|
90
|
+
# Mines the default branch's fix commits into rule candidates on every push to
|
|
91
|
+
# it (#2233). Jason's #1126 decision: each intake source runs on its own
|
|
92
|
+
# trigger, never on scaffold. Advisory: a reading never fails the run, and an
|
|
93
|
+
# UNMEASURED one is raised as a warning rather than read as zero.
|
|
94
|
+
name: Intake — fix commits
|
|
95
|
+
|
|
96
|
+
on:
|
|
97
|
+
push:
|
|
98
|
+
branches: [main]
|
|
99
|
+
|
|
100
|
+
permissions:
|
|
101
|
+
contents: read
|
|
102
|
+
|
|
103
|
+
jobs:
|
|
104
|
+
${INTAKE_JOB}:
|
|
105
|
+
runs-on: ${opts.runsOn}
|
|
106
|
+
steps:
|
|
107
|
+
# Full history: a depth-1 clone is the only reason this source reads
|
|
108
|
+
# UNMEASURED (SHALLOW_HISTORY) on a runner.
|
|
109
|
+
- uses: ${opts.checkout}
|
|
110
|
+
with:
|
|
111
|
+
fetch-depth: 0
|
|
112
|
+
- uses: ${opts.setupBun}
|
|
113
|
+
with:
|
|
114
|
+
${opts.setupBunWith}
|
|
115
|
+
${opts.setupBunEnv}
|
|
116
|
+
- name: Mine fix commits into rule candidates
|
|
117
|
+
run: |
|
|
118
|
+
${indent(buildIntakeScript(opts.version), " ")}
|
|
119
|
+
`;
|
|
120
|
+
}
|
|
@@ -210,6 +210,16 @@ export const OWNED_FILES: OwnedFile[] = [
|
|
|
210
210
|
customize: "Set `board.id` in config and secret `RUNGATE_PROJECT_TOKEN`; add your own jobs alongside the generated `add-to-board` job",
|
|
211
211
|
avoid: "Don't edit the generated `add-to-board` job",
|
|
212
212
|
},
|
|
213
|
+
{
|
|
214
|
+
// #2233: fix-commit intake on push to main, with full history. Jason's
|
|
215
|
+
// #1126 decision put each intake source on its own trigger; a consumer has
|
|
216
|
+
// no scripts/, so the generator writes it and calls the published CLI.
|
|
217
|
+
path: ".github/workflows/intake.yml",
|
|
218
|
+
strategy: "job-preserving",
|
|
219
|
+
servedBy: "writeManagedFile (lib/scaffold/steps.ts) via mergeManagedWorkflow — the harness owns named jobs, not the file",
|
|
220
|
+
customize: "Advisory — a reading never fails the run; add your own jobs alongside the generated `fix-commit-intake` job",
|
|
221
|
+
avoid: "Don't edit the generated `fix-commit-intake` job, and don't drop `fetch-depth: 0` — a shallow clone reads UNMEASURED",
|
|
222
|
+
},
|
|
213
223
|
{
|
|
214
224
|
// `regenerate` until #655, and the declaration was never consulted:
|
|
215
225
|
// `generateAgentBriefsStep` called `writeFileSync` directly, so the row
|
package/lib/scaffold/steps.ts
CHANGED
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
} from "./ai-review-unmeasured-workflow";
|
|
51
51
|
import { buildAdrAcceptedWorkflow } from "./adr-accepted-workflow";
|
|
52
52
|
import { buildClosingReferencesWorkflow } from "./closing-references-workflow";
|
|
53
|
+
import { buildIntakeWorkflow } from "./intake-workflow";
|
|
53
54
|
import { buildBoardReadyWorkflow } from "./board-ready-workflow";
|
|
54
55
|
import { buildBoardIntakeWorkflow } from "./board-intake-workflow";
|
|
55
56
|
import { CI_SUITE_FETCH_DEPTH, baseRefFetchDepth } from "../ci-history-depth";
|
|
@@ -4515,6 +4516,18 @@ ${buildAuthDetectStep(vertex)}${buildReviewRunStep(
|
|
|
4515
4516
|
version: harnessVersion(),
|
|
4516
4517
|
});
|
|
4517
4518
|
|
|
4519
|
+
// #2233: fix-commit intake on every push to main, with full history (Jason's
|
|
4520
|
+
// #1126 decision: each intake source on its own trigger, never on scaffold).
|
|
4521
|
+
// It calls the published CLI's `intake`, first shipped in 0.8.23.
|
|
4522
|
+
const intakeYml = buildIntakeWorkflow({
|
|
4523
|
+
runsOn,
|
|
4524
|
+
checkout: actionRef("actions/checkout"),
|
|
4525
|
+
setupBun: actionRef("oven-sh/setup-bun"),
|
|
4526
|
+
setupBunWith: bunStep,
|
|
4527
|
+
setupBunEnv: setupBunEnv(" "),
|
|
4528
|
+
version: harnessVersion(),
|
|
4529
|
+
});
|
|
4530
|
+
|
|
4518
4531
|
// #2166, ADR-014 row 4: new issues and pull requests onto the board. It
|
|
4519
4532
|
// names no board: the published CLI prints board.id from the default
|
|
4520
4533
|
// branch's config (#2173), and the add uses that node id.
|
|
@@ -4542,7 +4555,7 @@ ${buildAuthDetectStep(vertex)}${buildReviewRunStep(
|
|
|
4542
4555
|
// lib/conformity.ts reads back, and the two are held together by
|
|
4543
4556
|
// test/ai-review-workflow.test.ts AC-1, which looks for the constant's
|
|
4544
4557
|
// filename in what this loop actually wrote.
|
|
4545
|
-
for (const [name, content] of [["ci.yml", ciYml], ["gates.yml", gatesYml], ["ai-review.yml", aiReviewYml], ["ai-review-unmeasured.yml", aiReviewUnmeasuredYml], ["adr-accepted.yml", adrAcceptedYml], ["closing-references.yml", closingReferencesYml], ["board-ready.yml", boardReadyYml], ["board-intake.yml", boardIntakeYml]] as const) {
|
|
4558
|
+
for (const [name, content] of [["ci.yml", ciYml], ["gates.yml", gatesYml], ["ai-review.yml", aiReviewYml], ["ai-review-unmeasured.yml", aiReviewUnmeasuredYml], ["adr-accepted.yml", adrAcceptedYml], ["closing-references.yml", closingReferencesYml], ["board-ready.yml", boardReadyYml], ["board-intake.yml", boardIntakeYml], ["intake.yml", intakeYml]] as const) {
|
|
4546
4559
|
const label = `.github/workflows/${name}`;
|
|
4547
4560
|
try {
|
|
4548
4561
|
// Keyed by FILE, and the lookup is here because this loop is the only
|
package/package.json
CHANGED
|
@@ -94,7 +94,7 @@
|
|
|
94
94
|
import { aiReviewJuryGate } from "../lib/ai-review-jury";
|
|
95
95
|
import { productionSuppression } from "../lib/ai-review-suppression";
|
|
96
96
|
import { main } from "../lib/scaffold/ai-review-findings";
|
|
97
|
-
import { AI_REVIEW_BALLOT_DIR_ENV, AI_REVIEW_THREADS_PATH_VAR } from "../lib/scaffold/ai-review-step";
|
|
97
|
+
import { AI_REVIEW_BALLOT_DIR_ENV, AI_REVIEW_CHECKOUT_ROOT_VAR, AI_REVIEW_THREADS_PATH_VAR } from "../lib/scaffold/ai-review-step";
|
|
98
98
|
|
|
99
99
|
const ballotDir = process.env[AI_REVIEW_BALLOT_DIR_ENV];
|
|
100
100
|
const jury =
|
|
@@ -105,9 +105,13 @@ const jury =
|
|
|
105
105
|
// Assembled in `lib/`, not here. This command and `ai-review-threads.ts` must
|
|
106
106
|
// form the SAME gate from the same file, and two hand-written literals is how
|
|
107
107
|
// `anchorOccurrences` reached one of them and not the other (#1861 round 4).
|
|
108
|
+
// #2257: the checkout by name. The generated step runs this from the payload's
|
|
109
|
+
// directory on purpose, so the working directory is not the checkout; it is
|
|
110
|
+
// the fallback only for a step generated before the variable was set.
|
|
111
|
+
const checkoutRoot = process.env[AI_REVIEW_CHECKOUT_ROOT_VAR];
|
|
108
112
|
const { gate: suppress } = productionSuppression(
|
|
109
113
|
process.env[AI_REVIEW_THREADS_PATH_VAR],
|
|
110
|
-
process.cwd(),
|
|
114
|
+
checkoutRoot !== undefined && checkoutRoot !== "" ? checkoutRoot : process.cwd(),
|
|
111
115
|
);
|
|
112
116
|
|
|
113
117
|
// One line, and `process.exit(main(` is load-bearing text: AC-4 in
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"title": "Prove Evidence",
|
|
4
|
-
"description": "Output artifact from /prove skill",
|
|
5
|
-
"type": "object",
|
|
6
|
-
"required": ["contractVersion", "issueNumber", "verdict", "commitSHA", "capturedAt", "criteriaResults"],
|
|
7
|
-
"properties": {
|
|
8
|
-
"contractVersion": { "type": "string" },
|
|
9
|
-
"issueNumber": { "type": "integer" },
|
|
10
|
-
"verdict": { "type": "string", "enum": ["PROVEN", "UNPROVEN", "INCONCLUSIVE"] },
|
|
11
|
-
"commitSHA": { "type": "string" },
|
|
12
|
-
"capturedAt": { "type": "string", "format": "date-time" },
|
|
13
|
-
"beforeEvidence": {
|
|
14
|
-
"$comment": "#1540: UNMEASURED declares no type. The old shape filled an absent before-state with type \"none\", which is a value /goal legitimately writes for a run it did measure, so the two were indistinguishable. Artefacts written before #1540 carry no status and remain valid.",
|
|
15
|
-
"type": "object",
|
|
16
|
-
"properties": {
|
|
17
|
-
"status": { "type": "string", "enum": ["CAPTURED", "DECLARED_NONE", "UNMEASURED"] },
|
|
18
|
-
"type": { "type": "string", "enum": ["text", "screenshot", "api", "data", "none"] },
|
|
19
|
-
"path": { "type": ["string", "null"] },
|
|
20
|
-
"capturedAt": { "type": ["string", "null"] },
|
|
21
|
-
"description": { "type": "string" },
|
|
22
|
-
"reason": { "type": "string" }
|
|
23
|
-
},
|
|
24
|
-
"allOf": [
|
|
25
|
-
{
|
|
26
|
-
"$comment": "An unmeasured reading says why, and never claims a type.",
|
|
27
|
-
"if": { "properties": { "status": { "const": "UNMEASURED" } }, "required": ["status"] },
|
|
28
|
-
"then": { "required": ["reason"], "not": { "required": ["type"] } }
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
"$comment": "Any reading that is not UNMEASURED was taken, so it carries when.",
|
|
32
|
-
"if": {
|
|
33
|
-
"properties": { "status": { "enum": ["CAPTURED", "DECLARED_NONE"] } },
|
|
34
|
-
"required": ["status"]
|
|
35
|
-
},
|
|
36
|
-
"then": { "required": ["type", "capturedAt"] }
|
|
37
|
-
}
|
|
38
|
-
]
|
|
39
|
-
},
|
|
40
|
-
"afterEvidence": {
|
|
41
|
-
"type": "object",
|
|
42
|
-
"properties": {
|
|
43
|
-
"type": { "type": "string", "enum": ["text", "screenshot", "api"] },
|
|
44
|
-
"path": { "type": "string" },
|
|
45
|
-
"capturedAt": { "type": "string", "format": "date-time" },
|
|
46
|
-
"environment": { "type": "string", "enum": ["local", "prod"] },
|
|
47
|
-
"description": { "type": "string" }
|
|
48
|
-
}
|
|
49
|
-
},
|
|
50
|
-
"comparisonSummary": { "type": "string" },
|
|
51
|
-
"quinnVerdict": { "type": "string", "enum": ["PASS", "FAIL", "SKIP"] },
|
|
52
|
-
"reproduced": { "type": "boolean" },
|
|
53
|
-
"criteriaResults": {
|
|
54
|
-
"type": "array",
|
|
55
|
-
"minItems": 1,
|
|
56
|
-
"items": {
|
|
57
|
-
"type": "object",
|
|
58
|
-
"required": ["scId", "verdict", "evidence"],
|
|
59
|
-
"properties": {
|
|
60
|
-
"scId": { "type": "string" },
|
|
61
|
-
"verdict": { "type": "string", "enum": ["PASS", "FAIL", "SKIP"] },
|
|
62
|
-
"evidence": { "type": "string" },
|
|
63
|
-
"evidencePath": { "type": ["string", "null"] },
|
|
64
|
-
"exitCode": { "type": ["integer", "null"] }
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
},
|
|
68
|
-
"gaps": {
|
|
69
|
-
"type": "array",
|
|
70
|
-
"items": {
|
|
71
|
-
"type": "object",
|
|
72
|
-
"required": ["ac", "status"],
|
|
73
|
-
"properties": {
|
|
74
|
-
"ac": { "type": "string" },
|
|
75
|
-
"status": { "type": "string" },
|
|
76
|
-
"detail": { "type": "string" },
|
|
77
|
-
"priority": { "type": ["string", "null"] },
|
|
78
|
-
"followUpIssue": { "type": ["string", "null"] }
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|