rungate 0.8.23 → 0.8.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/gates/.gate-salt CHANGED
@@ -1 +1 @@
1
- e387efb0e00e0aa4e64f43d5fed7be9d6279267723195e1531aff90000f927f7
1
+ 55059bcaab0d2a5e265a824747d8d98b6a512c0b49e3b984deb3bc1db05deba3
@@ -38,7 +38,7 @@ import {
38
38
  measureEvidencePrevalidation,
39
39
  type EvidencePrevalidationReading,
40
40
  } from "../lib/evidence-prevalidator";
41
- import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
41
+ import { AdversaryReportSchema, EvidencePrevalidationSchema, ProveEvidenceSchema, type PrevalidationSource } from "./schema";
42
42
  import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
43
43
  import { deepMerge } from "../lib/deep-merge";
44
44
  import { livenessPlan, probeTarget } from "../lib/dev-liveness";
@@ -426,6 +426,170 @@ export function comparisonSummaryFor(
426
426
  return `compared against the ${reading.type} before-state captured at ${reading.capturedAt}; ${counts}`;
427
427
  }
428
428
 
429
+ // The validator moved to lib/ in #2255 so gates/orchestrator.ts can use it
430
+ // without importing this file back; re-exported so it has one definition.
431
+ import { validateArtifactSchema, type ArtifactSchema } from "../lib/artifact-schema";
432
+ export { validateArtifactSchema, type ArtifactSchema };
433
+
434
+ const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
435
+ /**
436
+ * The after-evidence types a reproducer may report, and the prompt's list
437
+ * exactly. `none` is B3's to write. `screenshot` is not the reproducer's to
438
+ * claim: it has Bash and Read only (gates/prompts/prove-reproducer.md), so a
439
+ * reported screenshot is a UI verification it could not have made.
440
+ */
441
+ const REPORTED_AFTER_TYPES = ["text", "api", "data"];
442
+ const AFTER_ENVIRONMENTS = ["local", "prod"];
443
+
444
+ /** What B3 knows about the run that the reproducer does not get to say. */
445
+ export interface ProveEvidenceContext {
446
+ issueNumber: number;
447
+ commitSHA: string;
448
+ capturedAt: string;
449
+ beforeReading: BeforeStateReading;
450
+ /** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
451
+ source?: string;
452
+ }
453
+
454
+ type AfterEvidence = {
455
+ type: string; path: string | null; capturedAt: string; environment: string; description: string;
456
+ };
457
+
458
+ /** The reproducer's `afterEvidence`, or why there is none. Never repaired into shape. */
459
+ function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
460
+ if (raw === undefined || raw === null) return { ok: false, reason: "the reproducer reported no afterEvidence" };
461
+ if (typeof raw !== "object" || Array.isArray(raw)) {
462
+ return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
463
+ }
464
+ const a = raw as Record<string, unknown>;
465
+ if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
466
+ return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
467
+ }
468
+ if (typeof a.type !== "string" || !REPORTED_AFTER_TYPES.includes(a.type)) {
469
+ return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one of ${REPORTED_AFTER_TYPES.join("|")}` };
470
+ }
471
+ if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
472
+ return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
473
+ }
474
+ if (typeof a.description !== "string" || a.description.trim() === "") {
475
+ return { ok: false, reason: "afterEvidence.description is empty — it must say what the after-state showed" };
476
+ }
477
+ return {
478
+ ok: true,
479
+ value: {
480
+ type: a.type, path: typeof a.path === "string" ? a.path : null,
481
+ capturedAt: a.capturedAt, environment: a.environment, description: a.description,
482
+ },
483
+ };
484
+ }
485
+
486
+ /**
487
+ * Build `prove-evidence.json` from the reproducer's report (#1559, Jason's
488
+ * option (a) of 2026-11-15).
489
+ *
490
+ * B3 used to write `{issueNumber, verdict, commitSHA, capturedAt,
491
+ * criteriaResults, reproduced, evidence, source}`, which lacks the
492
+ * `beforeEvidence`, `afterEvidence` and `comparisonSummary` that
493
+ * `ProveEvidenceSchema` requires. The before-evidence and the comparison are
494
+ * B3's own readings (#1540), so a reproducer cannot overwrite them. The
495
+ * after-evidence is the reproducer's report of the environment it tested and
496
+ * what it saw. When that report is missing or malformed, the record is
497
+ * `type: "none"` and a PROVEN becomes INCONCLUSIVE, because a proof with no
498
+ * observed after-state is not a proof.
499
+ *
500
+ * The `none` record says `environment: "local"` because the schema requires
501
+ * a value. `local` is the one that satisfies no STANDARD+ prod requirement
502
+ * (gates/prove.test.ts), so a missing after-evidence can never pass that check.
503
+ */
504
+ export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
505
+ const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
506
+ const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
507
+ const after = readReportedAfterEvidence(raw?.afterEvidence);
508
+
509
+ let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
510
+ let verdictDowngrade: { reported: unknown; reason: string } | undefined;
511
+ if (verdict !== raw?.verdict) {
512
+ verdictDowngrade = {
513
+ reported: raw?.verdict,
514
+ reason: `verdict ${JSON.stringify(raw?.verdict)} is not one of ${PROVE_VERDICTS.join("|")}`,
515
+ };
516
+ } else if (verdict === "PROVEN" && !after.ok) {
517
+ verdict = "INCONCLUSIVE";
518
+ verdictDowngrade = { reported: "PROVEN", reason: `PROVEN with no after-evidence: ${after.reason}` };
519
+ } else if (verdict === "PROVEN" && passed === 0) {
520
+ // A proof that verified no criterion is not a proof: an empty list, or
521
+ // nothing but SKIPs, is what a reproducer with no way to check says.
522
+ verdict = "INCONCLUSIVE";
523
+ verdictDowngrade = {
524
+ reported: "PROVEN",
525
+ reason: `PROVEN with no criterion verified: ${criteriaResults.length} criteriaResults row(s), none PASS`,
526
+ };
527
+ }
528
+
529
+ const afterEvidence: AfterEvidence = after.ok ? after.value : {
530
+ type: "none", path: null, capturedAt: ctx.capturedAt, environment: "local",
531
+ description: `no after-evidence: ${after.reason}`,
532
+ };
533
+
534
+ return {
535
+ issueNumber: ctx.issueNumber,
536
+ verdict,
537
+ ...(verdictDowngrade ? { verdictDowngrade } : {}),
538
+ commitSHA: ctx.commitSHA,
539
+ capturedAt: ctx.capturedAt,
540
+ beforeEvidence: beforeEvidenceFor(ctx.beforeReading),
541
+ afterEvidence,
542
+ comparisonSummary: comparisonSummaryFor(ctx.beforeReading, passed, criteriaResults.length),
543
+ reproduced: raw?.reproduced === true,
544
+ criteriaResults,
545
+ evidence: Array.isArray(raw?.evidence) ? raw.evidence : [],
546
+ source: ctx.source ?? "B3-prove-reproducer",
547
+ };
548
+ }
549
+
550
+ export type ProveEvidenceRecord =
551
+ | { written: true; artifact: ReturnType<typeof buildProveEvidence> }
552
+ | { written: false; reason: string };
553
+
554
+ /**
555
+ * Build, validate and write `prove-evidence.json`, and say what happened
556
+ * (#1559). Never throws.
557
+ *
558
+ * B3 has always degraded to a warning when it could not record the artifact
559
+ * (`prove.test.ts` writes its own), so a failing `git rev-parse` or a write
560
+ * into a missing directory must not crash the prove gate. Every outcome that
561
+ * is not a written file says why.
562
+ */
563
+ export function recordProveEvidence(
564
+ raw: Record<string, any>,
565
+ opts: {
566
+ issueNumber: number;
567
+ workDir: string;
568
+ beforeReading: BeforeStateReading;
569
+ readCommit: () => string;
570
+ schema?: ArtifactSchema;
571
+ source?: string;
572
+ },
573
+ ): ProveEvidenceRecord {
574
+ try {
575
+ const artifact = buildProveEvidence(raw, {
576
+ issueNumber: opts.issueNumber,
577
+ commitSHA: opts.readCommit(),
578
+ capturedAt: new Date().toISOString(),
579
+ beforeReading: opts.beforeReading,
580
+ source: opts.source,
581
+ });
582
+ const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
583
+ if (violations.length > 0) {
584
+ return { written: false, reason: `it fails ProveEvidenceSchema (SC-2770): ${violations.join("; ")}` };
585
+ }
586
+ writeFileSync(join(opts.workDir, "prove-evidence.json"), JSON.stringify(artifact, null, 2));
587
+ return { written: true, artifact };
588
+ } catch (e: any) {
589
+ return { written: false, reason: `could not record it: ${String(e?.message ?? e).slice(0, 200)}` };
590
+ }
591
+ }
592
+
429
593
  /** What a gate run does about the tests that existed when it last looked. */
430
594
  export type CensusVerdict =
431
595
  | { kind: "unmeasured"; reason: string }
@@ -1554,26 +1718,33 @@ async function runProveReproducer(state: Record<string, any>, issue: number, iss
1554
1718
  }
1555
1719
 
1556
1720
  const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
1721
+ let parsed: Record<string, any> | null = null;
1557
1722
  if (jsonMatch) {
1558
1723
  try {
1559
- const parsed = JSON.parse(jsonMatch[0]);
1560
- const proveEvidence = {
1561
- issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
1562
- commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
1563
- capturedAt: new Date().toISOString(),
1564
- criteriaResults: parsed.criteriaResults || [],
1565
- reproduced: parsed.reproduced ?? false,
1566
- evidence: parsed.evidence || [],
1567
- source: "B3-prove-reproducer",
1568
- };
1569
- writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
1570
- console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
1724
+ parsed = JSON.parse(jsonMatch[0]);
1571
1725
  } catch {
1572
1726
  console.warn("WARN: B3 reproducer JSON parse failed");
1573
1727
  }
1574
1728
  } else {
1575
1729
  console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
1576
1730
  }
1731
+ if (parsed && typeof parsed === "object") {
1732
+ // #1559: the artifact is built in one place and checked against its one
1733
+ // contract before it is written. An artifact the schema refuses is not
1734
+ // written, and the fields are named; nothing here can crash the gate.
1735
+ const record = recordProveEvidence(parsed, {
1736
+ issueNumber: issue, workDir, beforeReading,
1737
+ readCommit: () => execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
1738
+ });
1739
+ if (!record.written) {
1740
+ console.warn(`WARN: B3 prove-evidence.json NOT written — ${record.reason}; prove.test.ts will generate evidence`);
1741
+ } else {
1742
+ console.log(`B3: prove-evidence.json written — verdict: ${record.artifact.verdict}`);
1743
+ if (record.artifact.verdictDowngrade) {
1744
+ console.warn(`WARN: B3 reproducer reported ${JSON.stringify(record.artifact.verdictDowngrade.reported)}, recorded INCONCLUSIVE — ${record.artifact.verdictDowngrade.reason}`);
1745
+ }
1746
+ }
1747
+ }
1577
1748
  }
1578
1749
  }
1579
1750
 
@@ -2318,7 +2489,7 @@ function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, ha
2318
2489
  writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
2319
2490
  const { spawn } = require("child_process");
2320
2491
  const child = spawn("sh", ["-c",
2321
- `claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format json --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
2492
+ `claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format text --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
2322
2493
  ], { detached: true, stdio: "ignore", cwd: harnessRoot });
2323
2494
  child.unref();
2324
2495
  console.log("B1: AC Adversary spawned in background — verify gate will check result");
@@ -2424,40 +2595,138 @@ function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, resu
2424
2595
  results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
2425
2596
  console.log("B1: adversary report not yet available — verify FAIL");
2426
2597
  } else {
2598
+ let raw: string | null = null;
2427
2599
  try {
2428
- const raw = readFileSync(reportPath, "utf-8");
2429
- const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2430
- if (jsonMatch) {
2431
- const report = JSON.parse(jsonMatch[0]);
2432
- console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
2433
- if (report.gameable > 0) {
2434
- let outcomeCount = 0;
2435
- for (const exploit of report.exploits || []) {
2436
- const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
2437
- const isOutcome = ac?.type === "OUTCOME";
2438
- if (isOutcome) {
2439
- addedFails++;
2440
- outcomeCount++;
2441
- results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
2442
- } else {
2443
- addedWarns++;
2444
- results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2445
- }
2446
- }
2447
- if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
2448
- const codeCount = (report.gameable || 0) - outcomeCount;
2449
- if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
2450
- }
2451
- } else {
2452
- console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
2453
- }
2600
+ raw = readFileSync(reportPath, "utf-8");
2454
2601
  } catch (e: any) {
2455
- console.warn(`WARN: B1 report read failed: ${e.message?.slice(0, 200)}`);
2602
+ addedFails++;
2603
+ results.push({ check: "B1-adversary: report unreadable", result: "FAIL" as const, detail: `adversary-report.json could not be read: ${e.message?.slice(0, 200)}` });
2604
+ }
2605
+ if (raw !== null) {
2606
+ const reading = adversaryReportReading(raw, state.acs || []);
2607
+ addedFails += reading.fails;
2608
+ addedWarns += reading.warns;
2609
+ results.push(...reading.results);
2610
+ console.log(`B1: adversary report — ${reading.fails} FAIL, ${reading.warns} WARN`);
2456
2611
  }
2457
2612
  }
2458
2613
  return { fails: addedFails, warns: addedWarns };
2459
2614
  }
2460
2615
 
2616
+ /**
2617
+ * Read B1's adversary report into gate results (#2259, SC-2770).
2618
+ *
2619
+ * The report is raw output from the model B1 spawns. It was read with a
2620
+ * regex and `JSON.parse` and never checked against `AdversaryReportSchema`,
2621
+ * so `"gameable": "2"` with no exploits, or `exploits` that was not a list,
2622
+ * produced no result at all: the reading of a clean report. A report with no
2623
+ * JSON only printed a warning, so the gate passed on a report it never read.
2624
+ * Now an unreadable report and a report the schema refuses are each one FAIL
2625
+ * naming why, the same as a missing report. Exported so it runs on fixtures;
2626
+ * the schema is a parameter so a test can hand it a stricter copy.
2627
+ */
2628
+ export function adversaryReportReading(
2629
+ raw: string,
2630
+ acs: ReadonlyArray<{ id?: unknown; type?: unknown }>,
2631
+ schema: ArtifactSchema = AdversaryReportSchema,
2632
+ ): { fails: number; warns: number; results: GateResult[] } {
2633
+ const parsed = parseAdversaryReport(raw, schema);
2634
+ if (parsed.status !== "read") {
2635
+ return { fails: 1, warns: 0, results: [{ check: `B1-adversary: ${parsed.check}`, result: "FAIL" as const, detail: parsed.detail }] };
2636
+ }
2637
+ let fails = 0;
2638
+ let warns = 0;
2639
+ const results: GateResult[] = [];
2640
+ // Every listed exploit is read whatever `gameable` says: one dropped because
2641
+ // the count was 0 is the same silent pass in the other direction.
2642
+ for (const exploit of parsed.report.exploits) {
2643
+ const ac = acs.find(a => a.id === exploit.acId);
2644
+ if (ac?.type === "OUTCOME") {
2645
+ fails++;
2646
+ results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
2647
+ } else {
2648
+ warns++;
2649
+ results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2650
+ }
2651
+ }
2652
+ return { fails, warns, results };
2653
+ }
2654
+
2655
+ type AdversaryReport = {
2656
+ gameable: number;
2657
+ approved: boolean;
2658
+ exploits: Array<{ acId: string; exploit: string; recommendation: string }>;
2659
+ };
2660
+
2661
+ /**
2662
+ * THE ONE READ of a B1 adversary report (#2259), for the report on disk and
2663
+ * for the live re-run at verify alike. Both used to spell their own regex and
2664
+ * `JSON.parse` with no schema, and review round 1 found the second copy still
2665
+ * swallowing a malformed reply after the first was fixed. Two readers of one
2666
+ * shape are one guard with a dead twin; test/adversary-report-schema.test.ts
2667
+ * asserts the extraction appears once in this file.
2668
+ *
2669
+ * Not "read" when the output holds no JSON naming `gameable`, when that JSON
2670
+ * does not parse, when the schema refuses it, or when it claims more gameable
2671
+ * criteria than it lists exploits for (`{gameable: 3, exploits: []}` is
2672
+ * schema-valid and used to read as clean). More exploits than the count is
2673
+ * not a contradiction: two ways to game one criterion.
2674
+ */
2675
+ export function parseAdversaryReport(
2676
+ raw: string,
2677
+ schema: ArtifactSchema = AdversaryReportSchema,
2678
+ ): { status: "read"; report: AdversaryReport } | { status: "not-read"; check: string; detail: string } {
2679
+ const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2680
+ if (!jsonMatch) {
2681
+ return { status: "not-read", check: "report unreadable", detail: "the output holds no JSON object naming \"gameable\"" };
2682
+ }
2683
+ let report: any;
2684
+ try {
2685
+ report = JSON.parse(jsonMatch[0]);
2686
+ } catch (e: any) {
2687
+ return { status: "not-read", check: "report unreadable", detail: `the report's JSON does not parse: ${e.message?.slice(0, 200)}` };
2688
+ }
2689
+ const violations = validateArtifactSchema(report, schema);
2690
+ if (violations.length > 0) {
2691
+ return { status: "not-read", check: "report fails AdversaryReportSchema", detail: violations.join("; ") };
2692
+ }
2693
+ if (report.gameable > report.exploits.length) {
2694
+ return {
2695
+ status: "not-read",
2696
+ check: "report contradicts itself",
2697
+ detail: `gameable ${report.gameable} but ${report.exploits.length} exploit(s) listed, so which criteria are gameable is unknown`,
2698
+ };
2699
+ }
2700
+ return { status: "read", report };
2701
+ }
2702
+
2703
+ /**
2704
+ * The live B1 re-run at verify, read (#2259 review round 1). Advisory, as it
2705
+ * always was: every finding is a WARN. What changes is that a reply that could
2706
+ * not be read is a WARN naming why, and `agentResult` records it as
2707
+ * UNMEASURED, where it used to be `null` and the verify gate's record of the
2708
+ * adversary was never written at all.
2709
+ */
2710
+ export function liveAdversaryReading(
2711
+ output: string,
2712
+ ts: string,
2713
+ ): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
2714
+ const parsed = parseAdversaryReport(output);
2715
+ if (parsed.status !== "read") {
2716
+ return {
2717
+ agentResult: { ts, status: "UNMEASURED", reason: `${parsed.check}: ${parsed.detail}`, source: "B1-adversary-verify-agent" },
2718
+ warns: 1,
2719
+ results: [{ check: `B1-adversary-verify: ${parsed.check}`, result: "WARN" as const, detail: parsed.detail }],
2720
+ };
2721
+ }
2722
+ const { report } = parsed;
2723
+ return {
2724
+ agentResult: { ts, gameable: report.gameable, approved: report.approved, exploits: report.exploits, source: "B1-adversary-verify-agent" },
2725
+ warns: report.exploits.length,
2726
+ results: report.exploits.map(exploit => ({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit })),
2727
+ };
2728
+ }
2729
+
2461
2730
  function runB2EvidenceValidation(state: Record<string, any>): void {
2462
2731
  const tier = state.sizing?.ceremonyTier || "STANDARD";
2463
2732
  if (tier === "LIGHT") return;
@@ -2677,32 +2946,11 @@ function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, res
2677
2946
  console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
2678
2947
  }
2679
2948
 
2680
- const b1Json = b1Output.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2681
- if (b1Json) {
2682
- try {
2683
- const b1Report = JSON.parse(b1Json[0]);
2684
- agentResult = {
2685
- ts: new Date().toISOString(),
2686
- gameable: b1Report.gameable || 0,
2687
- approved: b1Report.approved ?? true,
2688
- exploits: b1Report.exploits || [],
2689
- source: "B1-adversary-verify-agent",
2690
- };
2691
- if (b1Report.gameable > 0) {
2692
- for (const exploit of b1Report.exploits || []) {
2693
- addedWarns++;
2694
- results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2695
- }
2696
- console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
2697
- } else {
2698
- console.log(`B1: All ACs approved at verify — no exploits found`);
2699
- }
2700
- } catch {
2701
- console.warn("WARN: B1 AC Adversary JSON parse failed");
2702
- }
2703
- } else {
2704
- console.warn("WARN: B1 AC Adversary produced no parseable JSON");
2705
- }
2949
+ const live = liveAdversaryReading(b1Output, new Date().toISOString());
2950
+ agentResult = live.agentResult;
2951
+ addedWarns += live.warns;
2952
+ results.push(...live.results);
2953
+ console.log(`B1: verify agent — ${live.warns} WARN${live.agentResult.status === "UNMEASURED" ? ` (UNMEASURED: ${live.agentResult.reason})` : ""}`);
2706
2954
 
2707
2955
  return { agentResult, warns: addedWarns };
2708
2956
  }
@@ -3198,7 +3446,7 @@ export async function executeGate(input: GateExecutorInput): Promise<GateExecuto
3198
3446
  const hash = generateHmac(sf, slug, issue, state.projectRoot);
3199
3447
  console.log(`HMAC: ${hash}`);
3200
3448
  const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
3201
- console.log(`Ship evidence: ${evidencePath}`);
3449
+ console.log(evidencePath ? `Ship evidence: ${evidencePath}` : "Ship evidence: NOT written — see the fields named above (#2255)");
3202
3450
  }
3203
3451
 
3204
3452
  // Verify-specific: convergence
@@ -1,7 +1,10 @@
1
1
  import { test, expect, describe, beforeEach, afterEach } from "bun:test";
2
2
  import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync, statSync } from "fs";
3
3
  import { join } from "path";
4
+ import { z } from "zod";
4
5
  import { gateSaltPath } from "../lib/paths";
6
+ import { validateArtifactSchema } from "../lib/artifact-schema";
7
+ import { ShipEvidenceSchema } from "./schema";
5
8
  import {
6
9
  writeGateResult,
7
10
  generateHmac,
@@ -419,8 +422,10 @@ describe("generateShipEvidence", () => {
419
422
 
420
423
  const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
421
424
 
422
- expect(existsSync(path)).toBe(true);
423
- const evidence = JSON.parse(readFileSync(path, "utf-8"));
425
+ // null means the contract refused the artifact (#2255).
426
+ expect(path).not.toBeNull();
427
+ expect(existsSync(path!)).toBe(true);
428
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
424
429
  expect(evidence.contractVersion).toBe("1.0");
425
430
  expect(evidence.issueNumber).toBe(432);
426
431
  expect(evidence.gateOut.status).toBe("PASS");
@@ -429,6 +434,107 @@ describe("generateShipEvidence", () => {
429
434
  expect(evidence.capturedAt).toBeTruthy();
430
435
  expect(evidence.iterationCount).toBeGreaterThanOrEqual(1);
431
436
  expect(evidence.sizing.predicted).toBe("M");
437
+ // #2255 / SC-2770: the artifact meets its declared contract, not only the
438
+ // eight fields this case happens to name.
439
+ expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
440
+ });
441
+
442
+ test("#2255 / SC-2770: a schema with one required field more than the writer emits refuses the artifact", () => {
443
+ writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP" }), null, 2));
444
+ const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
445
+ expect(path).not.toBeNull();
446
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
447
+ const stricter = ShipEvidenceSchema.extend({ reviewedBy: z.string() });
448
+ expect(validateArtifactSchema(evidence, stricter).join("\n")).toContain("reviewedBy");
449
+ });
450
+
451
+ test("#2255: an artifact the contract refuses is not written, and the field is named", () => {
452
+ // A verdict outside PASS|FAIL|SKIP is the live way the writer can emit
453
+ // what ShipEvidenceSchema refuses: `result: ac.verdict || "SKIP"`.
454
+ writeFileSync(SF, JSON.stringify(makeState({
455
+ phase: "SHIP",
456
+ acs: [{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }],
457
+ }), null, 2));
458
+ const errors: string[] = [];
459
+ const original = console.error;
460
+ console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); };
461
+ let path: string | null;
462
+ try {
463
+ path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
464
+ } finally {
465
+ console.error = original;
466
+ }
467
+ expect(path).toBeNull();
468
+ expect(existsSync(join(TMP_DIR, "ship-evidence.json"))).toBe(false);
469
+ expect(errors.join("\n")).toContain("gateOut.evidence.0.result");
470
+ });
471
+
472
+ test("#2255 review round 1: an OUTCOME criterion still PENDING at ship is written — a legitimate ship-time state", () => {
473
+ // gates/workflow.test.ts: "OUTCOME ACs may be PENDING at ship time — they
474
+ // require Quinn/prove for verification", and enrichment writes the literal
475
+ // "PENDING". The contract refused it, so validation would have dropped the
476
+ // evidence on every normal run with an OUTCOME criterion outstanding.
477
+ writeFileSync(SF, JSON.stringify(makeState({
478
+ phase: "SHIP",
479
+ acs: [{ id: "SC-1", statement: "s", type: "OUTCOME", verdict: "PENDING", evidence: null }],
480
+ }), null, 2));
481
+ const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
482
+ expect(path).not.toBeNull();
483
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
484
+ expect(evidence.gateOut.evidence[0].result).toBe("PENDING");
485
+ expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
486
+ });
487
+ });
488
+
489
+ describe("#2255 review round 1: a refused artifact is never reported to the issue as shipped", () => {
490
+ // A loopback stub stands in for the GitHub API (lib/github.ts allows exactly
491
+ // GitHub or loopback). `fixture/repo` makes verifiedPush refuse rather than
492
+ // push from this checkout (#430). The control case is what keeps the
493
+ // refusal's "no requests" from being vacuous: the same stub, a valid
494
+ // artifact, and the label and comment arrive.
495
+ async function requestsDuring(acs: Array<Record<string, unknown>>): Promise<string[]> {
496
+ const seen: string[] = [];
497
+ const server = Bun.serve({
498
+ port: 0,
499
+ hostname: "127.0.0.1",
500
+ fetch(req) {
501
+ seen.push(`${req.method} ${new URL(req.url).pathname}`);
502
+ return Response.json({});
503
+ },
504
+ });
505
+ try {
506
+ writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP", acs }), null, 2));
507
+ // A child process, so the credential is SUPPLIED rather than read: the
508
+ // tree resolves it in one place (resolveGitHubToken), and nothing here
509
+ // names it as a read. Async, because this process serves the stub, and
510
+ // a blocking spawn would stall the server the child is calling. The
511
+ // child's pending requests keep it alive until they have landed.
512
+ const child = Bun.spawn(
513
+ ["bun", "-e", `import { generateShipEvidence } from ${JSON.stringify(join(import.meta.dir, "orchestrator.ts"))};` +
514
+ `generateShipEvidence(${JSON.stringify(SF)}, ${JSON.stringify(TMP_DIR)}, 432, "fixture/repo", 1, 0, 0);`],
515
+ {
516
+ cwd: join(import.meta.dir, ".."),
517
+ env: { ...process.env, GH_TOKEN: "test-token-not-real", GITHUB_API_URL: `http://127.0.0.1:${server.port}` },
518
+ stdout: "ignore",
519
+ stderr: "ignore",
520
+ },
521
+ );
522
+ await child.exited;
523
+ } finally {
524
+ server.stop(true);
525
+ }
526
+ return seen;
527
+ }
528
+
529
+ test("control: a valid artifact labels the issue shipped and posts the comment", async () => {
530
+ const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "PASS", evidence: { content: "ok" } }]);
531
+ expect(seen.some(r => r.endsWith("/issues/432/labels"))).toBe(true);
532
+ expect(seen.some(r => r.endsWith("/issues/432/comments"))).toBe(true);
533
+ });
534
+
535
+ test("a refused artifact sends nothing: no shipped label, no Ship Complete comment", async () => {
536
+ const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }]);
537
+ expect(seen).toEqual([]);
432
538
  });
433
539
  });
434
540
 
@@ -2,7 +2,8 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync, statSync
2
2
  import { join, dirname } from "path";
3
3
  import { createHash, createHmac, randomBytes } from "crypto";
4
4
  import { execSync } from "child_process";
5
- import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS } from "./schema";
5
+ import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS, ShipEvidenceSchema } from "./schema";
6
+ import { validateArtifactSchema } from "../lib/artifact-schema";
6
7
  import { ZodError } from "zod";
7
8
  import { createGitHubClient, addLabels, addComment, type GitHubClient } from "../lib/github";
8
9
  import { computeACHash } from "../lib/workflow-security";
@@ -384,7 +385,7 @@ export function generateShipEvidence(
384
385
  passes: number,
385
386
  fails: number,
386
387
  warns: number,
387
- ): string {
388
+ ): string | null {
388
389
  const state = readState(sf);
389
390
  const sha = gitSha();
390
391
  const ts = isoNow();
@@ -483,12 +484,31 @@ export function generateShipEvidence(
483
484
  ...(pushOutcome ? { push: pushOutcome, pushLeftWorkLocal: leftWorkLocal(pushOutcome) } : {}),
484
485
  };
485
486
 
487
+ // #2255 / SC-2770: checked against its one contract before it is written,
488
+ // as prove-evidence.json is (#1559). An artifact the schema refuses is not
489
+ // written, and the fields are named. The gate's verdict is untouched: by
490
+ // here the ship gate has already passed and the push has happened, and the
491
+ // missing artifact is the visible consequence.
486
492
  const evidencePath = join(workDir, "ship-evidence.json");
487
- writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
493
+ const violations = validateArtifactSchema(evidence, ShipEvidenceSchema);
494
+ if (violations.length > 0) {
495
+ console.error("ship gate: ship-evidence.json NOT written — it fails ShipEvidenceSchema (SC-2770):");
496
+ for (const v of violations) console.error(` - ${v}`);
497
+ } else {
498
+ writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
499
+ }
488
500
 
489
501
  // GitHub: add shipped label + post comment via Octokit (best-effort, fire-and-forget)
490
502
  // Uses POST additive label strategy (D-5) — never PUT/replace
491
- if (issueRepo && issue > 0) {
503
+ //
504
+ // Not when the contract refused the artifact (#2255): labelling the issue
505
+ // shipped and posting "Ship Complete" beside a refusal is a public success
506
+ // report the run's own record contradicts, and a label is harder to take
507
+ // back than to withhold.
508
+ if (violations.length > 0 && issueRepo && issue > 0) {
509
+ console.error(`ship gate: #${issue} NOT labelled shipped and no Ship Complete comment posted — the evidence was not recorded`);
510
+ }
511
+ if (violations.length === 0 && issueRepo && issue > 0) {
492
512
  try {
493
513
  const github = createGitHubClient();
494
514
  addLabels(github, issueRepo, issue, ["shipped"]).catch(() => {});
@@ -512,7 +532,7 @@ export function generateShipEvidence(
512
532
  } catch {}
513
533
  }
514
534
 
515
- return evidencePath;
535
+ return violations.length > 0 ? null : evidencePath;
516
536
  }
517
537
 
518
538
  const GATE_PENDING_PATH = join(
@@ -2,7 +2,7 @@
2
2
  doc-type: reference
3
3
  status: active
4
4
  owner: jason
5
- updated: 2026-09-19
5
+ updated: 2026-11-15
6
6
  ---
7
7
 
8
8
  You are a prove reproducer. Your objective: reproduce the bug described
@@ -44,9 +44,22 @@ Output JSON:
44
44
  "evidence": [
45
45
  {"type": "screenshot|api-response|console", "description": "what this shows"}
46
46
  ],
47
+ "afterEvidence": {
48
+ "type": "text|api|data",
49
+ "path": null,
50
+ "capturedAt": "ISO-8601 time you observed the after-state",
51
+ "environment": "local|prod",
52
+ "description": "what the after-state showed"
53
+ },
47
54
  "gaps": []
48
55
  }
49
56
 
57
+ `afterEvidence` is the after-state you observed yourself. `environment` is
58
+ where you observed it: `local` for this checkout or a dev server, `prod` for
59
+ the production URL. If you observed no after-state, leave `afterEvidence`
60
+ out. Never fill it in to make the record complete: a PROVEN without it is
61
+ recorded as INCONCLUSIVE.
62
+
50
63
  Verdict rules:
51
64
  - PROVEN: bug was reproduced in before-state AND fix verified in after-state
52
65
  - UNPROVEN: fix does not resolve the issue (evidence shows problem persists)
@@ -1,3 +1,3 @@
1
1
  ba56e1fcb00c7a9bc1f2b05beca701cb264785e017cd2bf54a2dd07cdde331df ac-adversary.md
2
2
  7d4ff0ef4b787e68b59643c11ba51e62a9411f5e24509b97855a337598c67508 evidence-validator.md
3
- fd6e01a8e6ec1845a39bd29b15355320238ca7dcf9131e25f5b8cb1468f635ee prove-reproducer.md
3
+ 6f15ff6891118f749894f55f18fa21234211f993cf6eadfb3c90561a64b91691 prove-reproducer.md
@@ -1,7 +1,8 @@
1
1
  import { test, expect, describe } from "bun:test";
2
2
  import { readFileSync, existsSync, writeFileSync } from "fs";
3
3
  import { execSync } from "child_process";
4
- import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState } from "./gate-executor";
4
+ import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState, validateArtifactSchema } from "./gate-executor";
5
+ import { ProveEvidenceSchema } from "./schema";
5
6
 
6
7
  const TEST_DIR = process.env.TEST_WORK_DIR || "";
7
8
  const PROJECT_ROOT = process.env.PROJECT_ROOT || process.cwd();
@@ -145,6 +146,10 @@ describe("prove: run evidence commands from workflow-state", () => {
145
146
  criteriaResults,
146
147
  };
147
148
 
149
+ // #2254 / SC-2770: checked against the one contract before it is written,
150
+ // as B3's and prove.js's are. A required field this writer stops emitting
151
+ // fails here, with the field named, instead of in whatever reads it next.
152
+ expect(validateArtifactSchema(evidence, ProveEvidenceSchema)).toEqual([]);
148
153
  writeFileSync(`${TEST_DIR}/prove-evidence.json`, JSON.stringify(evidence, null, 2));
149
154
  expect(verdict).toBe("PROVEN");
150
155
  });
package/gates/schema.ts CHANGED
@@ -1193,7 +1193,10 @@ export const ShipEvidenceSchema = z.object({
1193
1193
  status: z.enum(["PASS", "FAIL"]),
1194
1194
  evidence: z.array(z.object({
1195
1195
  criterion: z.string(),
1196
- result: z.enum(["PASS", "FAIL", "SKIP"]),
1196
+ // PENDING is a legitimate ship-time state (#2255): an OUTCOME criterion
1197
+ // waits for Quinn or /prove, and the verify gate passes it through
1198
+ // (checkVerifyPendingACs). The writer copies the criterion's verdict.
1199
+ result: z.enum(["PASS", "FAIL", "SKIP", "PENDING"]),
1197
1200
  detail: z.any().optional(),
1198
1201
  })),
1199
1202
  }),
@@ -1252,9 +1255,26 @@ const BeforeEvidenceSchema = z.union([
1252
1255
  }),
1253
1256
  ]);
1254
1257
 
1258
+ /**
1259
+ * One row of `prove-evidence.json`'s `criteriaResults` (#1559). The shape all
1260
+ * three writers already emit — B3's reproducer, `gates/prove.test.ts` and
1261
+ * `workflows/prove.js` — and the one the deleted
1262
+ * `config/prove-evidence.schema.json` declared. SKIP is the reproducer's
1263
+ * verdict for a UI criterion it has no browser to check.
1264
+ */
1265
+ const CriterionResultSchema = z.object({
1266
+ scId: z.string(),
1267
+ verdict: z.enum(["PASS", "FAIL", "SKIP"]),
1268
+ evidence: z.string(),
1269
+ });
1270
+
1255
1271
  export const ProveEvidenceSchema = z.object({
1256
1272
  issueNumber: z.number(),
1257
1273
  verdict: z.enum(["PROVEN", "UNPROVEN", "INCONCLUSIVE"]),
1274
+ // Required, but not `.min(1)`: an INCONCLUSIVE with no criteria is a record
1275
+ // `gates/prove.test.ts` legitimately writes. That a PROVEN needs a PASS row
1276
+ // is `buildProveEvidence`'s rule, because it is about two fields at once.
1277
+ criteriaResults: z.array(CriterionResultSchema),
1258
1278
  commitSHA: z.string(),
1259
1279
  capturedAt: z.string(),
1260
1280
  beforeEvidence: BeforeEvidenceSchema,
@@ -453,6 +453,18 @@ export function verifyFanoutViolations(label: string, val: unknown, tier: unknow
453
453
  `is an absence with extra steps (#126)`,
454
454
  ];
455
455
  }
456
+ // #2259: the agent ran and its reply could not be read. Before, that reply
457
+ // left no record, which the absence branch above refused, so the ship
458
+ // outcome is unchanged. What changes is that the refusal names the cause
459
+ // instead of failing on a missing `gameable` further down.
460
+ if ((val as { status?: unknown }).status === "UNMEASURED") {
461
+ const reason = (val as { reason?: unknown }).reason;
462
+ return [
463
+ `gates.verify.${label} is UNMEASURED — the agent ran and its reply could not be read` +
464
+ (typeof reason === "string" && reason.trim() !== "" ? `: ${reason}` : "") +
465
+ `. An unread reply is not an approval (#126, #2259).`,
466
+ ];
467
+ }
456
468
  return [];
457
469
  }
458
470
 
@@ -224,8 +224,19 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
224
224
  const a = anchorOnlyKey(k);
225
225
  if (a !== null) anchors.add(a);
226
226
  }
227
+ // #2257: how many findings cite a line, and how many of those lines could
228
+ // be read. Read from the SAME calls the keys are built from, not a second
229
+ // pass, so the count cannot disagree with what the keys saw.
230
+ let located = 0;
231
+ let readable = 0;
227
232
  const repeat = candidates.map(c => {
228
- const key = carryKey(c, inputs.lineTextFor(c as ReviewFinding));
233
+ const cited = c as ReviewFinding;
234
+ const lineText = inputs.lineTextFor(cited);
235
+ if (typeof cited?.file === "string" && Number.isInteger(cited?.line) && (cited.line as number) >= 1) {
236
+ located++;
237
+ if (lineText !== undefined) readable++;
238
+ }
239
+ const key = carryKey(c, lineText);
229
240
  // Unkeyable CARRIES. It cannot be shown to be a repeat, and the
230
241
  // direction this is allowed to be wrong in is toward reporting.
231
242
  if (key === null) return null;
@@ -282,7 +293,20 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
282
293
  // this whole increment is a correction for.
283
294
  return null;
284
295
  });
285
- return { repeat, unmeasured: null };
296
+ // #2257. A gate that could read no cited line built every key from a line
297
+ // NUMBER, which never equals a key an earlier round recorded from line
298
+ // TEXT. "Nothing to suppress" from that gate is a reading of nothing, and
299
+ // it printed the same words as a gate that read every line (#1872: 0 of
300
+ // 420 in CI). Byte-key matches above still stand; this says the anchor
301
+ // half was blind.
302
+ const unmeasured =
303
+ located > 0 && readable === 0 && prior.keys.length > 0
304
+ ? `no cited line could be read for any of this round's ${located} finding(s) that cite one, ` +
305
+ `so no key could match one an earlier round recorded from the line's text, and a ` +
306
+ `restated finding could not be recognised. The checkout the gate reads is probably not ` +
307
+ `the one under review (#2257).`
308
+ : null;
309
+ return { repeat, unmeasured };
286
310
  };
287
311
  }
288
312
 
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Check a gate artifact against its declared schema, before it is written
3
+ * (#1559, #2255, SC-2770).
4
+ *
5
+ * Lives in `lib/` rather than beside its first caller in
6
+ * `gates/gate-executor.ts` because `gates/orchestrator.ts` needs it too, and
7
+ * gate-executor already imports orchestrator. `gates/gate-executor.ts`
8
+ * re-exports both names, so there is one definition and every existing import
9
+ * keeps working.
10
+ */
11
+
12
+ /**
13
+ * Anything with zod's `safeParse` shape. Structural, so callers keep no
14
+ * direct zod import and a test can hand it a stricter copy of a schema.
15
+ */
16
+ export interface ArtifactSchema {
17
+ safeParse(value: unknown): {
18
+ success: boolean;
19
+ error?: { issues: ReadonlyArray<{ path: ReadonlyArray<PropertyKey>; message: string }> };
20
+ };
21
+ }
22
+
23
+ /**
24
+ * One `path: message` line per violation, `[]` when it validates.
25
+ *
26
+ * `config/prove-evidence.schema.json` declared a contract nothing read, so
27
+ * writer and contract drifted apart: B3's artifact failed both the JSON
28
+ * schema and the zod one, and no check said so. A refusal that names no
29
+ * issue is still a refusal — an empty list there would read as a pass.
30
+ */
31
+ export function validateArtifactSchema(artifact: unknown, schema: ArtifactSchema): string[] {
32
+ const result = schema.safeParse(artifact);
33
+ if (result.success) return [];
34
+ const issues = result.error?.issues ?? [];
35
+ if (issues.length === 0) return ["(root): the schema refused the artifact without naming a field"];
36
+ return issues.map((i) => `${i.path.length > 0 ? i.path.map(String).join(".") : "(root)"}: ${i.message}`);
37
+ }
@@ -1822,6 +1822,20 @@ export const AI_REVIEW_LEDGER_SNAPSHOT_BASENAME = "ai-review-ledger-snapshot.txt
1822
1822
  */
1823
1823
  export const AI_REVIEW_THREADS_PATH_VAR = "AI_REVIEW_THREADS_PATH";
1824
1824
 
1825
+ /**
1826
+ * The checkout the adjudicator reads cited lines from (#2257).
1827
+ *
1828
+ * The adjudicator runs from the payload's own directory, outside the
1829
+ * checkout, because `bunx` resolves bunfig.toml, .npmrc and package.json by
1830
+ * walking up from its working directory, so the PR under review must not be
1831
+ * the tree it starts in. That made `process.cwd()` the payload directory,
1832
+ * the suppression gate could read no cited line, and the anchor rule matched
1833
+ * 0 of 420 round-2+ findings in CI (#1872). The root has to arrive by name.
1834
+ * `scripts/ai-review-findings.ts` falls back to its working directory when
1835
+ * this is unset, which is what a step generated before it was set does.
1836
+ */
1837
+ export const AI_REVIEW_CHECKOUT_ROOT_VAR = "AI_REVIEW_CHECKOUT_ROOT";
1838
+
1825
1839
  /** The basename inside the findings directory. One spelling. */
1826
1840
  export const AI_REVIEW_THREADS_BASENAME = "ai-review-threads.json";
1827
1841
 
@@ -0,0 +1,120 @@
1
+ /**
2
+ * `.github/workflows/intake.yml`, generated — #2233.
3
+ *
4
+ * Jason decided on #1126 (2026-11-15) that each intake source runs on its own
5
+ * trigger, not on scaffold: "scaffold" reads as a one-time setup step, so
6
+ * nobody re-runs it to re-mine, and intake should run when its data appears.
7
+ * For `fix-commits` the data is the history of the default branch, and the
8
+ * trigger is a push to it.
9
+ *
10
+ * ## Full history, or the reading is of the clone
11
+ *
12
+ * `mineFixCommits` reports `SHALLOW_HISTORY` on a shallow clone, and
13
+ * `actions/checkout` clones to depth 1 unless told otherwise. That depth is
14
+ * the only reason the source read UNMEASURED on a runner, so the checkout
15
+ * sets `fetch-depth: 0`.
16
+ *
17
+ * ## Advisory, and never silent
18
+ *
19
+ * `rungate intake` exits 0 on every reading (#2248), so a reading never turns
20
+ * the run red: an unreadable source says nothing about the push that fired
21
+ * it. UNMEASURED is printed as the command prints it and raised as a warning
22
+ * annotation, so it is visible on the run without being mistaken for zero.
23
+ * A non-zero exit is the command refusing its arguments or `bunx` failing,
24
+ * which is the tool broken, and the step goes red on it.
25
+ *
26
+ * ## It calls the published CLI
27
+ *
28
+ * A consumer has `node_modules/rungate`, not `scripts/`, so the step runs
29
+ * `bunx rungate@<version> intake`, at the generating version like every other
30
+ * pin. `intake` first shipped in 0.8.23 (#2250).
31
+ */
32
+
33
+ /** The file this module generates, under `.github/workflows/`. */
34
+ export const INTAKE_WORKFLOW_FILE = "intake.yml";
35
+
36
+ /** The one job in it. */
37
+ export const INTAKE_JOB = "fix-commit-intake";
38
+
39
+ /**
40
+ * The CLI command the step runs. `lib/cli-commands.ts` must register it, or
41
+ * the workflow calls a command `bin/rungate.ts` refuses (#675's defect).
42
+ */
43
+ export const INTAKE_COMMAND = "intake";
44
+
45
+ /**
46
+ * The step's `run:` body. Exported so `test/intake-workflow.test.ts` executes
47
+ * THIS text against a stub `bunx`, rather than a paraphrase.
48
+ *
49
+ * The output is captured so it can be both printed and inspected. Under
50
+ * `bash -e` a failing command substitution in an assignment stops the step,
51
+ * which is the red a broken tool should produce.
52
+ */
53
+ export function buildIntakeScript(version: string): string {
54
+ return `out=$(bunx rungate@${version} ${INTAKE_COMMAND} --source fix-commits)
55
+ printf '%s\\n' "$out"
56
+ case "$out" in
57
+ *UNMEASURED*)
58
+ echo "::warning::UNMEASURED — the fix-commits intake reading could not be taken on this run; the reason is printed above. This is not zero candidates."
59
+ ;;
60
+ esac
61
+ `;
62
+ }
63
+
64
+ export interface IntakeWorkflowOptions {
65
+ /** The `runs-on:` value, already JSON-quoted by the caller. */
66
+ runsOn: string;
67
+ /** The pinned `actions/checkout` reference. */
68
+ checkout: string;
69
+ /** The pinned `oven-sh/setup-bun` reference. */
70
+ setupBun: string;
71
+ /** The `with:` body of that step, at ten spaces — the project's single bun declaration (#622). */
72
+ setupBunWith: string;
73
+ /** The step's `env:` block, from `setupBunEnv` (#2039). */
74
+ setupBunEnv: string;
75
+ /** The published harness version the step runs through `bunx`. */
76
+ version: string;
77
+ }
78
+
79
+ /** The whole `intake.yml` body. */
80
+ export function buildIntakeWorkflow(opts: IntakeWorkflowOptions): string {
81
+ const indent = (text: string, pad: string) =>
82
+ text
83
+ .trimEnd()
84
+ .split("\n")
85
+ .map(line => (line === "" ? "" : `${pad}${line}`))
86
+ .join("\n");
87
+
88
+ return `# Managed by rungate — do not edit. Customize the ci section of .claude/rungate/config.json.
89
+ #
90
+ # Mines the default branch's fix commits into rule candidates on every push to
91
+ # it (#2233). Jason's #1126 decision: each intake source runs on its own
92
+ # trigger, never on scaffold. Advisory: a reading never fails the run, and an
93
+ # UNMEASURED one is raised as a warning rather than read as zero.
94
+ name: Intake — fix commits
95
+
96
+ on:
97
+ push:
98
+ branches: [main]
99
+
100
+ permissions:
101
+ contents: read
102
+
103
+ jobs:
104
+ ${INTAKE_JOB}:
105
+ runs-on: ${opts.runsOn}
106
+ steps:
107
+ # Full history: a depth-1 clone is the only reason this source reads
108
+ # UNMEASURED (SHALLOW_HISTORY) on a runner.
109
+ - uses: ${opts.checkout}
110
+ with:
111
+ fetch-depth: 0
112
+ - uses: ${opts.setupBun}
113
+ with:
114
+ ${opts.setupBunWith}
115
+ ${opts.setupBunEnv}
116
+ - name: Mine fix commits into rule candidates
117
+ run: |
118
+ ${indent(buildIntakeScript(opts.version), " ")}
119
+ `;
120
+ }
@@ -210,6 +210,16 @@ export const OWNED_FILES: OwnedFile[] = [
210
210
  customize: "Set `board.id` in config and secret `RUNGATE_PROJECT_TOKEN`; add your own jobs alongside the generated `add-to-board` job",
211
211
  avoid: "Don't edit the generated `add-to-board` job",
212
212
  },
213
+ {
214
+ // #2233: fix-commit intake on push to main, with full history. Jason's
215
+ // #1126 decision put each intake source on its own trigger; a consumer has
216
+ // no scripts/, so the generator writes it and calls the published CLI.
217
+ path: ".github/workflows/intake.yml",
218
+ strategy: "job-preserving",
219
+ servedBy: "writeManagedFile (lib/scaffold/steps.ts) via mergeManagedWorkflow — the harness owns named jobs, not the file",
220
+ customize: "Advisory — a reading never fails the run; add your own jobs alongside the generated `fix-commit-intake` job",
221
+ avoid: "Don't edit the generated `fix-commit-intake` job, and don't drop `fetch-depth: 0` — a shallow clone reads UNMEASURED",
222
+ },
213
223
  {
214
224
  // `regenerate` until #655, and the declaration was never consulted:
215
225
  // `generateAgentBriefsStep` called `writeFileSync` directly, so the row
@@ -50,6 +50,7 @@ import {
50
50
  } from "./ai-review-unmeasured-workflow";
51
51
  import { buildAdrAcceptedWorkflow } from "./adr-accepted-workflow";
52
52
  import { buildClosingReferencesWorkflow } from "./closing-references-workflow";
53
+ import { buildIntakeWorkflow } from "./intake-workflow";
53
54
  import { buildBoardReadyWorkflow } from "./board-ready-workflow";
54
55
  import { buildBoardIntakeWorkflow } from "./board-intake-workflow";
55
56
  import { CI_SUITE_FETCH_DEPTH, baseRefFetchDepth } from "../ci-history-depth";
@@ -4515,6 +4516,18 @@ ${buildAuthDetectStep(vertex)}${buildReviewRunStep(
4515
4516
  version: harnessVersion(),
4516
4517
  });
4517
4518
 
4519
+ // #2233: fix-commit intake on every push to main, with full history (Jason's
4520
+ // #1126 decision: each intake source on its own trigger, never on scaffold).
4521
+ // It calls the published CLI's `intake`, first shipped in 0.8.23.
4522
+ const intakeYml = buildIntakeWorkflow({
4523
+ runsOn,
4524
+ checkout: actionRef("actions/checkout"),
4525
+ setupBun: actionRef("oven-sh/setup-bun"),
4526
+ setupBunWith: bunStep,
4527
+ setupBunEnv: setupBunEnv(" "),
4528
+ version: harnessVersion(),
4529
+ });
4530
+
4518
4531
  // #2166, ADR-014 row 4: new issues and pull requests onto the board. It
4519
4532
  // names no board: the published CLI prints board.id from the default
4520
4533
  // branch's config (#2173), and the add uses that node id.
@@ -4542,7 +4555,7 @@ ${buildAuthDetectStep(vertex)}${buildReviewRunStep(
4542
4555
  // lib/conformity.ts reads back, and the two are held together by
4543
4556
  // test/ai-review-workflow.test.ts AC-1, which looks for the constant's
4544
4557
  // filename in what this loop actually wrote.
4545
- for (const [name, content] of [["ci.yml", ciYml], ["gates.yml", gatesYml], ["ai-review.yml", aiReviewYml], ["ai-review-unmeasured.yml", aiReviewUnmeasuredYml], ["adr-accepted.yml", adrAcceptedYml], ["closing-references.yml", closingReferencesYml], ["board-ready.yml", boardReadyYml], ["board-intake.yml", boardIntakeYml]] as const) {
4558
+ for (const [name, content] of [["ci.yml", ciYml], ["gates.yml", gatesYml], ["ai-review.yml", aiReviewYml], ["ai-review-unmeasured.yml", aiReviewUnmeasuredYml], ["adr-accepted.yml", adrAcceptedYml], ["closing-references.yml", closingReferencesYml], ["board-ready.yml", boardReadyYml], ["board-intake.yml", boardIntakeYml], ["intake.yml", intakeYml]] as const) {
4546
4559
  const label = `.github/workflows/${name}`;
4547
4560
  try {
4548
4561
  // Keyed by FILE, and the lookup is here because this loop is the only
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rungate",
3
- "version": "0.8.23",
3
+ "version": "0.8.24",
4
4
  "type": "module",
5
5
  "description": "Ship harness — conformity tests, scaffold, and agent briefs for AI-first development",
6
6
  "repository": {
@@ -94,7 +94,7 @@
94
94
  import { aiReviewJuryGate } from "../lib/ai-review-jury";
95
95
  import { productionSuppression } from "../lib/ai-review-suppression";
96
96
  import { main } from "../lib/scaffold/ai-review-findings";
97
- import { AI_REVIEW_BALLOT_DIR_ENV, AI_REVIEW_THREADS_PATH_VAR } from "../lib/scaffold/ai-review-step";
97
+ import { AI_REVIEW_BALLOT_DIR_ENV, AI_REVIEW_CHECKOUT_ROOT_VAR, AI_REVIEW_THREADS_PATH_VAR } from "../lib/scaffold/ai-review-step";
98
98
 
99
99
  const ballotDir = process.env[AI_REVIEW_BALLOT_DIR_ENV];
100
100
  const jury =
@@ -105,9 +105,13 @@ const jury =
105
105
  // Assembled in `lib/`, not here. This command and `ai-review-threads.ts` must
106
106
  // form the SAME gate from the same file, and two hand-written literals is how
107
107
  // `anchorOccurrences` reached one of them and not the other (#1861 round 4).
108
+ // #2257: the checkout by name. The generated step runs this from the payload's
109
+ // directory on purpose, so the working directory is not the checkout; it is
110
+ // the fallback only for a step generated before the variable was set.
111
+ const checkoutRoot = process.env[AI_REVIEW_CHECKOUT_ROOT_VAR];
108
112
  const { gate: suppress } = productionSuppression(
109
113
  process.env[AI_REVIEW_THREADS_PATH_VAR],
110
- process.cwd(),
114
+ checkoutRoot !== undefined && checkoutRoot !== "" ? checkoutRoot : process.cwd(),
111
115
  );
112
116
 
113
117
  // One line, and `process.exit(main(` is load-bearing text: AC-4 in
@@ -1,83 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "title": "Prove Evidence",
4
- "description": "Output artifact from /prove skill",
5
- "type": "object",
6
- "required": ["contractVersion", "issueNumber", "verdict", "commitSHA", "capturedAt", "criteriaResults"],
7
- "properties": {
8
- "contractVersion": { "type": "string" },
9
- "issueNumber": { "type": "integer" },
10
- "verdict": { "type": "string", "enum": ["PROVEN", "UNPROVEN", "INCONCLUSIVE"] },
11
- "commitSHA": { "type": "string" },
12
- "capturedAt": { "type": "string", "format": "date-time" },
13
- "beforeEvidence": {
14
- "$comment": "#1540: UNMEASURED declares no type. The old shape filled an absent before-state with type \"none\", which is a value /goal legitimately writes for a run it did measure, so the two were indistinguishable. Artefacts written before #1540 carry no status and remain valid.",
15
- "type": "object",
16
- "properties": {
17
- "status": { "type": "string", "enum": ["CAPTURED", "DECLARED_NONE", "UNMEASURED"] },
18
- "type": { "type": "string", "enum": ["text", "screenshot", "api", "data", "none"] },
19
- "path": { "type": ["string", "null"] },
20
- "capturedAt": { "type": ["string", "null"] },
21
- "description": { "type": "string" },
22
- "reason": { "type": "string" }
23
- },
24
- "allOf": [
25
- {
26
- "$comment": "An unmeasured reading says why, and never claims a type.",
27
- "if": { "properties": { "status": { "const": "UNMEASURED" } }, "required": ["status"] },
28
- "then": { "required": ["reason"], "not": { "required": ["type"] } }
29
- },
30
- {
31
- "$comment": "Any reading that is not UNMEASURED was taken, so it carries when.",
32
- "if": {
33
- "properties": { "status": { "enum": ["CAPTURED", "DECLARED_NONE"] } },
34
- "required": ["status"]
35
- },
36
- "then": { "required": ["type", "capturedAt"] }
37
- }
38
- ]
39
- },
40
- "afterEvidence": {
41
- "type": "object",
42
- "properties": {
43
- "type": { "type": "string", "enum": ["text", "screenshot", "api"] },
44
- "path": { "type": "string" },
45
- "capturedAt": { "type": "string", "format": "date-time" },
46
- "environment": { "type": "string", "enum": ["local", "prod"] },
47
- "description": { "type": "string" }
48
- }
49
- },
50
- "comparisonSummary": { "type": "string" },
51
- "quinnVerdict": { "type": "string", "enum": ["PASS", "FAIL", "SKIP"] },
52
- "reproduced": { "type": "boolean" },
53
- "criteriaResults": {
54
- "type": "array",
55
- "minItems": 1,
56
- "items": {
57
- "type": "object",
58
- "required": ["scId", "verdict", "evidence"],
59
- "properties": {
60
- "scId": { "type": "string" },
61
- "verdict": { "type": "string", "enum": ["PASS", "FAIL", "SKIP"] },
62
- "evidence": { "type": "string" },
63
- "evidencePath": { "type": ["string", "null"] },
64
- "exitCode": { "type": ["integer", "null"] }
65
- }
66
- }
67
- },
68
- "gaps": {
69
- "type": "array",
70
- "items": {
71
- "type": "object",
72
- "required": ["ac", "status"],
73
- "properties": {
74
- "ac": { "type": "string" },
75
- "status": { "type": "string" },
76
- "detail": { "type": "string" },
77
- "priority": { "type": ["string", "null"] },
78
- "followUpIssue": { "type": ["string", "null"] }
79
- }
80
- }
81
- }
82
- }
83
- }