rungate 0.8.23 → 0.8.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/gates/.gate-salt CHANGED
@@ -1 +1 @@
1
- e387efb0e00e0aa4e64f43d5fed7be9d6279267723195e1531aff90000f927f7
1
+ f7edfff5c8424ba64c95ad1a592d7b40c40c3a78197e968739a5d468cd671467
@@ -1,5 +1,5 @@
1
1
  import { execSync, execFileSync } from "child_process";
2
- import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync } from "fs";
2
+ import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync, statSync } from "fs";
3
3
  import { join } from "path";
4
4
  import { splitGateRuns } from "../lib/gate-run-split";
5
5
  import { gateSaltPath, workDir as slugWorkDir } from "../lib/paths";
@@ -38,7 +38,7 @@ import {
38
38
  measureEvidencePrevalidation,
39
39
  type EvidencePrevalidationReading,
40
40
  } from "../lib/evidence-prevalidator";
41
- import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
41
+ import { AdversaryReportSchema, EvidencePrevalidationSchema, ProveEvidenceSchema, type PrevalidationSource } from "./schema";
42
42
  import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
43
43
  import { deepMerge } from "../lib/deep-merge";
44
44
  import { livenessPlan, probeTarget } from "../lib/dev-liveness";
@@ -426,6 +426,205 @@ export function comparisonSummaryFor(
426
426
  return `compared against the ${reading.type} before-state captured at ${reading.capturedAt}; ${counts}`;
427
427
  }
428
428
 
429
+ // The validator moved to lib/ in #2255 so gates/orchestrator.ts can use it
430
+ // without importing this file back; re-exported so it has one definition.
431
+ import { validateArtifactSchema, type ArtifactSchema } from "../lib/artifact-schema";
432
+ export { validateArtifactSchema, type ArtifactSchema };
433
+
434
+ const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
435
+ /**
436
+ * Who observed the after-state, and the after-evidence types each can report
437
+ * (#2262). `none` is B3's to write. The reproducer has Bash and Read only
438
+ * (gates/prompts/prove-reproducer.md), so a screenshot it reports is a UI
439
+ * check it could not have made. Quinn drives a browser against the prove
440
+ * container (workflows/prove.js), so a screenshot is what Quinn observes.
441
+ *
442
+ * The CALLER names the observer, never the report: an agent writing
443
+ * `observedBy: "quinn"` into its own afterEvidence is ignored. The spawn
444
+ * schemas in workflows/prove.js and `ProveEvidenceSchema`'s `observedBy` restate
445
+ * these lists, and test/prove-workflow-evidence.test.ts holds them equal.
446
+ */
447
+ export const AFTER_OBSERVERS = {
448
+ reproducer: ["text", "api", "data"],
449
+ quinn: ["text", "api", "data", "screenshot"],
450
+ } as const;
451
+ export type AfterObserver = keyof typeof AFTER_OBSERVERS;
452
+ export const isAfterObserver = (v: unknown): v is AfterObserver =>
453
+ typeof v === "string" && Object.prototype.hasOwnProperty.call(AFTER_OBSERVERS, v);
454
+ const AFTER_ENVIRONMENTS = ["local", "prod"];
455
+
456
+ /** What B3 knows about the run that the reproducer does not get to say. */
457
+ export interface ProveEvidenceContext {
458
+ issueNumber: number;
459
+ commitSHA: string;
460
+ capturedAt: string;
461
+ beforeReading: BeforeStateReading;
462
+ /** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
463
+ source?: string;
464
+ /** Who observed the after-state. The reproducer unless the caller says otherwise (#2262). */
465
+ afterObserver?: AfterObserver;
466
+ }
467
+
468
+ type AfterEvidence = {
469
+ type: string; path: string | null; capturedAt: string; environment: string; description: string;
470
+ observedBy?: AfterObserver;
471
+ };
472
+
473
+ function isFile(path: string): boolean {
474
+ try {
475
+ return statSync(path).isFile();
476
+ } catch {
477
+ return false; // absent or unreadable: either way nobody can open it
478
+ }
479
+ }
480
+
481
+ /** The reported `afterEvidence`, or why there is none. Never repaired into shape. */
482
+ function readReportedAfterEvidence(raw: unknown, observer: AfterObserver): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
483
+ if (raw === undefined || raw === null) return { ok: false, reason: `the ${observer} reported no afterEvidence` };
484
+ if (typeof raw !== "object" || Array.isArray(raw)) {
485
+ return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
486
+ }
487
+ const a = raw as Record<string, unknown>;
488
+ const types: readonly string[] = AFTER_OBSERVERS[observer];
489
+ if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
490
+ return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
491
+ }
492
+ if (typeof a.type !== "string" || !types.includes(a.type)) {
493
+ return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one the ${observer} can observe (${types.join("|")})` };
494
+ }
495
+ if (a.type === "screenshot" && (typeof a.path !== "string" || a.path.trim() === "")) {
496
+ return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is empty — a screenshot with no file is not evidence` };
497
+ }
498
+ // Review round 1, PR #2266: a path is a claim. The file has to be there,
499
+ // or a screenshot nobody took records as proof.
500
+ if (a.type === "screenshot" && !isFile(a.path as string)) {
501
+ return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is not a file — a screenshot nobody can open is not evidence` };
502
+ }
503
+ if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
504
+ return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
505
+ }
506
+ if (typeof a.description !== "string" || a.description.trim() === "") {
507
+ return { ok: false, reason: "afterEvidence.description is empty — it must say what the after-state showed" };
508
+ }
509
+ return {
510
+ ok: true,
511
+ value: {
512
+ type: a.type, path: typeof a.path === "string" ? a.path : null,
513
+ capturedAt: a.capturedAt, environment: a.environment, description: a.description,
514
+ observedBy: observer,
515
+ },
516
+ };
517
+ }
518
+
519
+ /**
520
+ * Build `prove-evidence.json` from the reproducer's report (#1559, Jason's
521
+ * option (a) of 2026-11-15).
522
+ *
523
+ * B3 used to write `{issueNumber, verdict, commitSHA, capturedAt,
524
+ * criteriaResults, reproduced, evidence, source}`, which lacks the
525
+ * `beforeEvidence`, `afterEvidence` and `comparisonSummary` that
526
+ * `ProveEvidenceSchema` requires. The before-evidence and the comparison are
527
+ * B3's own readings (#1540), so a reproducer cannot overwrite them. The
528
+ * after-evidence is the reproducer's report of the environment it tested and
529
+ * what it saw. When that report is missing or malformed, the record is
530
+ * `type: "none"` and a PROVEN becomes INCONCLUSIVE, because a proof with no
531
+ * observed after-state is not a proof.
532
+ *
533
+ * The `none` record says `environment: "local"` because the schema requires
534
+ * a value. `local` is the one that satisfies no STANDARD+ prod requirement
535
+ * (gates/prove.test.ts), so a missing after-evidence can never pass that check.
536
+ */
537
+ export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
538
+ const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
539
+ const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
540
+ const after = readReportedAfterEvidence(raw?.afterEvidence, ctx.afterObserver ?? "reproducer");
541
+
542
+ let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
543
+ let verdictDowngrade: { reported: unknown; reason: string } | undefined;
544
+ if (verdict !== raw?.verdict) {
545
+ verdictDowngrade = {
546
+ reported: raw?.verdict,
547
+ reason: `verdict ${JSON.stringify(raw?.verdict)} is not one of ${PROVE_VERDICTS.join("|")}`,
548
+ };
549
+ } else if (verdict === "PROVEN" && !after.ok) {
550
+ verdict = "INCONCLUSIVE";
551
+ verdictDowngrade = { reported: "PROVEN", reason: `PROVEN with no after-evidence: ${after.reason}` };
552
+ } else if (verdict === "PROVEN" && passed === 0) {
553
+ // A proof that verified no criterion is not a proof: an empty list, or
554
+ // nothing but SKIPs, is what a reproducer with no way to check says.
555
+ verdict = "INCONCLUSIVE";
556
+ verdictDowngrade = {
557
+ reported: "PROVEN",
558
+ reason: `PROVEN with no criterion verified: ${criteriaResults.length} criteriaResults row(s), none PASS`,
559
+ };
560
+ }
561
+
562
+ const afterEvidence: AfterEvidence = after.ok ? after.value : {
563
+ type: "none", path: null, capturedAt: ctx.capturedAt, environment: "local",
564
+ description: `no after-evidence: ${after.reason}`,
565
+ };
566
+
567
+ return {
568
+ issueNumber: ctx.issueNumber,
569
+ verdict,
570
+ ...(verdictDowngrade ? { verdictDowngrade } : {}),
571
+ commitSHA: ctx.commitSHA,
572
+ capturedAt: ctx.capturedAt,
573
+ beforeEvidence: beforeEvidenceFor(ctx.beforeReading),
574
+ afterEvidence,
575
+ comparisonSummary: comparisonSummaryFor(ctx.beforeReading, passed, criteriaResults.length),
576
+ reproduced: raw?.reproduced === true,
577
+ criteriaResults,
578
+ evidence: Array.isArray(raw?.evidence) ? raw.evidence : [],
579
+ source: ctx.source ?? "B3-prove-reproducer",
580
+ };
581
+ }
582
+
583
+ export type ProveEvidenceRecord =
584
+ | { written: true; artifact: ReturnType<typeof buildProveEvidence> }
585
+ | { written: false; reason: string };
586
+
587
+ /**
588
+ * Build, validate and write `prove-evidence.json`, and say what happened
589
+ * (#1559). Never throws.
590
+ *
591
+ * B3 has always degraded to a warning when it could not record the artifact
592
+ * (`prove.test.ts` writes its own), so a failing `git rev-parse` or a write
593
+ * into a missing directory must not crash the prove gate. Every outcome that
594
+ * is not a written file says why.
595
+ */
596
+ export function recordProveEvidence(
597
+ raw: Record<string, any>,
598
+ opts: {
599
+ issueNumber: number;
600
+ workDir: string;
601
+ beforeReading: BeforeStateReading;
602
+ readCommit: () => string;
603
+ schema?: ArtifactSchema;
604
+ source?: string;
605
+ afterObserver?: AfterObserver;
606
+ },
607
+ ): ProveEvidenceRecord {
608
+ try {
609
+ const artifact = buildProveEvidence(raw, {
610
+ issueNumber: opts.issueNumber,
611
+ commitSHA: opts.readCommit(),
612
+ capturedAt: new Date().toISOString(),
613
+ beforeReading: opts.beforeReading,
614
+ source: opts.source,
615
+ afterObserver: opts.afterObserver,
616
+ });
617
+ const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
618
+ if (violations.length > 0) {
619
+ return { written: false, reason: `it fails ProveEvidenceSchema (SC-2770): ${violations.join("; ")}` };
620
+ }
621
+ writeFileSync(join(opts.workDir, "prove-evidence.json"), JSON.stringify(artifact, null, 2));
622
+ return { written: true, artifact };
623
+ } catch (e: any) {
624
+ return { written: false, reason: `could not record it: ${String(e?.message ?? e).slice(0, 200)}` };
625
+ }
626
+ }
627
+
429
628
  /** What a gate run does about the tests that existed when it last looked. */
430
629
  export type CensusVerdict =
431
630
  | { kind: "unmeasured"; reason: string }
@@ -1554,26 +1753,33 @@ async function runProveReproducer(state: Record<string, any>, issue: number, iss
1554
1753
  }
1555
1754
 
1556
1755
  const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
1756
+ let parsed: Record<string, any> | null = null;
1557
1757
  if (jsonMatch) {
1558
1758
  try {
1559
- const parsed = JSON.parse(jsonMatch[0]);
1560
- const proveEvidence = {
1561
- issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
1562
- commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
1563
- capturedAt: new Date().toISOString(),
1564
- criteriaResults: parsed.criteriaResults || [],
1565
- reproduced: parsed.reproduced ?? false,
1566
- evidence: parsed.evidence || [],
1567
- source: "B3-prove-reproducer",
1568
- };
1569
- writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
1570
- console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
1759
+ parsed = JSON.parse(jsonMatch[0]);
1571
1760
  } catch {
1572
1761
  console.warn("WARN: B3 reproducer JSON parse failed");
1573
1762
  }
1574
1763
  } else {
1575
1764
  console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
1576
1765
  }
1766
+ if (parsed && typeof parsed === "object") {
1767
+ // #1559: the artifact is built in one place and checked against its one
1768
+ // contract before it is written. An artifact the schema refuses is not
1769
+ // written, and the fields are named; nothing here can crash the gate.
1770
+ const record = recordProveEvidence(parsed, {
1771
+ issueNumber: issue, workDir, beforeReading,
1772
+ readCommit: () => execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
1773
+ });
1774
+ if (!record.written) {
1775
+ console.warn(`WARN: B3 prove-evidence.json NOT written — ${record.reason}; prove.test.ts will generate evidence`);
1776
+ } else {
1777
+ console.log(`B3: prove-evidence.json written — verdict: ${record.artifact.verdict}`);
1778
+ if (record.artifact.verdictDowngrade) {
1779
+ console.warn(`WARN: B3 reproducer reported ${JSON.stringify(record.artifact.verdictDowngrade.reported)}, recorded INCONCLUSIVE — ${record.artifact.verdictDowngrade.reason}`);
1780
+ }
1781
+ }
1782
+ }
1577
1783
  }
1578
1784
  }
1579
1785
 
@@ -2318,7 +2524,7 @@ function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, ha
2318
2524
  writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
2319
2525
  const { spawn } = require("child_process");
2320
2526
  const child = spawn("sh", ["-c",
2321
- `claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format json --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
2527
+ `claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format text --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
2322
2528
  ], { detached: true, stdio: "ignore", cwd: harnessRoot });
2323
2529
  child.unref();
2324
2530
  console.log("B1: AC Adversary spawned in background — verify gate will check result");
@@ -2424,40 +2630,138 @@ function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, resu
2424
2630
  results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
2425
2631
  console.log("B1: adversary report not yet available — verify FAIL");
2426
2632
  } else {
2633
+ let raw: string | null = null;
2427
2634
  try {
2428
- const raw = readFileSync(reportPath, "utf-8");
2429
- const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2430
- if (jsonMatch) {
2431
- const report = JSON.parse(jsonMatch[0]);
2432
- console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
2433
- if (report.gameable > 0) {
2434
- let outcomeCount = 0;
2435
- for (const exploit of report.exploits || []) {
2436
- const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
2437
- const isOutcome = ac?.type === "OUTCOME";
2438
- if (isOutcome) {
2439
- addedFails++;
2440
- outcomeCount++;
2441
- results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
2442
- } else {
2443
- addedWarns++;
2444
- results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2445
- }
2446
- }
2447
- if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
2448
- const codeCount = (report.gameable || 0) - outcomeCount;
2449
- if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
2450
- }
2451
- } else {
2452
- console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
2453
- }
2635
+ raw = readFileSync(reportPath, "utf-8");
2454
2636
  } catch (e: any) {
2455
- console.warn(`WARN: B1 report read failed: ${e.message?.slice(0, 200)}`);
2637
+ addedFails++;
2638
+ results.push({ check: "B1-adversary: report unreadable", result: "FAIL" as const, detail: `adversary-report.json could not be read: ${e.message?.slice(0, 200)}` });
2639
+ }
2640
+ if (raw !== null) {
2641
+ const reading = adversaryReportReading(raw, state.acs || []);
2642
+ addedFails += reading.fails;
2643
+ addedWarns += reading.warns;
2644
+ results.push(...reading.results);
2645
+ console.log(`B1: adversary report — ${reading.fails} FAIL, ${reading.warns} WARN`);
2456
2646
  }
2457
2647
  }
2458
2648
  return { fails: addedFails, warns: addedWarns };
2459
2649
  }
2460
2650
 
2651
+ /**
2652
+ * Read B1's adversary report into gate results (#2259, SC-2770).
2653
+ *
2654
+ * The report is raw output from the model B1 spawns. It was read with a
2655
+ * regex and `JSON.parse` and never checked against `AdversaryReportSchema`,
2656
+ * so `"gameable": "2"` with no exploits, or `exploits` that was not a list,
2657
+ * produced no result at all: the reading of a clean report. A report with no
2658
+ * JSON only printed a warning, so the gate passed on a report it never read.
2659
+ * Now an unreadable report and a report the schema refuses are each one FAIL
2660
+ * naming why, the same as a missing report. Exported so it runs on fixtures;
2661
+ * the schema is a parameter so a test can hand it a stricter copy.
2662
+ */
2663
+ export function adversaryReportReading(
2664
+ raw: string,
2665
+ acs: ReadonlyArray<{ id?: unknown; type?: unknown }>,
2666
+ schema: ArtifactSchema = AdversaryReportSchema,
2667
+ ): { fails: number; warns: number; results: GateResult[] } {
2668
+ const parsed = parseAdversaryReport(raw, schema);
2669
+ if (parsed.status !== "read") {
2670
+ return { fails: 1, warns: 0, results: [{ check: `B1-adversary: ${parsed.check}`, result: "FAIL" as const, detail: parsed.detail }] };
2671
+ }
2672
+ let fails = 0;
2673
+ let warns = 0;
2674
+ const results: GateResult[] = [];
2675
+ // Every listed exploit is read whatever `gameable` says: one dropped because
2676
+ // the count was 0 is the same silent pass in the other direction.
2677
+ for (const exploit of parsed.report.exploits) {
2678
+ const ac = acs.find(a => a.id === exploit.acId);
2679
+ if (ac?.type === "OUTCOME") {
2680
+ fails++;
2681
+ results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
2682
+ } else {
2683
+ warns++;
2684
+ results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2685
+ }
2686
+ }
2687
+ return { fails, warns, results };
2688
+ }
2689
+
2690
+ type AdversaryReport = {
2691
+ gameable: number;
2692
+ approved: boolean;
2693
+ exploits: Array<{ acId: string; exploit: string; recommendation: string }>;
2694
+ };
2695
+
2696
+ /**
2697
+ * THE ONE READ of a B1 adversary report (#2259), for the report on disk and
2698
+ * for the live re-run at verify alike. Both used to spell their own regex and
2699
+ * `JSON.parse` with no schema, and review round 1 found the second copy still
2700
+ * swallowing a malformed reply after the first was fixed. Two readers of one
2701
+ * shape are one guard with a dead twin; test/adversary-report-schema.test.ts
2702
+ * asserts the extraction appears once in this file.
2703
+ *
2704
+ * Not "read" when the output holds no JSON naming `gameable`, when that JSON
2705
+ * does not parse, when the schema refuses it, or when it claims more gameable
2706
+ * criteria than it lists exploits for (`{gameable: 3, exploits: []}` is
2707
+ * schema-valid and used to read as clean). More exploits than the count is
2708
+ * not a contradiction: two ways to game one criterion.
2709
+ */
2710
+ export function parseAdversaryReport(
2711
+ raw: string,
2712
+ schema: ArtifactSchema = AdversaryReportSchema,
2713
+ ): { status: "read"; report: AdversaryReport } | { status: "not-read"; check: string; detail: string } {
2714
+ const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2715
+ if (!jsonMatch) {
2716
+ return { status: "not-read", check: "report unreadable", detail: "the output holds no JSON object naming \"gameable\"" };
2717
+ }
2718
+ let report: any;
2719
+ try {
2720
+ report = JSON.parse(jsonMatch[0]);
2721
+ } catch (e: any) {
2722
+ return { status: "not-read", check: "report unreadable", detail: `the report's JSON does not parse: ${e.message?.slice(0, 200)}` };
2723
+ }
2724
+ const violations = validateArtifactSchema(report, schema);
2725
+ if (violations.length > 0) {
2726
+ return { status: "not-read", check: "report fails AdversaryReportSchema", detail: violations.join("; ") };
2727
+ }
2728
+ if (report.gameable > report.exploits.length) {
2729
+ return {
2730
+ status: "not-read",
2731
+ check: "report contradicts itself",
2732
+ detail: `gameable ${report.gameable} but ${report.exploits.length} exploit(s) listed, so which criteria are gameable is unknown`,
2733
+ };
2734
+ }
2735
+ return { status: "read", report };
2736
+ }
2737
+
2738
+ /**
2739
+ * The live B1 re-run at verify, read (#2259 review round 1). Advisory, as it
2740
+ * always was: every finding is a WARN. What changes is that a reply that could
2741
+ * not be read is a WARN naming why, and `agentResult` records it as
2742
+ * UNMEASURED, where it used to be `null` and the verify gate's record of the
2743
+ * adversary was never written at all.
2744
+ */
2745
+ export function liveAdversaryReading(
2746
+ output: string,
2747
+ ts: string,
2748
+ ): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
2749
+ const parsed = parseAdversaryReport(output);
2750
+ if (parsed.status !== "read") {
2751
+ return {
2752
+ agentResult: { ts, status: "UNMEASURED", reason: `${parsed.check}: ${parsed.detail}`, source: "B1-adversary-verify-agent" },
2753
+ warns: 1,
2754
+ results: [{ check: `B1-adversary-verify: ${parsed.check}`, result: "WARN" as const, detail: parsed.detail }],
2755
+ };
2756
+ }
2757
+ const { report } = parsed;
2758
+ return {
2759
+ agentResult: { ts, gameable: report.gameable, approved: report.approved, exploits: report.exploits, source: "B1-adversary-verify-agent" },
2760
+ warns: report.exploits.length,
2761
+ results: report.exploits.map(exploit => ({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit })),
2762
+ };
2763
+ }
2764
+
2461
2765
  function runB2EvidenceValidation(state: Record<string, any>): void {
2462
2766
  const tier = state.sizing?.ceremonyTier || "STANDARD";
2463
2767
  if (tier === "LIGHT") return;
@@ -2568,9 +2872,70 @@ export function fanoutSkipReason(
2568
2872
  return null;
2569
2873
  }
2570
2874
 
2875
+ /** One row of the B2 validator's reply, the shape gates/prompts/evidence-validator.md asks for. */
2876
+ export type B2Verdict = { acId: string; verdict: "PASS" | "FAIL"; rawOutput?: unknown; [field: string]: unknown };
2877
+
2878
+ /**
2879
+ * The B2 validator's reply, or why it could not be read (#2265). The one
2880
+ * place its JSON is extracted: two copies are two readers that can disagree,
2881
+ * which is what #2264's first review round found for B1.
2882
+ */
2883
+ export function parseB2Reply(raw: string): { status: "read"; verdicts: B2Verdict[] } | { status: "not-read"; detail: string } {
2884
+ const json = raw.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
2885
+ if (!json) return { status: "not-read", detail: 'the reply holds no JSON object naming "verdicts"' };
2886
+ let reply: any;
2887
+ try {
2888
+ reply = JSON.parse(json[0]);
2889
+ } catch (e: any) {
2890
+ return { status: "not-read", detail: `the reply's JSON does not parse: ${String(e?.message ?? e).slice(0, 200)}` };
2891
+ }
2892
+ if (!Array.isArray(reply?.verdicts)) {
2893
+ return { status: "not-read", detail: `verdicts is ${JSON.stringify(reply?.verdicts) ?? "absent"}, not a list` };
2894
+ }
2895
+ const bad = reply.verdicts.findIndex((v: any) =>
2896
+ v === null || typeof v !== "object" || typeof v.acId !== "string" || v.acId.trim() === "" ||
2897
+ (v.verdict !== "PASS" && v.verdict !== "FAIL"));
2898
+ if (bad >= 0) {
2899
+ return {
2900
+ status: "not-read",
2901
+ detail: `verdicts[${bad}] has no acId, or a verdict that is not PASS or FAIL: ${String(JSON.stringify(reply.verdicts[bad])).slice(0, 200)}`,
2902
+ };
2903
+ }
2904
+ return { status: "read", verdicts: reply.verdicts as B2Verdict[] };
2905
+ }
2906
+
2907
+ /**
2908
+ * The B2 validator's reply as verify-gate results and the reading verify
2909
+ * records (#2265). Advisory, as it always was: every result is a WARN. A
2910
+ * reply that could not be read is one WARN naming why, and the record says
2911
+ * UNMEASURED. Before, it was nothing: no result, and no record, so the run
2912
+ * looked like one whose every criterion's evidence was sound.
2913
+ */
2914
+ export function b2ReplyReading(
2915
+ output: string,
2916
+ ts: string,
2917
+ ): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
2918
+ const parsed = parseB2Reply(output);
2919
+ if (parsed.status !== "read") {
2920
+ return {
2921
+ agentResult: { ts, status: "UNMEASURED", reason: `reply unreadable: ${parsed.detail}`, source: "B2-evidence-validator-agent" },
2922
+ warns: 1,
2923
+ results: [{ check: "B2-validator: reply unreadable", result: "WARN" as const, detail: parsed.detail }],
2924
+ };
2925
+ }
2926
+ const fails = parsed.verdicts.filter(v => v.verdict === "FAIL");
2927
+ return {
2928
+ agentResult: { ts, verdicts: parsed.verdicts, source: "B2-evidence-validator-agent" },
2929
+ warns: fails.length,
2930
+ results: fails.map(v => ({
2931
+ check: `B2-validator: ${v.acId} evidence gap`,
2932
+ result: "WARN" as const,
2933
+ detail: typeof v.rawOutput === "string" && v.rawOutput !== "" ? v.rawOutput : "threshold not met",
2934
+ })),
2935
+ };
2936
+ }
2937
+
2571
2938
  function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
2572
- let addedWarns = 0;
2573
- let agentResult: any = null;
2574
2939
  const tier = state.sizing?.ceremonyTier || "STANDARD";
2575
2940
  // Recorded rather than returned empty (#126): the consumer allows a LIGHT
2576
2941
  // absence, but an artefact that SAYS why is one a reader does not have to
@@ -2610,33 +2975,16 @@ function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string
2610
2975
  console.warn(`WARN: B2 Evidence Validator exited with error: ${e.message?.slice(0, 200)}`);
2611
2976
  }
2612
2977
 
2613
- const b2Json = b2Output.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
2614
- if (b2Json) {
2615
- try {
2616
- const b2Report = JSON.parse(b2Json[0]);
2617
- agentResult = {
2618
- ts: new Date().toISOString(),
2619
- verdicts: b2Report.verdicts || [],
2620
- source: "B2-evidence-validator-agent",
2621
- };
2622
- const b2Fails = (b2Report.verdicts || []).filter((v: any) => v.verdict === "FAIL");
2623
- if (b2Fails.length > 0) {
2624
- for (const v of b2Fails) {
2625
- addedWarns++;
2626
- results.push({ check: `B2-validator: ${v.acId} evidence gap`, result: "WARN" as const, detail: v.rawOutput || "threshold not met" });
2627
- }
2628
- console.log(`B2: ${b2Fails.length} evidence gap(s) found — WARN`);
2629
- } else {
2630
- console.log(`B2: All evidence validated — ${(b2Report.verdicts || []).length} ACs checked`);
2631
- }
2632
- } catch {
2633
- console.warn("WARN: B2 Evidence Validator JSON parse failed");
2634
- }
2978
+ const reading = b2ReplyReading(b2Output, new Date().toISOString());
2979
+ results.push(...reading.results);
2980
+ if (reading.agentResult.status === "UNMEASURED") {
2981
+ console.warn(`WARN: B2 Evidence Validator reply UNMEASURED — ${reading.agentResult.reason}`);
2982
+ } else if (reading.warns > 0) {
2983
+ console.log(`B2: ${reading.warns} evidence gap(s) found — WARN`);
2635
2984
  } else {
2636
- console.warn("WARN: B2 Evidence Validator produced no parseable JSON");
2985
+ console.log(`B2: All evidence validated — ${(reading.agentResult.verdicts as unknown[]).length} ACs checked`);
2637
2986
  }
2638
-
2639
- return { agentResult, warns: addedWarns };
2987
+ return { agentResult: reading.agentResult, warns: reading.warns };
2640
2988
  }
2641
2989
 
2642
2990
  function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
@@ -2677,32 +3025,11 @@ function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, res
2677
3025
  console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
2678
3026
  }
2679
3027
 
2680
- const b1Json = b1Output.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
2681
- if (b1Json) {
2682
- try {
2683
- const b1Report = JSON.parse(b1Json[0]);
2684
- agentResult = {
2685
- ts: new Date().toISOString(),
2686
- gameable: b1Report.gameable || 0,
2687
- approved: b1Report.approved ?? true,
2688
- exploits: b1Report.exploits || [],
2689
- source: "B1-adversary-verify-agent",
2690
- };
2691
- if (b1Report.gameable > 0) {
2692
- for (const exploit of b1Report.exploits || []) {
2693
- addedWarns++;
2694
- results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
2695
- }
2696
- console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
2697
- } else {
2698
- console.log(`B1: All ACs approved at verify — no exploits found`);
2699
- }
2700
- } catch {
2701
- console.warn("WARN: B1 AC Adversary JSON parse failed");
2702
- }
2703
- } else {
2704
- console.warn("WARN: B1 AC Adversary produced no parseable JSON");
2705
- }
3028
+ const live = liveAdversaryReading(b1Output, new Date().toISOString());
3029
+ agentResult = live.agentResult;
3030
+ addedWarns += live.warns;
3031
+ results.push(...live.results);
3032
+ console.log(`B1: verify agent — ${live.warns} WARN${live.agentResult.status === "UNMEASURED" ? ` (UNMEASURED: ${live.agentResult.reason})` : ""}`);
2706
3033
 
2707
3034
  return { agentResult, warns: addedWarns };
2708
3035
  }
@@ -3198,7 +3525,7 @@ export async function executeGate(input: GateExecutorInput): Promise<GateExecuto
3198
3525
  const hash = generateHmac(sf, slug, issue, state.projectRoot);
3199
3526
  console.log(`HMAC: ${hash}`);
3200
3527
  const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
3201
- console.log(`Ship evidence: ${evidencePath}`);
3528
+ console.log(evidencePath ? `Ship evidence: ${evidencePath}` : "Ship evidence: NOT written — see the fields named above (#2255)");
3202
3529
  }
3203
3530
 
3204
3531
  // Verify-specific: convergence