rungate 0.8.23 → 0.8.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +423 -96
- package/gates/orchestrator.test.ts +108 -2
- package/gates/orchestrator.ts +25 -5
- package/gates/prompts/prove-reproducer.md +14 -1
- package/gates/prompts.sha256 +1 -1
- package/gates/prove.test.ts +6 -1
- package/gates/schema.ts +25 -1
- package/gates/workflow.test.ts +12 -0
- package/lib/agents-md-sections.ts +4 -1
- package/lib/ai-review-suppression.ts +26 -2
- package/lib/artifact-schema.ts +37 -0
- package/lib/bash-write-guard.ts +667 -257
- package/lib/ci-reading.ts +186 -0
- package/lib/cli-commands.ts +8 -0
- package/lib/rule-provenance.ts +54 -3
- package/lib/scaffold/ai-review-step.ts +95 -83
- package/lib/scaffold/intake-workflow.ts +120 -0
- package/lib/scaffold/marked-section.ts +75 -0
- package/lib/scaffold/owned-files.ts +12 -2
- package/lib/scaffold/steps.ts +167 -4
- package/lib/ship-mode.ts +15 -3
- package/package.json +3 -1
- package/scripts/ai-review-findings.ts +6 -2
- package/scripts/ci-reading.ts +192 -0
- package/scripts/detect-sc-drift.ts +44 -2
- package/scripts/scaffold-project.ts +13 -1
- package/config/prove-evidence.schema.json +0 -83
package/gates/.gate-salt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
f7edfff5c8424ba64c95ad1a592d7b40c40c3a78197e968739a5d468cd671467
|
package/gates/gate-executor.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { execSync, execFileSync } from "child_process";
|
|
2
|
-
import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync } from "fs";
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync, statSync } from "fs";
|
|
3
3
|
import { join } from "path";
|
|
4
4
|
import { splitGateRuns } from "../lib/gate-run-split";
|
|
5
5
|
import { gateSaltPath, workDir as slugWorkDir } from "../lib/paths";
|
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
measureEvidencePrevalidation,
|
|
39
39
|
type EvidencePrevalidationReading,
|
|
40
40
|
} from "../lib/evidence-prevalidator";
|
|
41
|
-
import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
|
|
41
|
+
import { AdversaryReportSchema, EvidencePrevalidationSchema, ProveEvidenceSchema, type PrevalidationSource } from "./schema";
|
|
42
42
|
import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
|
|
43
43
|
import { deepMerge } from "../lib/deep-merge";
|
|
44
44
|
import { livenessPlan, probeTarget } from "../lib/dev-liveness";
|
|
@@ -426,6 +426,205 @@ export function comparisonSummaryFor(
|
|
|
426
426
|
return `compared against the ${reading.type} before-state captured at ${reading.capturedAt}; ${counts}`;
|
|
427
427
|
}
|
|
428
428
|
|
|
429
|
+
// The validator moved to lib/ in #2255 so gates/orchestrator.ts can use it
|
|
430
|
+
// without importing this file back; re-exported so it has one definition.
|
|
431
|
+
import { validateArtifactSchema, type ArtifactSchema } from "../lib/artifact-schema";
|
|
432
|
+
export { validateArtifactSchema, type ArtifactSchema };
|
|
433
|
+
|
|
434
|
+
const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
435
|
+
/**
|
|
436
|
+
* Who observed the after-state, and the after-evidence types each can report
|
|
437
|
+
* (#2262). `none` is B3's to write. The reproducer has Bash and Read only
|
|
438
|
+
* (gates/prompts/prove-reproducer.md), so a screenshot it reports is a UI
|
|
439
|
+
* check it could not have made. Quinn drives a browser against the prove
|
|
440
|
+
* container (workflows/prove.js), so a screenshot is what Quinn observes.
|
|
441
|
+
*
|
|
442
|
+
* The CALLER names the observer, never the report: an agent writing
|
|
443
|
+
* `observedBy: "quinn"` into its own afterEvidence is ignored. The spawn
|
|
444
|
+
* schemas in workflows/prove.js and `ProveEvidenceSchema`'s `observedBy` restate
|
|
445
|
+
* these lists, and test/prove-workflow-evidence.test.ts holds them equal.
|
|
446
|
+
*/
|
|
447
|
+
export const AFTER_OBSERVERS = {
|
|
448
|
+
reproducer: ["text", "api", "data"],
|
|
449
|
+
quinn: ["text", "api", "data", "screenshot"],
|
|
450
|
+
} as const;
|
|
451
|
+
export type AfterObserver = keyof typeof AFTER_OBSERVERS;
|
|
452
|
+
export const isAfterObserver = (v: unknown): v is AfterObserver =>
|
|
453
|
+
typeof v === "string" && Object.prototype.hasOwnProperty.call(AFTER_OBSERVERS, v);
|
|
454
|
+
const AFTER_ENVIRONMENTS = ["local", "prod"];
|
|
455
|
+
|
|
456
|
+
/** What B3 knows about the run that the reproducer does not get to say. */
|
|
457
|
+
export interface ProveEvidenceContext {
|
|
458
|
+
issueNumber: number;
|
|
459
|
+
commitSHA: string;
|
|
460
|
+
capturedAt: string;
|
|
461
|
+
beforeReading: BeforeStateReading;
|
|
462
|
+
/** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
|
|
463
|
+
source?: string;
|
|
464
|
+
/** Who observed the after-state. The reproducer unless the caller says otherwise (#2262). */
|
|
465
|
+
afterObserver?: AfterObserver;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
type AfterEvidence = {
|
|
469
|
+
type: string; path: string | null; capturedAt: string; environment: string; description: string;
|
|
470
|
+
observedBy?: AfterObserver;
|
|
471
|
+
};
|
|
472
|
+
|
|
473
|
+
function isFile(path: string): boolean {
|
|
474
|
+
try {
|
|
475
|
+
return statSync(path).isFile();
|
|
476
|
+
} catch {
|
|
477
|
+
return false; // absent or unreadable: either way nobody can open it
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** The reported `afterEvidence`, or why there is none. Never repaired into shape. */
|
|
482
|
+
function readReportedAfterEvidence(raw: unknown, observer: AfterObserver): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
|
|
483
|
+
if (raw === undefined || raw === null) return { ok: false, reason: `the ${observer} reported no afterEvidence` };
|
|
484
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
485
|
+
return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
|
|
486
|
+
}
|
|
487
|
+
const a = raw as Record<string, unknown>;
|
|
488
|
+
const types: readonly string[] = AFTER_OBSERVERS[observer];
|
|
489
|
+
if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
|
|
490
|
+
return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
|
|
491
|
+
}
|
|
492
|
+
if (typeof a.type !== "string" || !types.includes(a.type)) {
|
|
493
|
+
return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one the ${observer} can observe (${types.join("|")})` };
|
|
494
|
+
}
|
|
495
|
+
if (a.type === "screenshot" && (typeof a.path !== "string" || a.path.trim() === "")) {
|
|
496
|
+
return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is empty — a screenshot with no file is not evidence` };
|
|
497
|
+
}
|
|
498
|
+
// Review round 1, PR #2266: a path is a claim. The file has to be there,
|
|
499
|
+
// or a screenshot nobody took records as proof.
|
|
500
|
+
if (a.type === "screenshot" && !isFile(a.path as string)) {
|
|
501
|
+
return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is not a file — a screenshot nobody can open is not evidence` };
|
|
502
|
+
}
|
|
503
|
+
if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
|
|
504
|
+
return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
|
|
505
|
+
}
|
|
506
|
+
if (typeof a.description !== "string" || a.description.trim() === "") {
|
|
507
|
+
return { ok: false, reason: "afterEvidence.description is empty — it must say what the after-state showed" };
|
|
508
|
+
}
|
|
509
|
+
return {
|
|
510
|
+
ok: true,
|
|
511
|
+
value: {
|
|
512
|
+
type: a.type, path: typeof a.path === "string" ? a.path : null,
|
|
513
|
+
capturedAt: a.capturedAt, environment: a.environment, description: a.description,
|
|
514
|
+
observedBy: observer,
|
|
515
|
+
},
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Build `prove-evidence.json` from the reproducer's report (#1559, Jason's
|
|
521
|
+
* option (a) of 2026-11-15).
|
|
522
|
+
*
|
|
523
|
+
* B3 used to write `{issueNumber, verdict, commitSHA, capturedAt,
|
|
524
|
+
* criteriaResults, reproduced, evidence, source}`, which lacks the
|
|
525
|
+
* `beforeEvidence`, `afterEvidence` and `comparisonSummary` that
|
|
526
|
+
* `ProveEvidenceSchema` requires. The before-evidence and the comparison are
|
|
527
|
+
* B3's own readings (#1540), so a reproducer cannot overwrite them. The
|
|
528
|
+
* after-evidence is the reproducer's report of the environment it tested and
|
|
529
|
+
* what it saw. When that report is missing or malformed, the record is
|
|
530
|
+
* `type: "none"` and a PROVEN becomes INCONCLUSIVE, because a proof with no
|
|
531
|
+
* observed after-state is not a proof.
|
|
532
|
+
*
|
|
533
|
+
* The `none` record says `environment: "local"` because the schema requires
|
|
534
|
+
* a value. `local` is the one that satisfies no STANDARD+ prod requirement
|
|
535
|
+
* (gates/prove.test.ts), so a missing after-evidence can never pass that check.
|
|
536
|
+
*/
|
|
537
|
+
export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
|
|
538
|
+
const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
|
|
539
|
+
const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
|
|
540
|
+
const after = readReportedAfterEvidence(raw?.afterEvidence, ctx.afterObserver ?? "reproducer");
|
|
541
|
+
|
|
542
|
+
let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
|
|
543
|
+
let verdictDowngrade: { reported: unknown; reason: string } | undefined;
|
|
544
|
+
if (verdict !== raw?.verdict) {
|
|
545
|
+
verdictDowngrade = {
|
|
546
|
+
reported: raw?.verdict,
|
|
547
|
+
reason: `verdict ${JSON.stringify(raw?.verdict)} is not one of ${PROVE_VERDICTS.join("|")}`,
|
|
548
|
+
};
|
|
549
|
+
} else if (verdict === "PROVEN" && !after.ok) {
|
|
550
|
+
verdict = "INCONCLUSIVE";
|
|
551
|
+
verdictDowngrade = { reported: "PROVEN", reason: `PROVEN with no after-evidence: ${after.reason}` };
|
|
552
|
+
} else if (verdict === "PROVEN" && passed === 0) {
|
|
553
|
+
// A proof that verified no criterion is not a proof: an empty list, or
|
|
554
|
+
// nothing but SKIPs, is what a reproducer with no way to check says.
|
|
555
|
+
verdict = "INCONCLUSIVE";
|
|
556
|
+
verdictDowngrade = {
|
|
557
|
+
reported: "PROVEN",
|
|
558
|
+
reason: `PROVEN with no criterion verified: ${criteriaResults.length} criteriaResults row(s), none PASS`,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
const afterEvidence: AfterEvidence = after.ok ? after.value : {
|
|
563
|
+
type: "none", path: null, capturedAt: ctx.capturedAt, environment: "local",
|
|
564
|
+
description: `no after-evidence: ${after.reason}`,
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
return {
|
|
568
|
+
issueNumber: ctx.issueNumber,
|
|
569
|
+
verdict,
|
|
570
|
+
...(verdictDowngrade ? { verdictDowngrade } : {}),
|
|
571
|
+
commitSHA: ctx.commitSHA,
|
|
572
|
+
capturedAt: ctx.capturedAt,
|
|
573
|
+
beforeEvidence: beforeEvidenceFor(ctx.beforeReading),
|
|
574
|
+
afterEvidence,
|
|
575
|
+
comparisonSummary: comparisonSummaryFor(ctx.beforeReading, passed, criteriaResults.length),
|
|
576
|
+
reproduced: raw?.reproduced === true,
|
|
577
|
+
criteriaResults,
|
|
578
|
+
evidence: Array.isArray(raw?.evidence) ? raw.evidence : [],
|
|
579
|
+
source: ctx.source ?? "B3-prove-reproducer",
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
export type ProveEvidenceRecord =
|
|
584
|
+
| { written: true; artifact: ReturnType<typeof buildProveEvidence> }
|
|
585
|
+
| { written: false; reason: string };
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* Build, validate and write `prove-evidence.json`, and say what happened
|
|
589
|
+
* (#1559). Never throws.
|
|
590
|
+
*
|
|
591
|
+
* B3 has always degraded to a warning when it could not record the artifact
|
|
592
|
+
* (`prove.test.ts` writes its own), so a failing `git rev-parse` or a write
|
|
593
|
+
* into a missing directory must not crash the prove gate. Every outcome that
|
|
594
|
+
* is not a written file says why.
|
|
595
|
+
*/
|
|
596
|
+
export function recordProveEvidence(
|
|
597
|
+
raw: Record<string, any>,
|
|
598
|
+
opts: {
|
|
599
|
+
issueNumber: number;
|
|
600
|
+
workDir: string;
|
|
601
|
+
beforeReading: BeforeStateReading;
|
|
602
|
+
readCommit: () => string;
|
|
603
|
+
schema?: ArtifactSchema;
|
|
604
|
+
source?: string;
|
|
605
|
+
afterObserver?: AfterObserver;
|
|
606
|
+
},
|
|
607
|
+
): ProveEvidenceRecord {
|
|
608
|
+
try {
|
|
609
|
+
const artifact = buildProveEvidence(raw, {
|
|
610
|
+
issueNumber: opts.issueNumber,
|
|
611
|
+
commitSHA: opts.readCommit(),
|
|
612
|
+
capturedAt: new Date().toISOString(),
|
|
613
|
+
beforeReading: opts.beforeReading,
|
|
614
|
+
source: opts.source,
|
|
615
|
+
afterObserver: opts.afterObserver,
|
|
616
|
+
});
|
|
617
|
+
const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
|
|
618
|
+
if (violations.length > 0) {
|
|
619
|
+
return { written: false, reason: `it fails ProveEvidenceSchema (SC-2770): ${violations.join("; ")}` };
|
|
620
|
+
}
|
|
621
|
+
writeFileSync(join(opts.workDir, "prove-evidence.json"), JSON.stringify(artifact, null, 2));
|
|
622
|
+
return { written: true, artifact };
|
|
623
|
+
} catch (e: any) {
|
|
624
|
+
return { written: false, reason: `could not record it: ${String(e?.message ?? e).slice(0, 200)}` };
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
429
628
|
/** What a gate run does about the tests that existed when it last looked. */
|
|
430
629
|
export type CensusVerdict =
|
|
431
630
|
| { kind: "unmeasured"; reason: string }
|
|
@@ -1554,26 +1753,33 @@ async function runProveReproducer(state: Record<string, any>, issue: number, iss
|
|
|
1554
1753
|
}
|
|
1555
1754
|
|
|
1556
1755
|
const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
|
|
1756
|
+
let parsed: Record<string, any> | null = null;
|
|
1557
1757
|
if (jsonMatch) {
|
|
1558
1758
|
try {
|
|
1559
|
-
|
|
1560
|
-
const proveEvidence = {
|
|
1561
|
-
issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
|
|
1562
|
-
commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1563
|
-
capturedAt: new Date().toISOString(),
|
|
1564
|
-
criteriaResults: parsed.criteriaResults || [],
|
|
1565
|
-
reproduced: parsed.reproduced ?? false,
|
|
1566
|
-
evidence: parsed.evidence || [],
|
|
1567
|
-
source: "B3-prove-reproducer",
|
|
1568
|
-
};
|
|
1569
|
-
writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
|
|
1570
|
-
console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
|
|
1759
|
+
parsed = JSON.parse(jsonMatch[0]);
|
|
1571
1760
|
} catch {
|
|
1572
1761
|
console.warn("WARN: B3 reproducer JSON parse failed");
|
|
1573
1762
|
}
|
|
1574
1763
|
} else {
|
|
1575
1764
|
console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
|
|
1576
1765
|
}
|
|
1766
|
+
if (parsed && typeof parsed === "object") {
|
|
1767
|
+
// #1559: the artifact is built in one place and checked against its one
|
|
1768
|
+
// contract before it is written. An artifact the schema refuses is not
|
|
1769
|
+
// written, and the fields are named; nothing here can crash the gate.
|
|
1770
|
+
const record = recordProveEvidence(parsed, {
|
|
1771
|
+
issueNumber: issue, workDir, beforeReading,
|
|
1772
|
+
readCommit: () => execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1773
|
+
});
|
|
1774
|
+
if (!record.written) {
|
|
1775
|
+
console.warn(`WARN: B3 prove-evidence.json NOT written — ${record.reason}; prove.test.ts will generate evidence`);
|
|
1776
|
+
} else {
|
|
1777
|
+
console.log(`B3: prove-evidence.json written — verdict: ${record.artifact.verdict}`);
|
|
1778
|
+
if (record.artifact.verdictDowngrade) {
|
|
1779
|
+
console.warn(`WARN: B3 reproducer reported ${JSON.stringify(record.artifact.verdictDowngrade.reported)}, recorded INCONCLUSIVE — ${record.artifact.verdictDowngrade.reason}`);
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1577
1783
|
}
|
|
1578
1784
|
}
|
|
1579
1785
|
|
|
@@ -2318,7 +2524,7 @@ function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, ha
|
|
|
2318
2524
|
writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
|
|
2319
2525
|
const { spawn } = require("child_process");
|
|
2320
2526
|
const child = spawn("sh", ["-c",
|
|
2321
|
-
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format
|
|
2527
|
+
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format text --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
|
|
2322
2528
|
], { detached: true, stdio: "ignore", cwd: harnessRoot });
|
|
2323
2529
|
child.unref();
|
|
2324
2530
|
console.log("B1: AC Adversary spawned in background — verify gate will check result");
|
|
@@ -2424,40 +2630,138 @@ function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, resu
|
|
|
2424
2630
|
results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
|
|
2425
2631
|
console.log("B1: adversary report not yet available — verify FAIL");
|
|
2426
2632
|
} else {
|
|
2633
|
+
let raw: string | null = null;
|
|
2427
2634
|
try {
|
|
2428
|
-
|
|
2429
|
-
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2430
|
-
if (jsonMatch) {
|
|
2431
|
-
const report = JSON.parse(jsonMatch[0]);
|
|
2432
|
-
console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
|
|
2433
|
-
if (report.gameable > 0) {
|
|
2434
|
-
let outcomeCount = 0;
|
|
2435
|
-
for (const exploit of report.exploits || []) {
|
|
2436
|
-
const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
|
|
2437
|
-
const isOutcome = ac?.type === "OUTCOME";
|
|
2438
|
-
if (isOutcome) {
|
|
2439
|
-
addedFails++;
|
|
2440
|
-
outcomeCount++;
|
|
2441
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2442
|
-
} else {
|
|
2443
|
-
addedWarns++;
|
|
2444
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2445
|
-
}
|
|
2446
|
-
}
|
|
2447
|
-
if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
|
|
2448
|
-
const codeCount = (report.gameable || 0) - outcomeCount;
|
|
2449
|
-
if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
|
|
2450
|
-
}
|
|
2451
|
-
} else {
|
|
2452
|
-
console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
|
|
2453
|
-
}
|
|
2635
|
+
raw = readFileSync(reportPath, "utf-8");
|
|
2454
2636
|
} catch (e: any) {
|
|
2455
|
-
|
|
2637
|
+
addedFails++;
|
|
2638
|
+
results.push({ check: "B1-adversary: report unreadable", result: "FAIL" as const, detail: `adversary-report.json could not be read: ${e.message?.slice(0, 200)}` });
|
|
2639
|
+
}
|
|
2640
|
+
if (raw !== null) {
|
|
2641
|
+
const reading = adversaryReportReading(raw, state.acs || []);
|
|
2642
|
+
addedFails += reading.fails;
|
|
2643
|
+
addedWarns += reading.warns;
|
|
2644
|
+
results.push(...reading.results);
|
|
2645
|
+
console.log(`B1: adversary report — ${reading.fails} FAIL, ${reading.warns} WARN`);
|
|
2456
2646
|
}
|
|
2457
2647
|
}
|
|
2458
2648
|
return { fails: addedFails, warns: addedWarns };
|
|
2459
2649
|
}
|
|
2460
2650
|
|
|
2651
|
+
/**
|
|
2652
|
+
* Read B1's adversary report into gate results (#2259, SC-2770).
|
|
2653
|
+
*
|
|
2654
|
+
* The report is raw output from the model B1 spawns. It was read with a
|
|
2655
|
+
* regex and `JSON.parse` and never checked against `AdversaryReportSchema`,
|
|
2656
|
+
* so `"gameable": "2"` with no exploits, or `exploits` that was not a list,
|
|
2657
|
+
* produced no result at all: the reading of a clean report. A report with no
|
|
2658
|
+
* JSON only printed a warning, so the gate passed on a report it never read.
|
|
2659
|
+
* Now an unreadable report and a report the schema refuses are each one FAIL
|
|
2660
|
+
* naming why, the same as a missing report. Exported so it runs on fixtures;
|
|
2661
|
+
* the schema is a parameter so a test can hand it a stricter copy.
|
|
2662
|
+
*/
|
|
2663
|
+
export function adversaryReportReading(
|
|
2664
|
+
raw: string,
|
|
2665
|
+
acs: ReadonlyArray<{ id?: unknown; type?: unknown }>,
|
|
2666
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2667
|
+
): { fails: number; warns: number; results: GateResult[] } {
|
|
2668
|
+
const parsed = parseAdversaryReport(raw, schema);
|
|
2669
|
+
if (parsed.status !== "read") {
|
|
2670
|
+
return { fails: 1, warns: 0, results: [{ check: `B1-adversary: ${parsed.check}`, result: "FAIL" as const, detail: parsed.detail }] };
|
|
2671
|
+
}
|
|
2672
|
+
let fails = 0;
|
|
2673
|
+
let warns = 0;
|
|
2674
|
+
const results: GateResult[] = [];
|
|
2675
|
+
// Every listed exploit is read whatever `gameable` says: one dropped because
|
|
2676
|
+
// the count was 0 is the same silent pass in the other direction.
|
|
2677
|
+
for (const exploit of parsed.report.exploits) {
|
|
2678
|
+
const ac = acs.find(a => a.id === exploit.acId);
|
|
2679
|
+
if (ac?.type === "OUTCOME") {
|
|
2680
|
+
fails++;
|
|
2681
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2682
|
+
} else {
|
|
2683
|
+
warns++;
|
|
2684
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
return { fails, warns, results };
|
|
2688
|
+
}
|
|
2689
|
+
|
|
2690
|
+
type AdversaryReport = {
|
|
2691
|
+
gameable: number;
|
|
2692
|
+
approved: boolean;
|
|
2693
|
+
exploits: Array<{ acId: string; exploit: string; recommendation: string }>;
|
|
2694
|
+
};
|
|
2695
|
+
|
|
2696
|
+
/**
|
|
2697
|
+
* THE ONE READ of a B1 adversary report (#2259), for the report on disk and
|
|
2698
|
+
* for the live re-run at verify alike. Both used to spell their own regex and
|
|
2699
|
+
* `JSON.parse` with no schema, and review round 1 found the second copy still
|
|
2700
|
+
* swallowing a malformed reply after the first was fixed. Two readers of one
|
|
2701
|
+
* shape are one guard with a dead twin; test/adversary-report-schema.test.ts
|
|
2702
|
+
* asserts the extraction appears once in this file.
|
|
2703
|
+
*
|
|
2704
|
+
* Not "read" when the output holds no JSON naming `gameable`, when that JSON
|
|
2705
|
+
* does not parse, when the schema refuses it, or when it claims more gameable
|
|
2706
|
+
* criteria than it lists exploits for (`{gameable: 3, exploits: []}` is
|
|
2707
|
+
* schema-valid and used to read as clean). More exploits than the count is
|
|
2708
|
+
* not a contradiction: two ways to game one criterion.
|
|
2709
|
+
*/
|
|
2710
|
+
export function parseAdversaryReport(
|
|
2711
|
+
raw: string,
|
|
2712
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2713
|
+
): { status: "read"; report: AdversaryReport } | { status: "not-read"; check: string; detail: string } {
|
|
2714
|
+
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2715
|
+
if (!jsonMatch) {
|
|
2716
|
+
return { status: "not-read", check: "report unreadable", detail: "the output holds no JSON object naming \"gameable\"" };
|
|
2717
|
+
}
|
|
2718
|
+
let report: any;
|
|
2719
|
+
try {
|
|
2720
|
+
report = JSON.parse(jsonMatch[0]);
|
|
2721
|
+
} catch (e: any) {
|
|
2722
|
+
return { status: "not-read", check: "report unreadable", detail: `the report's JSON does not parse: ${e.message?.slice(0, 200)}` };
|
|
2723
|
+
}
|
|
2724
|
+
const violations = validateArtifactSchema(report, schema);
|
|
2725
|
+
if (violations.length > 0) {
|
|
2726
|
+
return { status: "not-read", check: "report fails AdversaryReportSchema", detail: violations.join("; ") };
|
|
2727
|
+
}
|
|
2728
|
+
if (report.gameable > report.exploits.length) {
|
|
2729
|
+
return {
|
|
2730
|
+
status: "not-read",
|
|
2731
|
+
check: "report contradicts itself",
|
|
2732
|
+
detail: `gameable ${report.gameable} but ${report.exploits.length} exploit(s) listed, so which criteria are gameable is unknown`,
|
|
2733
|
+
};
|
|
2734
|
+
}
|
|
2735
|
+
return { status: "read", report };
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
/**
|
|
2739
|
+
* The live B1 re-run at verify, read (#2259 review round 1). Advisory, as it
|
|
2740
|
+
* always was: every finding is a WARN. What changes is that a reply that could
|
|
2741
|
+
* not be read is a WARN naming why, and `agentResult` records it as
|
|
2742
|
+
* UNMEASURED, where it used to be `null` and the verify gate's record of the
|
|
2743
|
+
* adversary was never written at all.
|
|
2744
|
+
*/
|
|
2745
|
+
export function liveAdversaryReading(
|
|
2746
|
+
output: string,
|
|
2747
|
+
ts: string,
|
|
2748
|
+
): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
|
|
2749
|
+
const parsed = parseAdversaryReport(output);
|
|
2750
|
+
if (parsed.status !== "read") {
|
|
2751
|
+
return {
|
|
2752
|
+
agentResult: { ts, status: "UNMEASURED", reason: `${parsed.check}: ${parsed.detail}`, source: "B1-adversary-verify-agent" },
|
|
2753
|
+
warns: 1,
|
|
2754
|
+
results: [{ check: `B1-adversary-verify: ${parsed.check}`, result: "WARN" as const, detail: parsed.detail }],
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
const { report } = parsed;
|
|
2758
|
+
return {
|
|
2759
|
+
agentResult: { ts, gameable: report.gameable, approved: report.approved, exploits: report.exploits, source: "B1-adversary-verify-agent" },
|
|
2760
|
+
warns: report.exploits.length,
|
|
2761
|
+
results: report.exploits.map(exploit => ({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit })),
|
|
2762
|
+
};
|
|
2763
|
+
}
|
|
2764
|
+
|
|
2461
2765
|
function runB2EvidenceValidation(state: Record<string, any>): void {
|
|
2462
2766
|
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2463
2767
|
if (tier === "LIGHT") return;
|
|
@@ -2568,9 +2872,70 @@ export function fanoutSkipReason(
|
|
|
2568
2872
|
return null;
|
|
2569
2873
|
}
|
|
2570
2874
|
|
|
2875
|
+
/** One row of the B2 validator's reply, the shape gates/prompts/evidence-validator.md asks for. */
|
|
2876
|
+
export type B2Verdict = { acId: string; verdict: "PASS" | "FAIL"; rawOutput?: unknown; [field: string]: unknown };
|
|
2877
|
+
|
|
2878
|
+
/**
|
|
2879
|
+
* The B2 validator's reply, or why it could not be read (#2265). The one
|
|
2880
|
+
* place its JSON is extracted: two copies are two readers that can disagree,
|
|
2881
|
+
* which is what #2264's first review round found for B1.
|
|
2882
|
+
*/
|
|
2883
|
+
export function parseB2Reply(raw: string): { status: "read"; verdicts: B2Verdict[] } | { status: "not-read"; detail: string } {
|
|
2884
|
+
const json = raw.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
|
|
2885
|
+
if (!json) return { status: "not-read", detail: 'the reply holds no JSON object naming "verdicts"' };
|
|
2886
|
+
let reply: any;
|
|
2887
|
+
try {
|
|
2888
|
+
reply = JSON.parse(json[0]);
|
|
2889
|
+
} catch (e: any) {
|
|
2890
|
+
return { status: "not-read", detail: `the reply's JSON does not parse: ${String(e?.message ?? e).slice(0, 200)}` };
|
|
2891
|
+
}
|
|
2892
|
+
if (!Array.isArray(reply?.verdicts)) {
|
|
2893
|
+
return { status: "not-read", detail: `verdicts is ${JSON.stringify(reply?.verdicts) ?? "absent"}, not a list` };
|
|
2894
|
+
}
|
|
2895
|
+
const bad = reply.verdicts.findIndex((v: any) =>
|
|
2896
|
+
v === null || typeof v !== "object" || typeof v.acId !== "string" || v.acId.trim() === "" ||
|
|
2897
|
+
(v.verdict !== "PASS" && v.verdict !== "FAIL"));
|
|
2898
|
+
if (bad >= 0) {
|
|
2899
|
+
return {
|
|
2900
|
+
status: "not-read",
|
|
2901
|
+
detail: `verdicts[${bad}] has no acId, or a verdict that is not PASS or FAIL: ${String(JSON.stringify(reply.verdicts[bad])).slice(0, 200)}`,
|
|
2902
|
+
};
|
|
2903
|
+
}
|
|
2904
|
+
return { status: "read", verdicts: reply.verdicts as B2Verdict[] };
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
/**
|
|
2908
|
+
* The B2 validator's reply as verify-gate results and the reading verify
|
|
2909
|
+
* records (#2265). Advisory, as it always was: every result is a WARN. A
|
|
2910
|
+
* reply that could not be read is one WARN naming why, and the record says
|
|
2911
|
+
* UNMEASURED. Before, it was nothing: no result, and no record, so the run
|
|
2912
|
+
* looked like one whose every criterion's evidence was sound.
|
|
2913
|
+
*/
|
|
2914
|
+
export function b2ReplyReading(
|
|
2915
|
+
output: string,
|
|
2916
|
+
ts: string,
|
|
2917
|
+
): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
|
|
2918
|
+
const parsed = parseB2Reply(output);
|
|
2919
|
+
if (parsed.status !== "read") {
|
|
2920
|
+
return {
|
|
2921
|
+
agentResult: { ts, status: "UNMEASURED", reason: `reply unreadable: ${parsed.detail}`, source: "B2-evidence-validator-agent" },
|
|
2922
|
+
warns: 1,
|
|
2923
|
+
results: [{ check: "B2-validator: reply unreadable", result: "WARN" as const, detail: parsed.detail }],
|
|
2924
|
+
};
|
|
2925
|
+
}
|
|
2926
|
+
const fails = parsed.verdicts.filter(v => v.verdict === "FAIL");
|
|
2927
|
+
return {
|
|
2928
|
+
agentResult: { ts, verdicts: parsed.verdicts, source: "B2-evidence-validator-agent" },
|
|
2929
|
+
warns: fails.length,
|
|
2930
|
+
results: fails.map(v => ({
|
|
2931
|
+
check: `B2-validator: ${v.acId} evidence gap`,
|
|
2932
|
+
result: "WARN" as const,
|
|
2933
|
+
detail: typeof v.rawOutput === "string" && v.rawOutput !== "" ? v.rawOutput : "threshold not met",
|
|
2934
|
+
})),
|
|
2935
|
+
};
|
|
2936
|
+
}
|
|
2937
|
+
|
|
2571
2938
|
function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
|
2572
|
-
let addedWarns = 0;
|
|
2573
|
-
let agentResult: any = null;
|
|
2574
2939
|
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2575
2940
|
// Recorded rather than returned empty (#126): the consumer allows a LIGHT
|
|
2576
2941
|
// absence, but an artefact that SAYS why is one a reader does not have to
|
|
@@ -2610,33 +2975,16 @@ function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string
|
|
|
2610
2975
|
console.warn(`WARN: B2 Evidence Validator exited with error: ${e.message?.slice(0, 200)}`);
|
|
2611
2976
|
}
|
|
2612
2977
|
|
|
2613
|
-
const
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
verdicts: b2Report.verdicts || [],
|
|
2620
|
-
source: "B2-evidence-validator-agent",
|
|
2621
|
-
};
|
|
2622
|
-
const b2Fails = (b2Report.verdicts || []).filter((v: any) => v.verdict === "FAIL");
|
|
2623
|
-
if (b2Fails.length > 0) {
|
|
2624
|
-
for (const v of b2Fails) {
|
|
2625
|
-
addedWarns++;
|
|
2626
|
-
results.push({ check: `B2-validator: ${v.acId} evidence gap`, result: "WARN" as const, detail: v.rawOutput || "threshold not met" });
|
|
2627
|
-
}
|
|
2628
|
-
console.log(`B2: ${b2Fails.length} evidence gap(s) found — WARN`);
|
|
2629
|
-
} else {
|
|
2630
|
-
console.log(`B2: All evidence validated — ${(b2Report.verdicts || []).length} ACs checked`);
|
|
2631
|
-
}
|
|
2632
|
-
} catch {
|
|
2633
|
-
console.warn("WARN: B2 Evidence Validator JSON parse failed");
|
|
2634
|
-
}
|
|
2978
|
+
const reading = b2ReplyReading(b2Output, new Date().toISOString());
|
|
2979
|
+
results.push(...reading.results);
|
|
2980
|
+
if (reading.agentResult.status === "UNMEASURED") {
|
|
2981
|
+
console.warn(`WARN: B2 Evidence Validator reply UNMEASURED — ${reading.agentResult.reason}`);
|
|
2982
|
+
} else if (reading.warns > 0) {
|
|
2983
|
+
console.log(`B2: ${reading.warns} evidence gap(s) found — WARN`);
|
|
2635
2984
|
} else {
|
|
2636
|
-
console.
|
|
2985
|
+
console.log(`B2: All evidence validated — ${(reading.agentResult.verdicts as unknown[]).length} ACs checked`);
|
|
2637
2986
|
}
|
|
2638
|
-
|
|
2639
|
-
return { agentResult, warns: addedWarns };
|
|
2987
|
+
return { agentResult: reading.agentResult, warns: reading.warns };
|
|
2640
2988
|
}
|
|
2641
2989
|
|
|
2642
2990
|
function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
|
@@ -2677,32 +3025,11 @@ function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, res
|
|
|
2677
3025
|
console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
|
|
2678
3026
|
}
|
|
2679
3027
|
|
|
2680
|
-
const
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
ts: new Date().toISOString(),
|
|
2686
|
-
gameable: b1Report.gameable || 0,
|
|
2687
|
-
approved: b1Report.approved ?? true,
|
|
2688
|
-
exploits: b1Report.exploits || [],
|
|
2689
|
-
source: "B1-adversary-verify-agent",
|
|
2690
|
-
};
|
|
2691
|
-
if (b1Report.gameable > 0) {
|
|
2692
|
-
for (const exploit of b1Report.exploits || []) {
|
|
2693
|
-
addedWarns++;
|
|
2694
|
-
results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2695
|
-
}
|
|
2696
|
-
console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
|
|
2697
|
-
} else {
|
|
2698
|
-
console.log(`B1: All ACs approved at verify — no exploits found`);
|
|
2699
|
-
}
|
|
2700
|
-
} catch {
|
|
2701
|
-
console.warn("WARN: B1 AC Adversary JSON parse failed");
|
|
2702
|
-
}
|
|
2703
|
-
} else {
|
|
2704
|
-
console.warn("WARN: B1 AC Adversary produced no parseable JSON");
|
|
2705
|
-
}
|
|
3028
|
+
const live = liveAdversaryReading(b1Output, new Date().toISOString());
|
|
3029
|
+
agentResult = live.agentResult;
|
|
3030
|
+
addedWarns += live.warns;
|
|
3031
|
+
results.push(...live.results);
|
|
3032
|
+
console.log(`B1: verify agent — ${live.warns} WARN${live.agentResult.status === "UNMEASURED" ? ` (UNMEASURED: ${live.agentResult.reason})` : ""}`);
|
|
2706
3033
|
|
|
2707
3034
|
return { agentResult, warns: addedWarns };
|
|
2708
3035
|
}
|
|
@@ -3198,7 +3525,7 @@ export async function executeGate(input: GateExecutorInput): Promise<GateExecuto
|
|
|
3198
3525
|
const hash = generateHmac(sf, slug, issue, state.projectRoot);
|
|
3199
3526
|
console.log(`HMAC: ${hash}`);
|
|
3200
3527
|
const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
|
|
3201
|
-
console.log(`Ship evidence: ${evidencePath}`);
|
|
3528
|
+
console.log(evidencePath ? `Ship evidence: ${evidencePath}` : "Ship evidence: NOT written — see the fields named above (#2255)");
|
|
3202
3529
|
}
|
|
3203
3530
|
|
|
3204
3531
|
// Verify-specific: convergence
|