rungate 0.8.22 → 0.8.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +316 -68
- package/gates/orchestrator.test.ts +108 -2
- package/gates/orchestrator.ts +25 -5
- package/gates/prompts/prove-reproducer.md +14 -1
- package/gates/prompts.sha256 +1 -1
- package/gates/prove.test.ts +6 -1
- package/gates/schema.ts +21 -1
- package/gates/workflow.test.ts +12 -0
- package/lib/ai-review-suppression.ts +26 -2
- package/lib/artifact-schema.ts +37 -0
- package/lib/cli-commands.ts +10 -0
- package/lib/scaffold/ai-review-step.ts +14 -0
- package/lib/scaffold/intake-workflow.ts +120 -0
- package/lib/scaffold/owned-files.ts +10 -0
- package/lib/scaffold/steps.ts +14 -1
- package/lib/stage-ladder.ts +62 -0
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +6 -2
- package/scripts/intake.ts +101 -0
- package/config/prove-evidence.schema.json +0 -83
package/gates/.gate-salt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
55059bcaab0d2a5e265a824747d8d98b6a512c0b49e3b984deb3bc1db05deba3
|
package/gates/gate-executor.ts
CHANGED
|
@@ -38,7 +38,7 @@ import {
|
|
|
38
38
|
measureEvidencePrevalidation,
|
|
39
39
|
type EvidencePrevalidationReading,
|
|
40
40
|
} from "../lib/evidence-prevalidator";
|
|
41
|
-
import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
|
|
41
|
+
import { AdversaryReportSchema, EvidencePrevalidationSchema, ProveEvidenceSchema, type PrevalidationSource } from "./schema";
|
|
42
42
|
import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
|
|
43
43
|
import { deepMerge } from "../lib/deep-merge";
|
|
44
44
|
import { livenessPlan, probeTarget } from "../lib/dev-liveness";
|
|
@@ -426,6 +426,170 @@ export function comparisonSummaryFor(
|
|
|
426
426
|
return `compared against the ${reading.type} before-state captured at ${reading.capturedAt}; ${counts}`;
|
|
427
427
|
}
|
|
428
428
|
|
|
429
|
+
// The validator moved to lib/ in #2255 so gates/orchestrator.ts can use it
|
|
430
|
+
// without importing this file back; re-exported so it has one definition.
|
|
431
|
+
import { validateArtifactSchema, type ArtifactSchema } from "../lib/artifact-schema";
|
|
432
|
+
export { validateArtifactSchema, type ArtifactSchema };
|
|
433
|
+
|
|
434
|
+
const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
435
|
+
/**
|
|
436
|
+
* The after-evidence types a reproducer may report, and the prompt's list
|
|
437
|
+
* exactly. `none` is B3's to write. `screenshot` is not the reproducer's to
|
|
438
|
+
* claim: it has Bash and Read only (gates/prompts/prove-reproducer.md), so a
|
|
439
|
+
* reported screenshot is a UI verification it could not have made.
|
|
440
|
+
*/
|
|
441
|
+
const REPORTED_AFTER_TYPES = ["text", "api", "data"];
|
|
442
|
+
const AFTER_ENVIRONMENTS = ["local", "prod"];
|
|
443
|
+
|
|
444
|
+
/** What B3 knows about the run that the reproducer does not get to say. */
|
|
445
|
+
export interface ProveEvidenceContext {
|
|
446
|
+
issueNumber: number;
|
|
447
|
+
commitSHA: string;
|
|
448
|
+
capturedAt: string;
|
|
449
|
+
beforeReading: BeforeStateReading;
|
|
450
|
+
/** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
|
|
451
|
+
source?: string;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
type AfterEvidence = {
|
|
455
|
+
type: string; path: string | null; capturedAt: string; environment: string; description: string;
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
/** The reproducer's `afterEvidence`, or why there is none. Never repaired into shape. */
|
|
459
|
+
function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
|
|
460
|
+
if (raw === undefined || raw === null) return { ok: false, reason: "the reproducer reported no afterEvidence" };
|
|
461
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
462
|
+
return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
|
|
463
|
+
}
|
|
464
|
+
const a = raw as Record<string, unknown>;
|
|
465
|
+
if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
|
|
466
|
+
return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
|
|
467
|
+
}
|
|
468
|
+
if (typeof a.type !== "string" || !REPORTED_AFTER_TYPES.includes(a.type)) {
|
|
469
|
+
return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one of ${REPORTED_AFTER_TYPES.join("|")}` };
|
|
470
|
+
}
|
|
471
|
+
if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
|
|
472
|
+
return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
|
|
473
|
+
}
|
|
474
|
+
if (typeof a.description !== "string" || a.description.trim() === "") {
|
|
475
|
+
return { ok: false, reason: "afterEvidence.description is empty — it must say what the after-state showed" };
|
|
476
|
+
}
|
|
477
|
+
return {
|
|
478
|
+
ok: true,
|
|
479
|
+
value: {
|
|
480
|
+
type: a.type, path: typeof a.path === "string" ? a.path : null,
|
|
481
|
+
capturedAt: a.capturedAt, environment: a.environment, description: a.description,
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Build `prove-evidence.json` from the reproducer's report (#1559, Jason's
|
|
488
|
+
* option (a) of 2026-11-15).
|
|
489
|
+
*
|
|
490
|
+
* B3 used to write `{issueNumber, verdict, commitSHA, capturedAt,
|
|
491
|
+
* criteriaResults, reproduced, evidence, source}`, which lacks the
|
|
492
|
+
* `beforeEvidence`, `afterEvidence` and `comparisonSummary` that
|
|
493
|
+
* `ProveEvidenceSchema` requires. The before-evidence and the comparison are
|
|
494
|
+
* B3's own readings (#1540), so a reproducer cannot overwrite them. The
|
|
495
|
+
* after-evidence is the reproducer's report of the environment it tested and
|
|
496
|
+
* what it saw. When that report is missing or malformed, the record is
|
|
497
|
+
* `type: "none"` and a PROVEN becomes INCONCLUSIVE, because a proof with no
|
|
498
|
+
* observed after-state is not a proof.
|
|
499
|
+
*
|
|
500
|
+
* The `none` record says `environment: "local"` because the schema requires
|
|
501
|
+
* a value. `local` is the one that satisfies no STANDARD+ prod requirement
|
|
502
|
+
* (gates/prove.test.ts), so a missing after-evidence can never pass that check.
|
|
503
|
+
*/
|
|
504
|
+
export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
|
|
505
|
+
const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
|
|
506
|
+
const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
|
|
507
|
+
const after = readReportedAfterEvidence(raw?.afterEvidence);
|
|
508
|
+
|
|
509
|
+
let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
|
|
510
|
+
let verdictDowngrade: { reported: unknown; reason: string } | undefined;
|
|
511
|
+
if (verdict !== raw?.verdict) {
|
|
512
|
+
verdictDowngrade = {
|
|
513
|
+
reported: raw?.verdict,
|
|
514
|
+
reason: `verdict ${JSON.stringify(raw?.verdict)} is not one of ${PROVE_VERDICTS.join("|")}`,
|
|
515
|
+
};
|
|
516
|
+
} else if (verdict === "PROVEN" && !after.ok) {
|
|
517
|
+
verdict = "INCONCLUSIVE";
|
|
518
|
+
verdictDowngrade = { reported: "PROVEN", reason: `PROVEN with no after-evidence: ${after.reason}` };
|
|
519
|
+
} else if (verdict === "PROVEN" && passed === 0) {
|
|
520
|
+
// A proof that verified no criterion is not a proof: an empty list, or
|
|
521
|
+
// nothing but SKIPs, is what a reproducer with no way to check says.
|
|
522
|
+
verdict = "INCONCLUSIVE";
|
|
523
|
+
verdictDowngrade = {
|
|
524
|
+
reported: "PROVEN",
|
|
525
|
+
reason: `PROVEN with no criterion verified: ${criteriaResults.length} criteriaResults row(s), none PASS`,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
const afterEvidence: AfterEvidence = after.ok ? after.value : {
|
|
530
|
+
type: "none", path: null, capturedAt: ctx.capturedAt, environment: "local",
|
|
531
|
+
description: `no after-evidence: ${after.reason}`,
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
return {
|
|
535
|
+
issueNumber: ctx.issueNumber,
|
|
536
|
+
verdict,
|
|
537
|
+
...(verdictDowngrade ? { verdictDowngrade } : {}),
|
|
538
|
+
commitSHA: ctx.commitSHA,
|
|
539
|
+
capturedAt: ctx.capturedAt,
|
|
540
|
+
beforeEvidence: beforeEvidenceFor(ctx.beforeReading),
|
|
541
|
+
afterEvidence,
|
|
542
|
+
comparisonSummary: comparisonSummaryFor(ctx.beforeReading, passed, criteriaResults.length),
|
|
543
|
+
reproduced: raw?.reproduced === true,
|
|
544
|
+
criteriaResults,
|
|
545
|
+
evidence: Array.isArray(raw?.evidence) ? raw.evidence : [],
|
|
546
|
+
source: ctx.source ?? "B3-prove-reproducer",
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
export type ProveEvidenceRecord =
|
|
551
|
+
| { written: true; artifact: ReturnType<typeof buildProveEvidence> }
|
|
552
|
+
| { written: false; reason: string };
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Build, validate and write `prove-evidence.json`, and say what happened
|
|
556
|
+
* (#1559). Never throws.
|
|
557
|
+
*
|
|
558
|
+
* B3 has always degraded to a warning when it could not record the artifact
|
|
559
|
+
* (`prove.test.ts` writes its own), so a failing `git rev-parse` or a write
|
|
560
|
+
* into a missing directory must not crash the prove gate. Every outcome that
|
|
561
|
+
* is not a written file says why.
|
|
562
|
+
*/
|
|
563
|
+
export function recordProveEvidence(
|
|
564
|
+
raw: Record<string, any>,
|
|
565
|
+
opts: {
|
|
566
|
+
issueNumber: number;
|
|
567
|
+
workDir: string;
|
|
568
|
+
beforeReading: BeforeStateReading;
|
|
569
|
+
readCommit: () => string;
|
|
570
|
+
schema?: ArtifactSchema;
|
|
571
|
+
source?: string;
|
|
572
|
+
},
|
|
573
|
+
): ProveEvidenceRecord {
|
|
574
|
+
try {
|
|
575
|
+
const artifact = buildProveEvidence(raw, {
|
|
576
|
+
issueNumber: opts.issueNumber,
|
|
577
|
+
commitSHA: opts.readCommit(),
|
|
578
|
+
capturedAt: new Date().toISOString(),
|
|
579
|
+
beforeReading: opts.beforeReading,
|
|
580
|
+
source: opts.source,
|
|
581
|
+
});
|
|
582
|
+
const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
|
|
583
|
+
if (violations.length > 0) {
|
|
584
|
+
return { written: false, reason: `it fails ProveEvidenceSchema (SC-2770): ${violations.join("; ")}` };
|
|
585
|
+
}
|
|
586
|
+
writeFileSync(join(opts.workDir, "prove-evidence.json"), JSON.stringify(artifact, null, 2));
|
|
587
|
+
return { written: true, artifact };
|
|
588
|
+
} catch (e: any) {
|
|
589
|
+
return { written: false, reason: `could not record it: ${String(e?.message ?? e).slice(0, 200)}` };
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
429
593
|
/** What a gate run does about the tests that existed when it last looked. */
|
|
430
594
|
export type CensusVerdict =
|
|
431
595
|
| { kind: "unmeasured"; reason: string }
|
|
@@ -1554,26 +1718,33 @@ async function runProveReproducer(state: Record<string, any>, issue: number, iss
|
|
|
1554
1718
|
}
|
|
1555
1719
|
|
|
1556
1720
|
const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
|
|
1721
|
+
let parsed: Record<string, any> | null = null;
|
|
1557
1722
|
if (jsonMatch) {
|
|
1558
1723
|
try {
|
|
1559
|
-
|
|
1560
|
-
const proveEvidence = {
|
|
1561
|
-
issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
|
|
1562
|
-
commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1563
|
-
capturedAt: new Date().toISOString(),
|
|
1564
|
-
criteriaResults: parsed.criteriaResults || [],
|
|
1565
|
-
reproduced: parsed.reproduced ?? false,
|
|
1566
|
-
evidence: parsed.evidence || [],
|
|
1567
|
-
source: "B3-prove-reproducer",
|
|
1568
|
-
};
|
|
1569
|
-
writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
|
|
1570
|
-
console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
|
|
1724
|
+
parsed = JSON.parse(jsonMatch[0]);
|
|
1571
1725
|
} catch {
|
|
1572
1726
|
console.warn("WARN: B3 reproducer JSON parse failed");
|
|
1573
1727
|
}
|
|
1574
1728
|
} else {
|
|
1575
1729
|
console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
|
|
1576
1730
|
}
|
|
1731
|
+
if (parsed && typeof parsed === "object") {
|
|
1732
|
+
// #1559: the artifact is built in one place and checked against its one
|
|
1733
|
+
// contract before it is written. An artifact the schema refuses is not
|
|
1734
|
+
// written, and the fields are named; nothing here can crash the gate.
|
|
1735
|
+
const record = recordProveEvidence(parsed, {
|
|
1736
|
+
issueNumber: issue, workDir, beforeReading,
|
|
1737
|
+
readCommit: () => execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1738
|
+
});
|
|
1739
|
+
if (!record.written) {
|
|
1740
|
+
console.warn(`WARN: B3 prove-evidence.json NOT written — ${record.reason}; prove.test.ts will generate evidence`);
|
|
1741
|
+
} else {
|
|
1742
|
+
console.log(`B3: prove-evidence.json written — verdict: ${record.artifact.verdict}`);
|
|
1743
|
+
if (record.artifact.verdictDowngrade) {
|
|
1744
|
+
console.warn(`WARN: B3 reproducer reported ${JSON.stringify(record.artifact.verdictDowngrade.reported)}, recorded INCONCLUSIVE — ${record.artifact.verdictDowngrade.reason}`);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1577
1748
|
}
|
|
1578
1749
|
}
|
|
1579
1750
|
|
|
@@ -2318,7 +2489,7 @@ function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, ha
|
|
|
2318
2489
|
writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
|
|
2319
2490
|
const { spawn } = require("child_process");
|
|
2320
2491
|
const child = spawn("sh", ["-c",
|
|
2321
|
-
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format
|
|
2492
|
+
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format text --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
|
|
2322
2493
|
], { detached: true, stdio: "ignore", cwd: harnessRoot });
|
|
2323
2494
|
child.unref();
|
|
2324
2495
|
console.log("B1: AC Adversary spawned in background — verify gate will check result");
|
|
@@ -2424,40 +2595,138 @@ function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, resu
|
|
|
2424
2595
|
results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
|
|
2425
2596
|
console.log("B1: adversary report not yet available — verify FAIL");
|
|
2426
2597
|
} else {
|
|
2598
|
+
let raw: string | null = null;
|
|
2427
2599
|
try {
|
|
2428
|
-
|
|
2429
|
-
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2430
|
-
if (jsonMatch) {
|
|
2431
|
-
const report = JSON.parse(jsonMatch[0]);
|
|
2432
|
-
console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
|
|
2433
|
-
if (report.gameable > 0) {
|
|
2434
|
-
let outcomeCount = 0;
|
|
2435
|
-
for (const exploit of report.exploits || []) {
|
|
2436
|
-
const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
|
|
2437
|
-
const isOutcome = ac?.type === "OUTCOME";
|
|
2438
|
-
if (isOutcome) {
|
|
2439
|
-
addedFails++;
|
|
2440
|
-
outcomeCount++;
|
|
2441
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2442
|
-
} else {
|
|
2443
|
-
addedWarns++;
|
|
2444
|
-
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2445
|
-
}
|
|
2446
|
-
}
|
|
2447
|
-
if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
|
|
2448
|
-
const codeCount = (report.gameable || 0) - outcomeCount;
|
|
2449
|
-
if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
|
|
2450
|
-
}
|
|
2451
|
-
} else {
|
|
2452
|
-
console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
|
|
2453
|
-
}
|
|
2600
|
+
raw = readFileSync(reportPath, "utf-8");
|
|
2454
2601
|
} catch (e: any) {
|
|
2455
|
-
|
|
2602
|
+
addedFails++;
|
|
2603
|
+
results.push({ check: "B1-adversary: report unreadable", result: "FAIL" as const, detail: `adversary-report.json could not be read: ${e.message?.slice(0, 200)}` });
|
|
2604
|
+
}
|
|
2605
|
+
if (raw !== null) {
|
|
2606
|
+
const reading = adversaryReportReading(raw, state.acs || []);
|
|
2607
|
+
addedFails += reading.fails;
|
|
2608
|
+
addedWarns += reading.warns;
|
|
2609
|
+
results.push(...reading.results);
|
|
2610
|
+
console.log(`B1: adversary report — ${reading.fails} FAIL, ${reading.warns} WARN`);
|
|
2456
2611
|
}
|
|
2457
2612
|
}
|
|
2458
2613
|
return { fails: addedFails, warns: addedWarns };
|
|
2459
2614
|
}
|
|
2460
2615
|
|
|
2616
|
+
/**
|
|
2617
|
+
* Read B1's adversary report into gate results (#2259, SC-2770).
|
|
2618
|
+
*
|
|
2619
|
+
* The report is raw output from the model B1 spawns. It was read with a
|
|
2620
|
+
* regex and `JSON.parse` and never checked against `AdversaryReportSchema`,
|
|
2621
|
+
* so `"gameable": "2"` with no exploits, or `exploits` that was not a list,
|
|
2622
|
+
* produced no result at all: the reading of a clean report. A report with no
|
|
2623
|
+
* JSON only printed a warning, so the gate passed on a report it never read.
|
|
2624
|
+
* Now an unreadable report and a report the schema refuses are each one FAIL
|
|
2625
|
+
* naming why, the same as a missing report. Exported so it runs on fixtures;
|
|
2626
|
+
* the schema is a parameter so a test can hand it a stricter copy.
|
|
2627
|
+
*/
|
|
2628
|
+
export function adversaryReportReading(
|
|
2629
|
+
raw: string,
|
|
2630
|
+
acs: ReadonlyArray<{ id?: unknown; type?: unknown }>,
|
|
2631
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2632
|
+
): { fails: number; warns: number; results: GateResult[] } {
|
|
2633
|
+
const parsed = parseAdversaryReport(raw, schema);
|
|
2634
|
+
if (parsed.status !== "read") {
|
|
2635
|
+
return { fails: 1, warns: 0, results: [{ check: `B1-adversary: ${parsed.check}`, result: "FAIL" as const, detail: parsed.detail }] };
|
|
2636
|
+
}
|
|
2637
|
+
let fails = 0;
|
|
2638
|
+
let warns = 0;
|
|
2639
|
+
const results: GateResult[] = [];
|
|
2640
|
+
// Every listed exploit is read whatever `gameable` says: one dropped because
|
|
2641
|
+
// the count was 0 is the same silent pass in the other direction.
|
|
2642
|
+
for (const exploit of parsed.report.exploits) {
|
|
2643
|
+
const ac = acs.find(a => a.id === exploit.acId);
|
|
2644
|
+
if (ac?.type === "OUTCOME") {
|
|
2645
|
+
fails++;
|
|
2646
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2647
|
+
} else {
|
|
2648
|
+
warns++;
|
|
2649
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
return { fails, warns, results };
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
type AdversaryReport = {
|
|
2656
|
+
gameable: number;
|
|
2657
|
+
approved: boolean;
|
|
2658
|
+
exploits: Array<{ acId: string; exploit: string; recommendation: string }>;
|
|
2659
|
+
};
|
|
2660
|
+
|
|
2661
|
+
/**
|
|
2662
|
+
* THE ONE READ of a B1 adversary report (#2259), for the report on disk and
|
|
2663
|
+
* for the live re-run at verify alike. Both used to spell their own regex and
|
|
2664
|
+
* `JSON.parse` with no schema, and review round 1 found the second copy still
|
|
2665
|
+
* swallowing a malformed reply after the first was fixed. Two readers of one
|
|
2666
|
+
* shape are one guard with a dead twin; test/adversary-report-schema.test.ts
|
|
2667
|
+
* asserts the extraction appears once in this file.
|
|
2668
|
+
*
|
|
2669
|
+
* Not "read" when the output holds no JSON naming `gameable`, when that JSON
|
|
2670
|
+
* does not parse, when the schema refuses it, or when it claims more gameable
|
|
2671
|
+
* criteria than it lists exploits for (`{gameable: 3, exploits: []}` is
|
|
2672
|
+
* schema-valid and used to read as clean). More exploits than the count is
|
|
2673
|
+
* not a contradiction: two ways to game one criterion.
|
|
2674
|
+
*/
|
|
2675
|
+
export function parseAdversaryReport(
|
|
2676
|
+
raw: string,
|
|
2677
|
+
schema: ArtifactSchema = AdversaryReportSchema,
|
|
2678
|
+
): { status: "read"; report: AdversaryReport } | { status: "not-read"; check: string; detail: string } {
|
|
2679
|
+
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2680
|
+
if (!jsonMatch) {
|
|
2681
|
+
return { status: "not-read", check: "report unreadable", detail: "the output holds no JSON object naming \"gameable\"" };
|
|
2682
|
+
}
|
|
2683
|
+
let report: any;
|
|
2684
|
+
try {
|
|
2685
|
+
report = JSON.parse(jsonMatch[0]);
|
|
2686
|
+
} catch (e: any) {
|
|
2687
|
+
return { status: "not-read", check: "report unreadable", detail: `the report's JSON does not parse: ${e.message?.slice(0, 200)}` };
|
|
2688
|
+
}
|
|
2689
|
+
const violations = validateArtifactSchema(report, schema);
|
|
2690
|
+
if (violations.length > 0) {
|
|
2691
|
+
return { status: "not-read", check: "report fails AdversaryReportSchema", detail: violations.join("; ") };
|
|
2692
|
+
}
|
|
2693
|
+
if (report.gameable > report.exploits.length) {
|
|
2694
|
+
return {
|
|
2695
|
+
status: "not-read",
|
|
2696
|
+
check: "report contradicts itself",
|
|
2697
|
+
detail: `gameable ${report.gameable} but ${report.exploits.length} exploit(s) listed, so which criteria are gameable is unknown`,
|
|
2698
|
+
};
|
|
2699
|
+
}
|
|
2700
|
+
return { status: "read", report };
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
/**
|
|
2704
|
+
* The live B1 re-run at verify, read (#2259 review round 1). Advisory, as it
|
|
2705
|
+
* always was: every finding is a WARN. What changes is that a reply that could
|
|
2706
|
+
* not be read is a WARN naming why, and `agentResult` records it as
|
|
2707
|
+
* UNMEASURED, where it used to be `null` and the verify gate's record of the
|
|
2708
|
+
* adversary was never written at all.
|
|
2709
|
+
*/
|
|
2710
|
+
export function liveAdversaryReading(
|
|
2711
|
+
output: string,
|
|
2712
|
+
ts: string,
|
|
2713
|
+
): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
|
|
2714
|
+
const parsed = parseAdversaryReport(output);
|
|
2715
|
+
if (parsed.status !== "read") {
|
|
2716
|
+
return {
|
|
2717
|
+
agentResult: { ts, status: "UNMEASURED", reason: `${parsed.check}: ${parsed.detail}`, source: "B1-adversary-verify-agent" },
|
|
2718
|
+
warns: 1,
|
|
2719
|
+
results: [{ check: `B1-adversary-verify: ${parsed.check}`, result: "WARN" as const, detail: parsed.detail }],
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
const { report } = parsed;
|
|
2723
|
+
return {
|
|
2724
|
+
agentResult: { ts, gameable: report.gameable, approved: report.approved, exploits: report.exploits, source: "B1-adversary-verify-agent" },
|
|
2725
|
+
warns: report.exploits.length,
|
|
2726
|
+
results: report.exploits.map(exploit => ({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit })),
|
|
2727
|
+
};
|
|
2728
|
+
}
|
|
2729
|
+
|
|
2461
2730
|
function runB2EvidenceValidation(state: Record<string, any>): void {
|
|
2462
2731
|
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2463
2732
|
if (tier === "LIGHT") return;
|
|
@@ -2677,32 +2946,11 @@ function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, res
|
|
|
2677
2946
|
console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
|
|
2678
2947
|
}
|
|
2679
2948
|
|
|
2680
|
-
const
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
ts: new Date().toISOString(),
|
|
2686
|
-
gameable: b1Report.gameable || 0,
|
|
2687
|
-
approved: b1Report.approved ?? true,
|
|
2688
|
-
exploits: b1Report.exploits || [],
|
|
2689
|
-
source: "B1-adversary-verify-agent",
|
|
2690
|
-
};
|
|
2691
|
-
if (b1Report.gameable > 0) {
|
|
2692
|
-
for (const exploit of b1Report.exploits || []) {
|
|
2693
|
-
addedWarns++;
|
|
2694
|
-
results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2695
|
-
}
|
|
2696
|
-
console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
|
|
2697
|
-
} else {
|
|
2698
|
-
console.log(`B1: All ACs approved at verify — no exploits found`);
|
|
2699
|
-
}
|
|
2700
|
-
} catch {
|
|
2701
|
-
console.warn("WARN: B1 AC Adversary JSON parse failed");
|
|
2702
|
-
}
|
|
2703
|
-
} else {
|
|
2704
|
-
console.warn("WARN: B1 AC Adversary produced no parseable JSON");
|
|
2705
|
-
}
|
|
2949
|
+
const live = liveAdversaryReading(b1Output, new Date().toISOString());
|
|
2950
|
+
agentResult = live.agentResult;
|
|
2951
|
+
addedWarns += live.warns;
|
|
2952
|
+
results.push(...live.results);
|
|
2953
|
+
console.log(`B1: verify agent — ${live.warns} WARN${live.agentResult.status === "UNMEASURED" ? ` (UNMEASURED: ${live.agentResult.reason})` : ""}`);
|
|
2706
2954
|
|
|
2707
2955
|
return { agentResult, warns: addedWarns };
|
|
2708
2956
|
}
|
|
@@ -3198,7 +3446,7 @@ export async function executeGate(input: GateExecutorInput): Promise<GateExecuto
|
|
|
3198
3446
|
const hash = generateHmac(sf, slug, issue, state.projectRoot);
|
|
3199
3447
|
console.log(`HMAC: ${hash}`);
|
|
3200
3448
|
const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
|
|
3201
|
-
console.log(`Ship evidence: ${evidencePath}`);
|
|
3449
|
+
console.log(evidencePath ? `Ship evidence: ${evidencePath}` : "Ship evidence: NOT written — see the fields named above (#2255)");
|
|
3202
3450
|
}
|
|
3203
3451
|
|
|
3204
3452
|
// Verify-specific: convergence
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
|
2
2
|
import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync, statSync } from "fs";
|
|
3
3
|
import { join } from "path";
|
|
4
|
+
import { z } from "zod";
|
|
4
5
|
import { gateSaltPath } from "../lib/paths";
|
|
6
|
+
import { validateArtifactSchema } from "../lib/artifact-schema";
|
|
7
|
+
import { ShipEvidenceSchema } from "./schema";
|
|
5
8
|
import {
|
|
6
9
|
writeGateResult,
|
|
7
10
|
generateHmac,
|
|
@@ -419,8 +422,10 @@ describe("generateShipEvidence", () => {
|
|
|
419
422
|
|
|
420
423
|
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
421
424
|
|
|
422
|
-
|
|
423
|
-
|
|
425
|
+
// null means the contract refused the artifact (#2255).
|
|
426
|
+
expect(path).not.toBeNull();
|
|
427
|
+
expect(existsSync(path!)).toBe(true);
|
|
428
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
424
429
|
expect(evidence.contractVersion).toBe("1.0");
|
|
425
430
|
expect(evidence.issueNumber).toBe(432);
|
|
426
431
|
expect(evidence.gateOut.status).toBe("PASS");
|
|
@@ -429,6 +434,107 @@ describe("generateShipEvidence", () => {
|
|
|
429
434
|
expect(evidence.capturedAt).toBeTruthy();
|
|
430
435
|
expect(evidence.iterationCount).toBeGreaterThanOrEqual(1);
|
|
431
436
|
expect(evidence.sizing.predicted).toBe("M");
|
|
437
|
+
// #2255 / SC-2770: the artifact meets its declared contract, not only the
|
|
438
|
+
// eight fields this case happens to name.
|
|
439
|
+
expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
test("#2255 / SC-2770: a schema with one required field more than the writer emits refuses the artifact", () => {
|
|
443
|
+
writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP" }), null, 2));
|
|
444
|
+
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
445
|
+
expect(path).not.toBeNull();
|
|
446
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
447
|
+
const stricter = ShipEvidenceSchema.extend({ reviewedBy: z.string() });
|
|
448
|
+
expect(validateArtifactSchema(evidence, stricter).join("\n")).toContain("reviewedBy");
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
test("#2255: an artifact the contract refuses is not written, and the field is named", () => {
|
|
452
|
+
// A verdict outside PASS|FAIL|SKIP is the live way the writer can emit
|
|
453
|
+
// what ShipEvidenceSchema refuses: `result: ac.verdict || "SKIP"`.
|
|
454
|
+
writeFileSync(SF, JSON.stringify(makeState({
|
|
455
|
+
phase: "SHIP",
|
|
456
|
+
acs: [{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }],
|
|
457
|
+
}), null, 2));
|
|
458
|
+
const errors: string[] = [];
|
|
459
|
+
const original = console.error;
|
|
460
|
+
console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); };
|
|
461
|
+
let path: string | null;
|
|
462
|
+
try {
|
|
463
|
+
path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
464
|
+
} finally {
|
|
465
|
+
console.error = original;
|
|
466
|
+
}
|
|
467
|
+
expect(path).toBeNull();
|
|
468
|
+
expect(existsSync(join(TMP_DIR, "ship-evidence.json"))).toBe(false);
|
|
469
|
+
expect(errors.join("\n")).toContain("gateOut.evidence.0.result");
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
test("#2255 review round 1: an OUTCOME criterion still PENDING at ship is written — a legitimate ship-time state", () => {
|
|
473
|
+
// gates/workflow.test.ts: "OUTCOME ACs may be PENDING at ship time — they
|
|
474
|
+
// require Quinn/prove for verification", and enrichment writes the literal
|
|
475
|
+
// "PENDING". The contract refused it, so validation would have dropped the
|
|
476
|
+
// evidence on every normal run with an OUTCOME criterion outstanding.
|
|
477
|
+
writeFileSync(SF, JSON.stringify(makeState({
|
|
478
|
+
phase: "SHIP",
|
|
479
|
+
acs: [{ id: "SC-1", statement: "s", type: "OUTCOME", verdict: "PENDING", evidence: null }],
|
|
480
|
+
}), null, 2));
|
|
481
|
+
const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
|
|
482
|
+
expect(path).not.toBeNull();
|
|
483
|
+
const evidence = JSON.parse(readFileSync(path!, "utf-8"));
|
|
484
|
+
expect(evidence.gateOut.evidence[0].result).toBe("PENDING");
|
|
485
|
+
expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
|
|
486
|
+
});
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
describe("#2255 review round 1: a refused artifact is never reported to the issue as shipped", () => {
|
|
490
|
+
// A loopback stub stands in for the GitHub API (lib/github.ts allows exactly
|
|
491
|
+
// GitHub or loopback). `fixture/repo` makes verifiedPush refuse rather than
|
|
492
|
+
// push from this checkout (#430). The control case is what keeps the
|
|
493
|
+
// refusal's "no requests" from being vacuous: the same stub, a valid
|
|
494
|
+
// artifact, and the label and comment arrive.
|
|
495
|
+
async function requestsDuring(acs: Array<Record<string, unknown>>): Promise<string[]> {
|
|
496
|
+
const seen: string[] = [];
|
|
497
|
+
const server = Bun.serve({
|
|
498
|
+
port: 0,
|
|
499
|
+
hostname: "127.0.0.1",
|
|
500
|
+
fetch(req) {
|
|
501
|
+
seen.push(`${req.method} ${new URL(req.url).pathname}`);
|
|
502
|
+
return Response.json({});
|
|
503
|
+
},
|
|
504
|
+
});
|
|
505
|
+
try {
|
|
506
|
+
writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP", acs }), null, 2));
|
|
507
|
+
// A child process, so the credential is SUPPLIED rather than read: the
|
|
508
|
+
// tree resolves it in one place (resolveGitHubToken), and nothing here
|
|
509
|
+
// names it as a read. Async, because this process serves the stub, and
|
|
510
|
+
// a blocking spawn would stall the server the child is calling. The
|
|
511
|
+
// child's pending requests keep it alive until they have landed.
|
|
512
|
+
const child = Bun.spawn(
|
|
513
|
+
["bun", "-e", `import { generateShipEvidence } from ${JSON.stringify(join(import.meta.dir, "orchestrator.ts"))};` +
|
|
514
|
+
`generateShipEvidence(${JSON.stringify(SF)}, ${JSON.stringify(TMP_DIR)}, 432, "fixture/repo", 1, 0, 0);`],
|
|
515
|
+
{
|
|
516
|
+
cwd: join(import.meta.dir, ".."),
|
|
517
|
+
env: { ...process.env, GH_TOKEN: "test-token-not-real", GITHUB_API_URL: `http://127.0.0.1:${server.port}` },
|
|
518
|
+
stdout: "ignore",
|
|
519
|
+
stderr: "ignore",
|
|
520
|
+
},
|
|
521
|
+
);
|
|
522
|
+
await child.exited;
|
|
523
|
+
} finally {
|
|
524
|
+
server.stop(true);
|
|
525
|
+
}
|
|
526
|
+
return seen;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
test("control: a valid artifact labels the issue shipped and posts the comment", async () => {
|
|
530
|
+
const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "PASS", evidence: { content: "ok" } }]);
|
|
531
|
+
expect(seen.some(r => r.endsWith("/issues/432/labels"))).toBe(true);
|
|
532
|
+
expect(seen.some(r => r.endsWith("/issues/432/comments"))).toBe(true);
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
test("a refused artifact sends nothing: no shipped label, no Ship Complete comment", async () => {
|
|
536
|
+
const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }]);
|
|
537
|
+
expect(seen).toEqual([]);
|
|
432
538
|
});
|
|
433
539
|
});
|
|
434
540
|
|
package/gates/orchestrator.ts
CHANGED
|
@@ -2,7 +2,8 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync, statSync
|
|
|
2
2
|
import { join, dirname } from "path";
|
|
3
3
|
import { createHash, createHmac, randomBytes } from "crypto";
|
|
4
4
|
import { execSync } from "child_process";
|
|
5
|
-
import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS } from "./schema";
|
|
5
|
+
import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS, ShipEvidenceSchema } from "./schema";
|
|
6
|
+
import { validateArtifactSchema } from "../lib/artifact-schema";
|
|
6
7
|
import { ZodError } from "zod";
|
|
7
8
|
import { createGitHubClient, addLabels, addComment, type GitHubClient } from "../lib/github";
|
|
8
9
|
import { computeACHash } from "../lib/workflow-security";
|
|
@@ -384,7 +385,7 @@ export function generateShipEvidence(
|
|
|
384
385
|
passes: number,
|
|
385
386
|
fails: number,
|
|
386
387
|
warns: number,
|
|
387
|
-
): string {
|
|
388
|
+
): string | null {
|
|
388
389
|
const state = readState(sf);
|
|
389
390
|
const sha = gitSha();
|
|
390
391
|
const ts = isoNow();
|
|
@@ -483,12 +484,31 @@ export function generateShipEvidence(
|
|
|
483
484
|
...(pushOutcome ? { push: pushOutcome, pushLeftWorkLocal: leftWorkLocal(pushOutcome) } : {}),
|
|
484
485
|
};
|
|
485
486
|
|
|
487
|
+
// #2255 / SC-2770: checked against its one contract before it is written,
|
|
488
|
+
// as prove-evidence.json is (#1559). An artifact the schema refuses is not
|
|
489
|
+
// written, and the fields are named. The gate's verdict is untouched: by
|
|
490
|
+
// here the ship gate has already passed and the push has happened, and the
|
|
491
|
+
// missing artifact is the visible consequence.
|
|
486
492
|
const evidencePath = join(workDir, "ship-evidence.json");
|
|
487
|
-
|
|
493
|
+
const violations = validateArtifactSchema(evidence, ShipEvidenceSchema);
|
|
494
|
+
if (violations.length > 0) {
|
|
495
|
+
console.error("ship gate: ship-evidence.json NOT written — it fails ShipEvidenceSchema (SC-2770):");
|
|
496
|
+
for (const v of violations) console.error(` - ${v}`);
|
|
497
|
+
} else {
|
|
498
|
+
writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
|
|
499
|
+
}
|
|
488
500
|
|
|
489
501
|
// GitHub: add shipped label + post comment via Octokit (best-effort, fire-and-forget)
|
|
490
502
|
// Uses POST additive label strategy (D-5) — never PUT/replace
|
|
491
|
-
|
|
503
|
+
//
|
|
504
|
+
// Not when the contract refused the artifact (#2255): labelling the issue
|
|
505
|
+
// shipped and posting "Ship Complete" beside a refusal is a public success
|
|
506
|
+
// report the run's own record contradicts, and a label is harder to take
|
|
507
|
+
// back than to withhold.
|
|
508
|
+
if (violations.length > 0 && issueRepo && issue > 0) {
|
|
509
|
+
console.error(`ship gate: #${issue} NOT labelled shipped and no Ship Complete comment posted — the evidence was not recorded`);
|
|
510
|
+
}
|
|
511
|
+
if (violations.length === 0 && issueRepo && issue > 0) {
|
|
492
512
|
try {
|
|
493
513
|
const github = createGitHubClient();
|
|
494
514
|
addLabels(github, issueRepo, issue, ["shipped"]).catch(() => {});
|
|
@@ -512,7 +532,7 @@ export function generateShipEvidence(
|
|
|
512
532
|
} catch {}
|
|
513
533
|
}
|
|
514
534
|
|
|
515
|
-
return evidencePath;
|
|
535
|
+
return violations.length > 0 ? null : evidencePath;
|
|
516
536
|
}
|
|
517
537
|
|
|
518
538
|
const GATE_PENDING_PATH = join(
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
doc-type: reference
|
|
3
3
|
status: active
|
|
4
4
|
owner: jason
|
|
5
|
-
updated: 2026-
|
|
5
|
+
updated: 2026-11-15
|
|
6
6
|
---
|
|
7
7
|
|
|
8
8
|
You are a prove reproducer. Your objective: reproduce the bug described
|
|
@@ -44,9 +44,22 @@ Output JSON:
|
|
|
44
44
|
"evidence": [
|
|
45
45
|
{"type": "screenshot|api-response|console", "description": "what this shows"}
|
|
46
46
|
],
|
|
47
|
+
"afterEvidence": {
|
|
48
|
+
"type": "text|api|data",
|
|
49
|
+
"path": null,
|
|
50
|
+
"capturedAt": "ISO-8601 time you observed the after-state",
|
|
51
|
+
"environment": "local|prod",
|
|
52
|
+
"description": "what the after-state showed"
|
|
53
|
+
},
|
|
47
54
|
"gaps": []
|
|
48
55
|
}
|
|
49
56
|
|
|
57
|
+
`afterEvidence` is the after-state you observed yourself. `environment` is
|
|
58
|
+
where you observed it: `local` for this checkout or a dev server, `prod` for
|
|
59
|
+
the production URL. If you observed no after-state, leave `afterEvidence`
|
|
60
|
+
out. Never fill it in to make the record complete: a PROVEN without it is
|
|
61
|
+
recorded as INCONCLUSIVE.
|
|
62
|
+
|
|
50
63
|
Verdict rules:
|
|
51
64
|
- PROVEN: bug was reproduced in before-state AND fix verified in after-state
|
|
52
65
|
- UNPROVEN: fix does not resolve the issue (evidence shows problem persists)
|