rungate 0.8.24 → 0.8.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/gates/.gate-salt CHANGED
@@ -1 +1 @@
1
- 55059bcaab0d2a5e265a824747d8d98b6a512c0b49e3b984deb3bc1db05deba3
1
+ f7edfff5c8424ba64c95ad1a592d7b40c40c3a78197e968739a5d468cd671467
@@ -1,5 +1,5 @@
1
1
  import { execSync, execFileSync } from "child_process";
2
- import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync } from "fs";
2
+ import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync, statSync } from "fs";
3
3
  import { join } from "path";
4
4
  import { splitGateRuns } from "../lib/gate-run-split";
5
5
  import { gateSaltPath, workDir as slugWorkDir } from "../lib/paths";
@@ -433,12 +433,24 @@ export { validateArtifactSchema, type ArtifactSchema };
433
433
 
434
434
  const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
435
435
  /**
436
- * The after-evidence types a reproducer may report, and the prompt's list
437
- * exactly. `none` is B3's to write. `screenshot` is not the reproducer's to
438
- * claim: it has Bash and Read only (gates/prompts/prove-reproducer.md), so a
439
- * reported screenshot is a UI verification it could not have made.
436
+ * Who observed the after-state, and the after-evidence types each can report
437
+ * (#2262). `none` is B3's to write. The reproducer has Bash and Read only
438
+ * (gates/prompts/prove-reproducer.md), so a screenshot it reports is a UI
439
+ * check it could not have made. Quinn drives a browser against the prove
440
+ * container (workflows/prove.js), so a screenshot is what Quinn observes.
441
+ *
442
+ * The CALLER names the observer, never the report: an agent writing
443
+ * `observedBy: "quinn"` into its own afterEvidence is ignored. The spawn
444
+ * schemas in workflows/prove.js and `ProveEvidenceSchema`'s `observedBy` restate
445
+ * these lists, and test/prove-workflow-evidence.test.ts holds them equal.
440
446
  */
441
- const REPORTED_AFTER_TYPES = ["text", "api", "data"];
447
+ export const AFTER_OBSERVERS = {
448
+ reproducer: ["text", "api", "data"],
449
+ quinn: ["text", "api", "data", "screenshot"],
450
+ } as const;
451
+ export type AfterObserver = keyof typeof AFTER_OBSERVERS;
452
+ export const isAfterObserver = (v: unknown): v is AfterObserver =>
453
+ typeof v === "string" && Object.prototype.hasOwnProperty.call(AFTER_OBSERVERS, v);
442
454
  const AFTER_ENVIRONMENTS = ["local", "prod"];
443
455
 
444
456
  /** What B3 knows about the run that the reproducer does not get to say. */
@@ -449,24 +461,44 @@ export interface ProveEvidenceContext {
449
461
  beforeReading: BeforeStateReading;
450
462
  /** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
451
463
  source?: string;
464
+ /** Who observed the after-state. The reproducer unless the caller says otherwise (#2262). */
465
+ afterObserver?: AfterObserver;
452
466
  }
453
467
 
454
468
  type AfterEvidence = {
455
469
  type: string; path: string | null; capturedAt: string; environment: string; description: string;
470
+ observedBy?: AfterObserver;
456
471
  };
457
472
 
458
- /** The reproducer's `afterEvidence`, or why there is none. Never repaired into shape. */
459
- function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
460
- if (raw === undefined || raw === null) return { ok: false, reason: "the reproducer reported no afterEvidence" };
473
+ function isFile(path: string): boolean {
474
+ try {
475
+ return statSync(path).isFile();
476
+ } catch {
477
+ return false; // absent or unreadable: either way nobody can open it
478
+ }
479
+ }
480
+
481
+ /** The reported `afterEvidence`, or why there is none. Never repaired into shape. */
482
+ function readReportedAfterEvidence(raw: unknown, observer: AfterObserver): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
483
+ if (raw === undefined || raw === null) return { ok: false, reason: `the ${observer} reported no afterEvidence` };
461
484
  if (typeof raw !== "object" || Array.isArray(raw)) {
462
485
  return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
463
486
  }
464
487
  const a = raw as Record<string, unknown>;
488
+ const types: readonly string[] = AFTER_OBSERVERS[observer];
465
489
  if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
466
490
  return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
467
491
  }
468
- if (typeof a.type !== "string" || !REPORTED_AFTER_TYPES.includes(a.type)) {
469
- return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one of ${REPORTED_AFTER_TYPES.join("|")}` };
492
+ if (typeof a.type !== "string" || !types.includes(a.type)) {
493
+ return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one the ${observer} can observe (${types.join("|")})` };
494
+ }
495
+ if (a.type === "screenshot" && (typeof a.path !== "string" || a.path.trim() === "")) {
496
+ return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is empty — a screenshot with no file is not evidence` };
497
+ }
498
+ // Review round 1, PR #2266: a path is a claim. The file has to be there,
499
+ // or a screenshot nobody took records as proof.
500
+ if (a.type === "screenshot" && !isFile(a.path as string)) {
501
+ return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is not a file — a screenshot nobody can open is not evidence` };
470
502
  }
471
503
  if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
472
504
  return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
@@ -479,6 +511,7 @@ function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEviden
479
511
  value: {
480
512
  type: a.type, path: typeof a.path === "string" ? a.path : null,
481
513
  capturedAt: a.capturedAt, environment: a.environment, description: a.description,
514
+ observedBy: observer,
482
515
  },
483
516
  };
484
517
  }
@@ -504,7 +537,7 @@ function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEviden
504
537
  export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
505
538
  const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
506
539
  const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
507
- const after = readReportedAfterEvidence(raw?.afterEvidence);
540
+ const after = readReportedAfterEvidence(raw?.afterEvidence, ctx.afterObserver ?? "reproducer");
508
541
 
509
542
  let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
510
543
  let verdictDowngrade: { reported: unknown; reason: string } | undefined;
@@ -569,6 +602,7 @@ export function recordProveEvidence(
569
602
  readCommit: () => string;
570
603
  schema?: ArtifactSchema;
571
604
  source?: string;
605
+ afterObserver?: AfterObserver;
572
606
  },
573
607
  ): ProveEvidenceRecord {
574
608
  try {
@@ -578,6 +612,7 @@ export function recordProveEvidence(
578
612
  capturedAt: new Date().toISOString(),
579
613
  beforeReading: opts.beforeReading,
580
614
  source: opts.source,
615
+ afterObserver: opts.afterObserver,
581
616
  });
582
617
  const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
583
618
  if (violations.length > 0) {
@@ -2837,9 +2872,70 @@ export function fanoutSkipReason(
2837
2872
  return null;
2838
2873
  }
2839
2874
 
2875
+ /** One row of the B2 validator's reply, the shape gates/prompts/evidence-validator.md asks for. */
2876
+ export type B2Verdict = { acId: string; verdict: "PASS" | "FAIL"; rawOutput?: unknown; [field: string]: unknown };
2877
+
2878
+ /**
2879
+ * The B2 validator's reply, or why it could not be read (#2265). The one
2880
+ * place its JSON is extracted: two copies are two readers that can disagree,
2881
+ * which is what #2264's first review round found for B1.
2882
+ */
2883
+ export function parseB2Reply(raw: string): { status: "read"; verdicts: B2Verdict[] } | { status: "not-read"; detail: string } {
2884
+ const json = raw.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
2885
+ if (!json) return { status: "not-read", detail: 'the reply holds no JSON object naming "verdicts"' };
2886
+ let reply: any;
2887
+ try {
2888
+ reply = JSON.parse(json[0]);
2889
+ } catch (e: any) {
2890
+ return { status: "not-read", detail: `the reply's JSON does not parse: ${String(e?.message ?? e).slice(0, 200)}` };
2891
+ }
2892
+ if (!Array.isArray(reply?.verdicts)) {
2893
+ return { status: "not-read", detail: `verdicts is ${JSON.stringify(reply?.verdicts) ?? "absent"}, not a list` };
2894
+ }
2895
+ const bad = reply.verdicts.findIndex((v: any) =>
2896
+ v === null || typeof v !== "object" || typeof v.acId !== "string" || v.acId.trim() === "" ||
2897
+ (v.verdict !== "PASS" && v.verdict !== "FAIL"));
2898
+ if (bad >= 0) {
2899
+ return {
2900
+ status: "not-read",
2901
+ detail: `verdicts[${bad}] has no acId, or a verdict that is not PASS or FAIL: ${String(JSON.stringify(reply.verdicts[bad])).slice(0, 200)}`,
2902
+ };
2903
+ }
2904
+ return { status: "read", verdicts: reply.verdicts as B2Verdict[] };
2905
+ }
2906
+
2907
+ /**
2908
+ * The B2 validator's reply as verify-gate results and the reading verify
2909
+ * records (#2265). Advisory, as it always was: every result is a WARN. A
2910
+ * reply that could not be read is one WARN naming why, and the record says
2911
+ * UNMEASURED. Before, it was nothing: no result, and no record, so the run
2912
+ * looked like one whose every criterion's evidence was sound.
2913
+ */
2914
+ export function b2ReplyReading(
2915
+ output: string,
2916
+ ts: string,
2917
+ ): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
2918
+ const parsed = parseB2Reply(output);
2919
+ if (parsed.status !== "read") {
2920
+ return {
2921
+ agentResult: { ts, status: "UNMEASURED", reason: `reply unreadable: ${parsed.detail}`, source: "B2-evidence-validator-agent" },
2922
+ warns: 1,
2923
+ results: [{ check: "B2-validator: reply unreadable", result: "WARN" as const, detail: parsed.detail }],
2924
+ };
2925
+ }
2926
+ const fails = parsed.verdicts.filter(v => v.verdict === "FAIL");
2927
+ return {
2928
+ agentResult: { ts, verdicts: parsed.verdicts, source: "B2-evidence-validator-agent" },
2929
+ warns: fails.length,
2930
+ results: fails.map(v => ({
2931
+ check: `B2-validator: ${v.acId} evidence gap`,
2932
+ result: "WARN" as const,
2933
+ detail: typeof v.rawOutput === "string" && v.rawOutput !== "" ? v.rawOutput : "threshold not met",
2934
+ })),
2935
+ };
2936
+ }
2937
+
2840
2938
  function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
2841
- let addedWarns = 0;
2842
- let agentResult: any = null;
2843
2939
  const tier = state.sizing?.ceremonyTier || "STANDARD";
2844
2940
  // Recorded rather than returned empty (#126): the consumer allows a LIGHT
2845
2941
  // absence, but an artefact that SAYS why is one a reader does not have to
@@ -2879,33 +2975,16 @@ function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string
2879
2975
  console.warn(`WARN: B2 Evidence Validator exited with error: ${e.message?.slice(0, 200)}`);
2880
2976
  }
2881
2977
 
2882
- const b2Json = b2Output.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
2883
- if (b2Json) {
2884
- try {
2885
- const b2Report = JSON.parse(b2Json[0]);
2886
- agentResult = {
2887
- ts: new Date().toISOString(),
2888
- verdicts: b2Report.verdicts || [],
2889
- source: "B2-evidence-validator-agent",
2890
- };
2891
- const b2Fails = (b2Report.verdicts || []).filter((v: any) => v.verdict === "FAIL");
2892
- if (b2Fails.length > 0) {
2893
- for (const v of b2Fails) {
2894
- addedWarns++;
2895
- results.push({ check: `B2-validator: ${v.acId} evidence gap`, result: "WARN" as const, detail: v.rawOutput || "threshold not met" });
2896
- }
2897
- console.log(`B2: ${b2Fails.length} evidence gap(s) found — WARN`);
2898
- } else {
2899
- console.log(`B2: All evidence validated — ${(b2Report.verdicts || []).length} ACs checked`);
2900
- }
2901
- } catch {
2902
- console.warn("WARN: B2 Evidence Validator JSON parse failed");
2903
- }
2978
+ const reading = b2ReplyReading(b2Output, new Date().toISOString());
2979
+ results.push(...reading.results);
2980
+ if (reading.agentResult.status === "UNMEASURED") {
2981
+ console.warn(`WARN: B2 Evidence Validator reply UNMEASURED — ${reading.agentResult.reason}`);
2982
+ } else if (reading.warns > 0) {
2983
+ console.log(`B2: ${reading.warns} evidence gap(s) found — WARN`);
2904
2984
  } else {
2905
- console.warn("WARN: B2 Evidence Validator produced no parseable JSON");
2985
+ console.log(`B2: All evidence validated — ${(reading.agentResult.verdicts as unknown[]).length} ACs checked`);
2906
2986
  }
2907
-
2908
- return { agentResult, warns: addedWarns };
2987
+ return { agentResult: reading.agentResult, warns: reading.warns };
2909
2988
  }
2910
2989
 
2911
2990
  function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
package/gates/schema.ts CHANGED
@@ -1284,6 +1284,10 @@ export const ProveEvidenceSchema = z.object({
1284
1284
  capturedAt: z.string(),
1285
1285
  environment: z.enum(["local", "prod"]),
1286
1286
  description: z.string(),
1287
+ // Who observed the after-state (#2262): `AFTER_OBSERVERS` in
1288
+ // gates/gate-executor.ts, which decides the types each may report. Optional
1289
+ // because a `none` record has no observer.
1290
+ observedBy: z.enum(["reproducer", "quinn"]).optional(),
1287
1291
  }),
1288
1292
  comparisonSummary: z.string(),
1289
1293
  quinnVerdict: z.enum(["PASS", "FAIL", "SKIP"]).optional(),
@@ -119,6 +119,9 @@ export interface SectionReading {
119
119
  text: string;
120
120
  /** Its non-blank lines, heading included. The unit the loss check counts. */
121
121
  lines: string[];
122
+ /** Where the heading matched, as an offset into the content (#2285): the one
123
+ * position, so a caller never re-finds the section by searching for its text. */
124
+ start: number;
122
125
  }
123
126
 
124
127
  export interface PreserveResult {
@@ -161,7 +164,7 @@ export function readPreservedSection(
161
164
  // The heading alone is not a section. One line means nothing survived the
162
165
  // last edit, and there is no content to promise anybody.
163
166
  if (lines.length < 2) return null;
164
- return { text, lines };
167
+ return { text, lines, start };
165
168
  }
166
169
 
167
170
  /**