rungate 0.8.23 → 0.8.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,10 @@
1
1
  import { test, expect, describe, beforeEach, afterEach } from "bun:test";
2
2
  import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync, statSync } from "fs";
3
3
  import { join } from "path";
4
+ import { z } from "zod";
4
5
  import { gateSaltPath } from "../lib/paths";
6
+ import { validateArtifactSchema } from "../lib/artifact-schema";
7
+ import { ShipEvidenceSchema } from "./schema";
5
8
  import {
6
9
  writeGateResult,
7
10
  generateHmac,
@@ -419,8 +422,10 @@ describe("generateShipEvidence", () => {
419
422
 
420
423
  const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
421
424
 
422
- expect(existsSync(path)).toBe(true);
423
- const evidence = JSON.parse(readFileSync(path, "utf-8"));
425
+ // null means the contract refused the artifact (#2255).
426
+ expect(path).not.toBeNull();
427
+ expect(existsSync(path!)).toBe(true);
428
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
424
429
  expect(evidence.contractVersion).toBe("1.0");
425
430
  expect(evidence.issueNumber).toBe(432);
426
431
  expect(evidence.gateOut.status).toBe("PASS");
@@ -429,6 +434,107 @@ describe("generateShipEvidence", () => {
429
434
  expect(evidence.capturedAt).toBeTruthy();
430
435
  expect(evidence.iterationCount).toBeGreaterThanOrEqual(1);
431
436
  expect(evidence.sizing.predicted).toBe("M");
437
+ // #2255 / SC-2770: the artifact meets its declared contract, not only the
438
+ // eight fields this case happens to name.
439
+ expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
440
+ });
441
+
442
+ test("#2255 / SC-2770: a schema with one required field more than the writer emits refuses the artifact", () => {
443
+ writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP" }), null, 2));
444
+ const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
445
+ expect(path).not.toBeNull();
446
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
447
+ const stricter = ShipEvidenceSchema.extend({ reviewedBy: z.string() });
448
+ expect(validateArtifactSchema(evidence, stricter).join("\n")).toContain("reviewedBy");
449
+ });
450
+
451
+ test("#2255: an artifact the contract refuses is not written, and the field is named", () => {
452
+ // A verdict outside PASS|FAIL|SKIP is the live way the writer can emit
453
+ // what ShipEvidenceSchema refuses: `result: ac.verdict || "SKIP"`.
454
+ writeFileSync(SF, JSON.stringify(makeState({
455
+ phase: "SHIP",
456
+ acs: [{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }],
457
+ }), null, 2));
458
+ const errors: string[] = [];
459
+ const original = console.error;
460
+ console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); };
461
+ let path: string | null;
462
+ try {
463
+ path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
464
+ } finally {
465
+ console.error = original;
466
+ }
467
+ expect(path).toBeNull();
468
+ expect(existsSync(join(TMP_DIR, "ship-evidence.json"))).toBe(false);
469
+ expect(errors.join("\n")).toContain("gateOut.evidence.0.result");
470
+ });
471
+
472
+ test("#2255 review round 1: an OUTCOME criterion still PENDING at ship is written — a legitimate ship-time state", () => {
473
+ // gates/workflow.test.ts: "OUTCOME ACs may be PENDING at ship time — they
474
+ // require Quinn/prove for verification", and enrichment writes the literal
475
+ // "PENDING". The contract refused it, so validation would have dropped the
476
+ // evidence on every normal run with an OUTCOME criterion outstanding.
477
+ writeFileSync(SF, JSON.stringify(makeState({
478
+ phase: "SHIP",
479
+ acs: [{ id: "SC-1", statement: "s", type: "OUTCOME", verdict: "PENDING", evidence: null }],
480
+ }), null, 2));
481
+ const path = generateShipEvidence(SF, TMP_DIR, 432, "", 1, 0, 0);
482
+ expect(path).not.toBeNull();
483
+ const evidence = JSON.parse(readFileSync(path!, "utf-8"));
484
+ expect(evidence.gateOut.evidence[0].result).toBe("PENDING");
485
+ expect(validateArtifactSchema(evidence, ShipEvidenceSchema)).toEqual([]);
486
+ });
487
+ });
488
+
489
+ describe("#2255 review round 1: a refused artifact is never reported to the issue as shipped", () => {
490
+ // A loopback stub stands in for the GitHub API (lib/github.ts allows exactly
491
+ // GitHub or loopback). `fixture/repo` makes verifiedPush refuse rather than
492
+ // push from this checkout (#430). The control case is what keeps the
493
+ // refusal's "no requests" from being vacuous: the same stub, a valid
494
+ // artifact, and the label and comment arrive.
495
+ async function requestsDuring(acs: Array<Record<string, unknown>>): Promise<string[]> {
496
+ const seen: string[] = [];
497
+ const server = Bun.serve({
498
+ port: 0,
499
+ hostname: "127.0.0.1",
500
+ fetch(req) {
501
+ seen.push(`${req.method} ${new URL(req.url).pathname}`);
502
+ return Response.json({});
503
+ },
504
+ });
505
+ try {
506
+ writeFileSync(SF, JSON.stringify(makeState({ phase: "SHIP", acs }), null, 2));
507
+ // A child process, so the credential is SUPPLIED rather than read: the
508
+ // tree resolves it in one place (resolveGitHubToken), and nothing here
509
+ // names it as a read. Async, because this process serves the stub, and
510
+ // a blocking spawn would stall the server the child is calling. The
511
+ // child's pending requests keep it alive until they have landed.
512
+ const child = Bun.spawn(
513
+ ["bun", "-e", `import { generateShipEvidence } from ${JSON.stringify(join(import.meta.dir, "orchestrator.ts"))};` +
514
+ `generateShipEvidence(${JSON.stringify(SF)}, ${JSON.stringify(TMP_DIR)}, 432, "fixture/repo", 1, 0, 0);`],
515
+ {
516
+ cwd: join(import.meta.dir, ".."),
517
+ env: { ...process.env, GH_TOKEN: "test-token-not-real", GITHUB_API_URL: `http://127.0.0.1:${server.port}` },
518
+ stdout: "ignore",
519
+ stderr: "ignore",
520
+ },
521
+ );
522
+ await child.exited;
523
+ } finally {
524
+ server.stop(true);
525
+ }
526
+ return seen;
527
+ }
528
+
529
+ test("control: a valid artifact labels the issue shipped and posts the comment", async () => {
530
+ const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "PASS", evidence: { content: "ok" } }]);
531
+ expect(seen.some(r => r.endsWith("/issues/432/labels"))).toBe(true);
532
+ expect(seen.some(r => r.endsWith("/issues/432/comments"))).toBe(true);
533
+ });
534
+
535
+ test("a refused artifact sends nothing: no shipped label, no Ship Complete comment", async () => {
536
+ const seen = await requestsDuring([{ id: "SC-1", statement: "s", verdict: "UNMEASURED", evidence: { content: "x" } }]);
537
+ expect(seen).toEqual([]);
432
538
  });
433
539
  });
434
540
 
@@ -2,7 +2,8 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync, statSync
2
2
  import { join, dirname } from "path";
3
3
  import { createHash, createHmac, randomBytes } from "crypto";
4
4
  import { execSync } from "child_process";
5
- import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS } from "./schema";
5
+ import { WorkflowStateSchema, ACSchema, AfkBatchPlanSchema, REJECTED_SKIP_REASONS, ShipEvidenceSchema } from "./schema";
6
+ import { validateArtifactSchema } from "../lib/artifact-schema";
6
7
  import { ZodError } from "zod";
7
8
  import { createGitHubClient, addLabels, addComment, type GitHubClient } from "../lib/github";
8
9
  import { computeACHash } from "../lib/workflow-security";
@@ -384,7 +385,7 @@ export function generateShipEvidence(
384
385
  passes: number,
385
386
  fails: number,
386
387
  warns: number,
387
- ): string {
388
+ ): string | null {
388
389
  const state = readState(sf);
389
390
  const sha = gitSha();
390
391
  const ts = isoNow();
@@ -483,12 +484,31 @@ export function generateShipEvidence(
483
484
  ...(pushOutcome ? { push: pushOutcome, pushLeftWorkLocal: leftWorkLocal(pushOutcome) } : {}),
484
485
  };
485
486
 
487
+ // #2255 / SC-2770: checked against its one contract before it is written,
488
+ // as prove-evidence.json is (#1559). An artifact the schema refuses is not
489
+ // written, and the fields are named. The gate's verdict is untouched: by
490
+ // here the ship gate has already passed and the push has happened, and the
491
+ // missing artifact is the visible consequence.
486
492
  const evidencePath = join(workDir, "ship-evidence.json");
487
- writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
493
+ const violations = validateArtifactSchema(evidence, ShipEvidenceSchema);
494
+ if (violations.length > 0) {
495
+ console.error("ship gate: ship-evidence.json NOT written — it fails ShipEvidenceSchema (SC-2770):");
496
+ for (const v of violations) console.error(` - ${v}`);
497
+ } else {
498
+ writeFileSync(evidencePath, JSON.stringify(evidence, null, 2));
499
+ }
488
500
 
489
501
  // GitHub: add shipped label + post comment via Octokit (best-effort, fire-and-forget)
490
502
  // Uses POST additive label strategy (D-5) — never PUT/replace
491
- if (issueRepo && issue > 0) {
503
+ //
504
+ // Not when the contract refused the artifact (#2255): labelling the issue
505
+ // shipped and posting "Ship Complete" beside a refusal is a public success
506
+ // report the run's own record contradicts, and a label is harder to take
507
+ // back than to withhold.
508
+ if (violations.length > 0 && issueRepo && issue > 0) {
509
+ console.error(`ship gate: #${issue} NOT labelled shipped and no Ship Complete comment posted — the evidence was not recorded`);
510
+ }
511
+ if (violations.length === 0 && issueRepo && issue > 0) {
492
512
  try {
493
513
  const github = createGitHubClient();
494
514
  addLabels(github, issueRepo, issue, ["shipped"]).catch(() => {});
@@ -512,7 +532,7 @@ export function generateShipEvidence(
512
532
  } catch {}
513
533
  }
514
534
 
515
- return evidencePath;
535
+ return violations.length > 0 ? null : evidencePath;
516
536
  }
517
537
 
518
538
  const GATE_PENDING_PATH = join(
@@ -2,7 +2,7 @@
2
2
  doc-type: reference
3
3
  status: active
4
4
  owner: jason
5
- updated: 2026-09-19
5
+ updated: 2026-11-15
6
6
  ---
7
7
 
8
8
  You are a prove reproducer. Your objective: reproduce the bug described
@@ -44,9 +44,22 @@ Output JSON:
44
44
  "evidence": [
45
45
  {"type": "screenshot|api-response|console", "description": "what this shows"}
46
46
  ],
47
+ "afterEvidence": {
48
+ "type": "text|api|data",
49
+ "path": null,
50
+ "capturedAt": "ISO-8601 time you observed the after-state",
51
+ "environment": "local|prod",
52
+ "description": "what the after-state showed"
53
+ },
47
54
  "gaps": []
48
55
  }
49
56
 
57
+ `afterEvidence` is the after-state you observed yourself. `environment` is
58
+ where you observed it: `local` for this checkout or a dev server, `prod` for
59
+ the production URL. If you observed no after-state, leave `afterEvidence`
60
+ out. Never fill it in to make the record complete: a PROVEN without it is
61
+ recorded as INCONCLUSIVE.
62
+
50
63
  Verdict rules:
51
64
  - PROVEN: bug was reproduced in before-state AND fix verified in after-state
52
65
  - UNPROVEN: fix does not resolve the issue (evidence shows problem persists)
@@ -1,3 +1,3 @@
1
1
  ba56e1fcb00c7a9bc1f2b05beca701cb264785e017cd2bf54a2dd07cdde331df ac-adversary.md
2
2
  7d4ff0ef4b787e68b59643c11ba51e62a9411f5e24509b97855a337598c67508 evidence-validator.md
3
- fd6e01a8e6ec1845a39bd29b15355320238ca7dcf9131e25f5b8cb1468f635ee prove-reproducer.md
3
+ 6f15ff6891118f749894f55f18fa21234211f993cf6eadfb3c90561a64b91691 prove-reproducer.md
@@ -1,7 +1,8 @@
1
1
  import { test, expect, describe } from "bun:test";
2
2
  import { readFileSync, existsSync, writeFileSync } from "fs";
3
3
  import { execSync } from "child_process";
4
- import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState } from "./gate-executor";
4
+ import { beforeEvidenceFor, comparisonSummaryFor, readBeforeState, validateArtifactSchema } from "./gate-executor";
5
+ import { ProveEvidenceSchema } from "./schema";
5
6
 
6
7
  const TEST_DIR = process.env.TEST_WORK_DIR || "";
7
8
  const PROJECT_ROOT = process.env.PROJECT_ROOT || process.cwd();
@@ -145,6 +146,10 @@ describe("prove: run evidence commands from workflow-state", () => {
145
146
  criteriaResults,
146
147
  };
147
148
 
149
+ // #2254 / SC-2770: checked against the one contract before it is written,
150
+ // as B3's and prove.js's are. A required field this writer stops emitting
151
+ // fails here, with the field named, instead of in whatever reads it next.
152
+ expect(validateArtifactSchema(evidence, ProveEvidenceSchema)).toEqual([]);
148
153
  writeFileSync(`${TEST_DIR}/prove-evidence.json`, JSON.stringify(evidence, null, 2));
149
154
  expect(verdict).toBe("PROVEN");
150
155
  });
package/gates/schema.ts CHANGED
@@ -1193,7 +1193,10 @@ export const ShipEvidenceSchema = z.object({
1193
1193
  status: z.enum(["PASS", "FAIL"]),
1194
1194
  evidence: z.array(z.object({
1195
1195
  criterion: z.string(),
1196
- result: z.enum(["PASS", "FAIL", "SKIP"]),
1196
+ // PENDING is a legitimate ship-time state (#2255): an OUTCOME criterion
1197
+ // waits for Quinn or /prove, and the verify gate passes it through
1198
+ // (checkVerifyPendingACs). The writer copies the criterion's verdict.
1199
+ result: z.enum(["PASS", "FAIL", "SKIP", "PENDING"]),
1197
1200
  detail: z.any().optional(),
1198
1201
  })),
1199
1202
  }),
@@ -1252,9 +1255,26 @@ const BeforeEvidenceSchema = z.union([
1252
1255
  }),
1253
1256
  ]);
1254
1257
 
1258
+ /**
1259
+ * One row of `prove-evidence.json`'s `criteriaResults` (#1559). The shape all
1260
+ * three writers already emit — B3's reproducer, `gates/prove.test.ts` and
1261
+ * `workflows/prove.js` — and the one the deleted
1262
+ * `config/prove-evidence.schema.json` declared. SKIP is the reproducer's
1263
+ * verdict for a UI criterion it has no browser to check.
1264
+ */
1265
+ const CriterionResultSchema = z.object({
1266
+ scId: z.string(),
1267
+ verdict: z.enum(["PASS", "FAIL", "SKIP"]),
1268
+ evidence: z.string(),
1269
+ });
1270
+
1255
1271
  export const ProveEvidenceSchema = z.object({
1256
1272
  issueNumber: z.number(),
1257
1273
  verdict: z.enum(["PROVEN", "UNPROVEN", "INCONCLUSIVE"]),
1274
+ // Required, but not `.min(1)`: an INCONCLUSIVE with no criteria is a record
1275
+ // `gates/prove.test.ts` legitimately writes. That a PROVEN needs a PASS row
1276
+ // is `buildProveEvidence`'s rule, because it is about two fields at once.
1277
+ criteriaResults: z.array(CriterionResultSchema),
1258
1278
  commitSHA: z.string(),
1259
1279
  capturedAt: z.string(),
1260
1280
  beforeEvidence: BeforeEvidenceSchema,
@@ -1264,6 +1284,10 @@ export const ProveEvidenceSchema = z.object({
1264
1284
  capturedAt: z.string(),
1265
1285
  environment: z.enum(["local", "prod"]),
1266
1286
  description: z.string(),
1287
+ // Who observed the after-state (#2262): `AFTER_OBSERVERS` in
1288
+ // gates/gate-executor.ts, which decides the types each may report. Optional
1289
+ // because a `none` record has no observer.
1290
+ observedBy: z.enum(["reproducer", "quinn"]).optional(),
1267
1291
  }),
1268
1292
  comparisonSummary: z.string(),
1269
1293
  quinnVerdict: z.enum(["PASS", "FAIL", "SKIP"]).optional(),
@@ -453,6 +453,18 @@ export function verifyFanoutViolations(label: string, val: unknown, tier: unknow
453
453
  `is an absence with extra steps (#126)`,
454
454
  ];
455
455
  }
456
+ // #2259: the agent ran and its reply could not be read. Before, that reply
457
+ // left no record, which the absence branch above refused, so the ship
458
+ // outcome is unchanged. What changes is that the refusal names the cause
459
+ // instead of failing on a missing `gameable` further down.
460
+ if ((val as { status?: unknown }).status === "UNMEASURED") {
461
+ const reason = (val as { reason?: unknown }).reason;
462
+ return [
463
+ `gates.verify.${label} is UNMEASURED — the agent ran and its reply could not be read` +
464
+ (typeof reason === "string" && reason.trim() !== "" ? `: ${reason}` : "") +
465
+ `. An unread reply is not an approval (#126, #2259).`,
466
+ ];
467
+ }
456
468
  return [];
457
469
  }
458
470
 
@@ -119,6 +119,9 @@ export interface SectionReading {
119
119
  text: string;
120
120
  /** Its non-blank lines, heading included. The unit the loss check counts. */
121
121
  lines: string[];
122
+ /** Where the heading matched, as an offset into the content (#2285): the one
123
+ * position, so a caller never re-finds the section by searching for its text. */
124
+ start: number;
122
125
  }
123
126
 
124
127
  export interface PreserveResult {
@@ -161,7 +164,7 @@ export function readPreservedSection(
161
164
  // The heading alone is not a section. One line means nothing survived the
162
165
  // last edit, and there is no content to promise anybody.
163
166
  if (lines.length < 2) return null;
164
- return { text, lines };
167
+ return { text, lines, start };
165
168
  }
166
169
 
167
170
  /**
@@ -224,8 +224,19 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
224
224
  const a = anchorOnlyKey(k);
225
225
  if (a !== null) anchors.add(a);
226
226
  }
227
+ // #2257: how many findings cite a line, and how many of those lines could
228
+ // be read. Read from the SAME calls the keys are built from, not a second
229
+ // pass, so the count cannot disagree with what the keys saw.
230
+ let located = 0;
231
+ let readable = 0;
227
232
  const repeat = candidates.map(c => {
228
- const key = carryKey(c, inputs.lineTextFor(c as ReviewFinding));
233
+ const cited = c as ReviewFinding;
234
+ const lineText = inputs.lineTextFor(cited);
235
+ if (typeof cited?.file === "string" && Number.isInteger(cited?.line) && (cited.line as number) >= 1) {
236
+ located++;
237
+ if (lineText !== undefined) readable++;
238
+ }
239
+ const key = carryKey(c, lineText);
229
240
  // Unkeyable CARRIES. It cannot be shown to be a repeat, and the
230
241
  // direction this is allowed to be wrong in is toward reporting.
231
242
  if (key === null) return null;
@@ -282,7 +293,20 @@ export function aiReviewSuppressionGate(inputs: SuppressionInputs): AiReviewSupp
282
293
  // this whole increment is a correction for.
283
294
  return null;
284
295
  });
285
- return { repeat, unmeasured: null };
296
+ // #2257. A gate that could read no cited line built every key from a line
297
+ // NUMBER, which never equals a key an earlier round recorded from line
298
+ // TEXT. "Nothing to suppress" from that gate is a reading of nothing, and
299
+ // it printed the same words as a gate that read every line (#1872: 0 of
300
+ // 420 in CI). Byte-key matches above still stand; this says the anchor
301
+ // half was blind.
302
+ const unmeasured =
303
+ located > 0 && readable === 0 && prior.keys.length > 0
304
+ ? `no cited line could be read for any of this round's ${located} finding(s) that cite one, ` +
305
+ `so no key could match one an earlier round recorded from the line's text, and a ` +
306
+ `restated finding could not be recognised. The checkout the gate reads is probably not ` +
307
+ `the one under review (#2257).`
308
+ : null;
309
+ return { repeat, unmeasured };
286
310
  };
287
311
  }
288
312
 
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Check a gate artifact against its declared schema, before it is written
3
+ * (#1559, #2255, SC-2770).
4
+ *
5
+ * Lives in `lib/` rather than beside its first caller in
6
+ * `gates/gate-executor.ts` because `gates/orchestrator.ts` needs it too, and
7
+ * gate-executor already imports orchestrator. `gates/gate-executor.ts`
8
+ * re-exports both names, so there is one definition and every existing import
9
+ * keeps working.
10
+ */
11
+
12
+ /**
13
+ * Anything with zod's `safeParse` shape. Structural, so callers keep no
14
+ * direct zod import and a test can hand it a stricter copy of a schema.
15
+ */
16
+ export interface ArtifactSchema {
17
+ safeParse(value: unknown): {
18
+ success: boolean;
19
+ error?: { issues: ReadonlyArray<{ path: ReadonlyArray<PropertyKey>; message: string }> };
20
+ };
21
+ }
22
+
23
+ /**
24
+ * One `path: message` line per violation, `[]` when it validates.
25
+ *
26
+ * `config/prove-evidence.schema.json` declared a contract nothing read, so
27
+ * writer and contract drifted apart: B3's artifact failed both the JSON
28
+ * schema and the zod one, and no check said so. A refusal that names no
29
+ * issue is still a refusal — an empty list there would read as a pass.
30
+ */
31
+ export function validateArtifactSchema(artifact: unknown, schema: ArtifactSchema): string[] {
32
+ const result = schema.safeParse(artifact);
33
+ if (result.success) return [];
34
+ const issues = result.error?.issues ?? [];
35
+ if (issues.length === 0) return ["(root): the schema refused the artifact without naming a field"];
36
+ return issues.map((i) => `${i.path.length > 0 ? i.path.map(String).join(".") : "(root)"}: ${i.message}`);
37
+ }