rungate 0.8.24 → 0.8.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +118 -39
- package/gates/schema.ts +4 -0
- package/lib/agents-md-sections.ts +4 -1
- package/lib/bash-write-guard.ts +667 -257
- package/lib/ci-reading.ts +186 -0
- package/lib/cli-commands.ts +8 -0
- package/lib/rule-provenance.ts +54 -3
- package/lib/scaffold/ai-review-step.ts +81 -83
- package/lib/scaffold/marked-section.ts +75 -0
- package/lib/scaffold/owned-files.ts +2 -2
- package/lib/scaffold/steps.ts +153 -3
- package/lib/ship-mode.ts +15 -3
- package/package.json +3 -1
- package/scripts/ci-reading.ts +192 -0
- package/scripts/detect-sc-drift.ts +44 -2
- package/scripts/scaffold-project.ts +13 -1
package/gates/.gate-salt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
f7edfff5c8424ba64c95ad1a592d7b40c40c3a78197e968739a5d468cd671467
|
package/gates/gate-executor.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { execSync, execFileSync } from "child_process";
|
|
2
|
-
import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync } from "fs";
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync, statSync } from "fs";
|
|
3
3
|
import { join } from "path";
|
|
4
4
|
import { splitGateRuns } from "../lib/gate-run-split";
|
|
5
5
|
import { gateSaltPath, workDir as slugWorkDir } from "../lib/paths";
|
|
@@ -433,12 +433,24 @@ export { validateArtifactSchema, type ArtifactSchema };
|
|
|
433
433
|
|
|
434
434
|
const PROVE_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
435
435
|
/**
|
|
436
|
-
*
|
|
437
|
-
*
|
|
438
|
-
*
|
|
439
|
-
*
|
|
436
|
+
* Who observed the after-state, and the after-evidence types each can report
|
|
437
|
+
* (#2262). `none` is B3's to write. The reproducer has Bash and Read only
|
|
438
|
+
* (gates/prompts/prove-reproducer.md), so a screenshot it reports is a UI
|
|
439
|
+
* check it could not have made. Quinn drives a browser against the prove
|
|
440
|
+
* container (workflows/prove.js), so a screenshot is what Quinn observes.
|
|
441
|
+
*
|
|
442
|
+
* The CALLER names the observer, never the report: an agent writing
|
|
443
|
+
* `observedBy: "quinn"` into its own afterEvidence is ignored. The spawn
|
|
444
|
+
* schemas in workflows/prove.js and `ProveEvidenceSchema`'s `observedBy` restate
|
|
445
|
+
* these lists, and test/prove-workflow-evidence.test.ts holds them equal.
|
|
440
446
|
*/
|
|
441
|
-
const
|
|
447
|
+
export const AFTER_OBSERVERS = {
|
|
448
|
+
reproducer: ["text", "api", "data"],
|
|
449
|
+
quinn: ["text", "api", "data", "screenshot"],
|
|
450
|
+
} as const;
|
|
451
|
+
export type AfterObserver = keyof typeof AFTER_OBSERVERS;
|
|
452
|
+
export const isAfterObserver = (v: unknown): v is AfterObserver =>
|
|
453
|
+
typeof v === "string" && Object.prototype.hasOwnProperty.call(AFTER_OBSERVERS, v);
|
|
442
454
|
const AFTER_ENVIRONMENTS = ["local", "prod"];
|
|
443
455
|
|
|
444
456
|
/** What B3 knows about the run that the reproducer does not get to say. */
|
|
@@ -449,24 +461,44 @@ export interface ProveEvidenceContext {
|
|
|
449
461
|
beforeReading: BeforeStateReading;
|
|
450
462
|
/** Which writer this is. B3 by default; `workflows/prove.js` says `prove-workflow` (#2254). */
|
|
451
463
|
source?: string;
|
|
464
|
+
/** Who observed the after-state. The reproducer unless the caller says otherwise (#2262). */
|
|
465
|
+
afterObserver?: AfterObserver;
|
|
452
466
|
}
|
|
453
467
|
|
|
454
468
|
type AfterEvidence = {
|
|
455
469
|
type: string; path: string | null; capturedAt: string; environment: string; description: string;
|
|
470
|
+
observedBy?: AfterObserver;
|
|
456
471
|
};
|
|
457
472
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
473
|
+
function isFile(path: string): boolean {
|
|
474
|
+
try {
|
|
475
|
+
return statSync(path).isFile();
|
|
476
|
+
} catch {
|
|
477
|
+
return false; // absent or unreadable: either way nobody can open it
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** The reported `afterEvidence`, or why there is none. Never repaired into shape. */
|
|
482
|
+
function readReportedAfterEvidence(raw: unknown, observer: AfterObserver): { ok: true; value: AfterEvidence } | { ok: false; reason: string } {
|
|
483
|
+
if (raw === undefined || raw === null) return { ok: false, reason: `the ${observer} reported no afterEvidence` };
|
|
461
484
|
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
462
485
|
return { ok: false, reason: `afterEvidence is not an object: ${JSON.stringify(raw)}` };
|
|
463
486
|
}
|
|
464
487
|
const a = raw as Record<string, unknown>;
|
|
488
|
+
const types: readonly string[] = AFTER_OBSERVERS[observer];
|
|
465
489
|
if (typeof a.environment !== "string" || !AFTER_ENVIRONMENTS.includes(a.environment)) {
|
|
466
490
|
return { ok: false, reason: `afterEvidence.environment ${JSON.stringify(a.environment)} is not local or prod` };
|
|
467
491
|
}
|
|
468
|
-
if (typeof a.type !== "string" || !
|
|
469
|
-
return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one
|
|
492
|
+
if (typeof a.type !== "string" || !types.includes(a.type)) {
|
|
493
|
+
return { ok: false, reason: `afterEvidence.type ${JSON.stringify(a.type)} is not one the ${observer} can observe (${types.join("|")})` };
|
|
494
|
+
}
|
|
495
|
+
if (a.type === "screenshot" && (typeof a.path !== "string" || a.path.trim() === "")) {
|
|
496
|
+
return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is empty — a screenshot with no file is not evidence` };
|
|
497
|
+
}
|
|
498
|
+
// Review round 1, PR #2266: a path is a claim. The file has to be there,
|
|
499
|
+
// or a screenshot nobody took records as proof.
|
|
500
|
+
if (a.type === "screenshot" && !isFile(a.path as string)) {
|
|
501
|
+
return { ok: false, reason: `afterEvidence.path ${JSON.stringify(a.path)} is not a file — a screenshot nobody can open is not evidence` };
|
|
470
502
|
}
|
|
471
503
|
if (typeof a.capturedAt !== "string" || a.capturedAt === "") {
|
|
472
504
|
return { ok: false, reason: `afterEvidence.capturedAt ${JSON.stringify(a.capturedAt)} is not a time` };
|
|
@@ -479,6 +511,7 @@ function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEviden
|
|
|
479
511
|
value: {
|
|
480
512
|
type: a.type, path: typeof a.path === "string" ? a.path : null,
|
|
481
513
|
capturedAt: a.capturedAt, environment: a.environment, description: a.description,
|
|
514
|
+
observedBy: observer,
|
|
482
515
|
},
|
|
483
516
|
};
|
|
484
517
|
}
|
|
@@ -504,7 +537,7 @@ function readReportedAfterEvidence(raw: unknown): { ok: true; value: AfterEviden
|
|
|
504
537
|
export function buildProveEvidence(raw: Record<string, any>, ctx: ProveEvidenceContext) {
|
|
505
538
|
const criteriaResults: any[] = Array.isArray(raw?.criteriaResults) ? raw.criteriaResults : [];
|
|
506
539
|
const passed = criteriaResults.filter((r) => r?.verdict === "PASS").length;
|
|
507
|
-
const after = readReportedAfterEvidence(raw?.afterEvidence);
|
|
540
|
+
const after = readReportedAfterEvidence(raw?.afterEvidence, ctx.afterObserver ?? "reproducer");
|
|
508
541
|
|
|
509
542
|
let verdict = typeof raw?.verdict === "string" && PROVE_VERDICTS.includes(raw.verdict) ? raw.verdict : "INCONCLUSIVE";
|
|
510
543
|
let verdictDowngrade: { reported: unknown; reason: string } | undefined;
|
|
@@ -569,6 +602,7 @@ export function recordProveEvidence(
|
|
|
569
602
|
readCommit: () => string;
|
|
570
603
|
schema?: ArtifactSchema;
|
|
571
604
|
source?: string;
|
|
605
|
+
afterObserver?: AfterObserver;
|
|
572
606
|
},
|
|
573
607
|
): ProveEvidenceRecord {
|
|
574
608
|
try {
|
|
@@ -578,6 +612,7 @@ export function recordProveEvidence(
|
|
|
578
612
|
capturedAt: new Date().toISOString(),
|
|
579
613
|
beforeReading: opts.beforeReading,
|
|
580
614
|
source: opts.source,
|
|
615
|
+
afterObserver: opts.afterObserver,
|
|
581
616
|
});
|
|
582
617
|
const violations = validateArtifactSchema(artifact, opts.schema ?? ProveEvidenceSchema);
|
|
583
618
|
if (violations.length > 0) {
|
|
@@ -2837,9 +2872,70 @@ export function fanoutSkipReason(
|
|
|
2837
2872
|
return null;
|
|
2838
2873
|
}
|
|
2839
2874
|
|
|
2875
|
+
/** One row of the B2 validator's reply, the shape gates/prompts/evidence-validator.md asks for. */
|
|
2876
|
+
export type B2Verdict = { acId: string; verdict: "PASS" | "FAIL"; rawOutput?: unknown; [field: string]: unknown };
|
|
2877
|
+
|
|
2878
|
+
/**
|
|
2879
|
+
* The B2 validator's reply, or why it could not be read (#2265). The one
|
|
2880
|
+
* place its JSON is extracted: two copies are two readers that can disagree,
|
|
2881
|
+
* which is what #2264's first review round found for B1.
|
|
2882
|
+
*/
|
|
2883
|
+
export function parseB2Reply(raw: string): { status: "read"; verdicts: B2Verdict[] } | { status: "not-read"; detail: string } {
|
|
2884
|
+
const json = raw.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
|
|
2885
|
+
if (!json) return { status: "not-read", detail: 'the reply holds no JSON object naming "verdicts"' };
|
|
2886
|
+
let reply: any;
|
|
2887
|
+
try {
|
|
2888
|
+
reply = JSON.parse(json[0]);
|
|
2889
|
+
} catch (e: any) {
|
|
2890
|
+
return { status: "not-read", detail: `the reply's JSON does not parse: ${String(e?.message ?? e).slice(0, 200)}` };
|
|
2891
|
+
}
|
|
2892
|
+
if (!Array.isArray(reply?.verdicts)) {
|
|
2893
|
+
return { status: "not-read", detail: `verdicts is ${JSON.stringify(reply?.verdicts) ?? "absent"}, not a list` };
|
|
2894
|
+
}
|
|
2895
|
+
const bad = reply.verdicts.findIndex((v: any) =>
|
|
2896
|
+
v === null || typeof v !== "object" || typeof v.acId !== "string" || v.acId.trim() === "" ||
|
|
2897
|
+
(v.verdict !== "PASS" && v.verdict !== "FAIL"));
|
|
2898
|
+
if (bad >= 0) {
|
|
2899
|
+
return {
|
|
2900
|
+
status: "not-read",
|
|
2901
|
+
detail: `verdicts[${bad}] has no acId, or a verdict that is not PASS or FAIL: ${String(JSON.stringify(reply.verdicts[bad])).slice(0, 200)}`,
|
|
2902
|
+
};
|
|
2903
|
+
}
|
|
2904
|
+
return { status: "read", verdicts: reply.verdicts as B2Verdict[] };
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
/**
|
|
2908
|
+
* The B2 validator's reply as verify-gate results and the reading verify
|
|
2909
|
+
* records (#2265). Advisory, as it always was: every result is a WARN. A
|
|
2910
|
+
* reply that could not be read is one WARN naming why, and the record says
|
|
2911
|
+
* UNMEASURED. Before, it was nothing: no result, and no record, so the run
|
|
2912
|
+
* looked like one whose every criterion's evidence was sound.
|
|
2913
|
+
*/
|
|
2914
|
+
export function b2ReplyReading(
|
|
2915
|
+
output: string,
|
|
2916
|
+
ts: string,
|
|
2917
|
+
): { agentResult: Record<string, unknown>; warns: number; results: GateResult[] } {
|
|
2918
|
+
const parsed = parseB2Reply(output);
|
|
2919
|
+
if (parsed.status !== "read") {
|
|
2920
|
+
return {
|
|
2921
|
+
agentResult: { ts, status: "UNMEASURED", reason: `reply unreadable: ${parsed.detail}`, source: "B2-evidence-validator-agent" },
|
|
2922
|
+
warns: 1,
|
|
2923
|
+
results: [{ check: "B2-validator: reply unreadable", result: "WARN" as const, detail: parsed.detail }],
|
|
2924
|
+
};
|
|
2925
|
+
}
|
|
2926
|
+
const fails = parsed.verdicts.filter(v => v.verdict === "FAIL");
|
|
2927
|
+
return {
|
|
2928
|
+
agentResult: { ts, verdicts: parsed.verdicts, source: "B2-evidence-validator-agent" },
|
|
2929
|
+
warns: fails.length,
|
|
2930
|
+
results: fails.map(v => ({
|
|
2931
|
+
check: `B2-validator: ${v.acId} evidence gap`,
|
|
2932
|
+
result: "WARN" as const,
|
|
2933
|
+
detail: typeof v.rawOutput === "string" && v.rawOutput !== "" ? v.rawOutput : "threshold not met",
|
|
2934
|
+
})),
|
|
2935
|
+
};
|
|
2936
|
+
}
|
|
2937
|
+
|
|
2840
2938
|
function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
|
2841
|
-
let addedWarns = 0;
|
|
2842
|
-
let agentResult: any = null;
|
|
2843
2939
|
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2844
2940
|
// Recorded rather than returned empty (#126): the consumer allows a LIGHT
|
|
2845
2941
|
// absence, but an artefact that SAYS why is one a reader does not have to
|
|
@@ -2879,33 +2975,16 @@ function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string
|
|
|
2879
2975
|
console.warn(`WARN: B2 Evidence Validator exited with error: ${e.message?.slice(0, 200)}`);
|
|
2880
2976
|
}
|
|
2881
2977
|
|
|
2882
|
-
const
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
verdicts: b2Report.verdicts || [],
|
|
2889
|
-
source: "B2-evidence-validator-agent",
|
|
2890
|
-
};
|
|
2891
|
-
const b2Fails = (b2Report.verdicts || []).filter((v: any) => v.verdict === "FAIL");
|
|
2892
|
-
if (b2Fails.length > 0) {
|
|
2893
|
-
for (const v of b2Fails) {
|
|
2894
|
-
addedWarns++;
|
|
2895
|
-
results.push({ check: `B2-validator: ${v.acId} evidence gap`, result: "WARN" as const, detail: v.rawOutput || "threshold not met" });
|
|
2896
|
-
}
|
|
2897
|
-
console.log(`B2: ${b2Fails.length} evidence gap(s) found — WARN`);
|
|
2898
|
-
} else {
|
|
2899
|
-
console.log(`B2: All evidence validated — ${(b2Report.verdicts || []).length} ACs checked`);
|
|
2900
|
-
}
|
|
2901
|
-
} catch {
|
|
2902
|
-
console.warn("WARN: B2 Evidence Validator JSON parse failed");
|
|
2903
|
-
}
|
|
2978
|
+
const reading = b2ReplyReading(b2Output, new Date().toISOString());
|
|
2979
|
+
results.push(...reading.results);
|
|
2980
|
+
if (reading.agentResult.status === "UNMEASURED") {
|
|
2981
|
+
console.warn(`WARN: B2 Evidence Validator reply UNMEASURED — ${reading.agentResult.reason}`);
|
|
2982
|
+
} else if (reading.warns > 0) {
|
|
2983
|
+
console.log(`B2: ${reading.warns} evidence gap(s) found — WARN`);
|
|
2904
2984
|
} else {
|
|
2905
|
-
console.
|
|
2985
|
+
console.log(`B2: All evidence validated — ${(reading.agentResult.verdicts as unknown[]).length} ACs checked`);
|
|
2906
2986
|
}
|
|
2907
|
-
|
|
2908
|
-
return { agentResult, warns: addedWarns };
|
|
2987
|
+
return { agentResult: reading.agentResult, warns: reading.warns };
|
|
2909
2988
|
}
|
|
2910
2989
|
|
|
2911
2990
|
function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
package/gates/schema.ts
CHANGED
|
@@ -1284,6 +1284,10 @@ export const ProveEvidenceSchema = z.object({
|
|
|
1284
1284
|
capturedAt: z.string(),
|
|
1285
1285
|
environment: z.enum(["local", "prod"]),
|
|
1286
1286
|
description: z.string(),
|
|
1287
|
+
// Who observed the after-state (#2262): `AFTER_OBSERVERS` in
|
|
1288
|
+
// gates/gate-executor.ts, which decides the types each may report. Optional
|
|
1289
|
+
// because a `none` record has no observer.
|
|
1290
|
+
observedBy: z.enum(["reproducer", "quinn"]).optional(),
|
|
1287
1291
|
}),
|
|
1288
1292
|
comparisonSummary: z.string(),
|
|
1289
1293
|
quinnVerdict: z.enum(["PASS", "FAIL", "SKIP"]).optional(),
|
|
@@ -119,6 +119,9 @@ export interface SectionReading {
|
|
|
119
119
|
text: string;
|
|
120
120
|
/** Its non-blank lines, heading included. The unit the loss check counts. */
|
|
121
121
|
lines: string[];
|
|
122
|
+
/** Where the heading matched, as an offset into the content (#2285): the one
|
|
123
|
+
* position, so a caller never re-finds the section by searching for its text. */
|
|
124
|
+
start: number;
|
|
122
125
|
}
|
|
123
126
|
|
|
124
127
|
export interface PreserveResult {
|
|
@@ -161,7 +164,7 @@ export function readPreservedSection(
|
|
|
161
164
|
// The heading alone is not a section. One line means nothing survived the
|
|
162
165
|
// last edit, and there is no content to promise anybody.
|
|
163
166
|
if (lines.length < 2) return null;
|
|
164
|
-
return { text, lines };
|
|
167
|
+
return { text, lines, start };
|
|
165
168
|
}
|
|
166
169
|
|
|
167
170
|
/**
|