rungate 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/aditi.md +47 -0
- package/.claude/agents/marcus.md +50 -0
- package/.claude/agents/quinn.md +58 -0
- package/.claude/agents/rook.md +47 -0
- package/.claude/agents/serena.md +46 -0
- package/.claude/rungate.json +11 -0
- package/.fallowrc.json +28 -0
- package/.github/copilot-instructions.md +20 -0
- package/.rungate/conformity-findings.json +648 -0
- package/AGENTS.md +106 -0
- package/CODE-MAP.md +50 -0
- package/HARNESS.md +88 -0
- package/bun.lock +26 -0
- package/bunfig.toml +2 -0
- package/config/brief-policies.json +28 -0
- package/config/ceremony-profiles.json +212 -0
- package/config/consumer-cascade-map.json +24 -0
- package/config/doc-cascade-map.json +24 -0
- package/config/eval.json +84 -0
- package/config/profiles/account-plan.yaml +36 -0
- package/config/profiles/campaign.yaml +94 -0
- package/config/prove-evidence.schema.json +65 -0
- package/config/spec-policies.json +36 -0
- package/config/workflow-schema.json +281 -0
- package/gates/SCHEMA-GUIDE.md +121 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +274 -0
- package/gates/brief-assembler.test.ts +429 -0
- package/gates/brief-assembler.ts +334 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +249 -0
- package/gates/error-classifier.ts +89 -0
- package/gates/orchestrator.test.ts +950 -0
- package/gates/orchestrator.ts +569 -0
- package/gates/preload.ts +77 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prove.test.ts +164 -0
- package/gates/run-gate.ts +936 -0
- package/gates/schema-parity.test.ts +197 -0
- package/gates/schema.ts +362 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +121 -0
- package/gates/ship-orchestrator.test.ts +379 -0
- package/gates/ship-orchestrator.ts +363 -0
- package/gates/test-fixtures/broken-no-environments.json +60 -0
- package/gates/test-fixtures/valid-standard.json +54 -0
- package/gates/test-utils.ts +43 -0
- package/gates/witness.ts +220 -0
- package/gates/workflow.test.ts +1053 -0
- package/hooks/AgentBriefGuard.hook.ts +586 -0
- package/hooks/AgentVerdictCapture.hook.ts +134 -0
- package/hooks/AutoVerifyGate.hook.ts +76 -0
- package/hooks/GateEnforcement.hook.ts +273 -0
- package/hooks/IssueCloseGuard.hook.ts +116 -0
- package/hooks/MergeGuard.hook.ts +55 -0
- package/hooks/StaleTTLCleanup.hook.ts +133 -0
- package/hooks/VerifyPhaseLock.hook.ts +81 -0
- package/hooks/WorkflowStateGuard.hook.ts +37 -0
- package/hooks/lib/agentDetection.ts +70 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +81 -0
- package/lib/conformity.ts +959 -0
- package/lib/paths.ts +23 -0
- package/lib/rungate-schema.ts +60 -0
- package/lib/signal-phrases.ts +7 -0
- package/package.json +20 -0
- package/prompts/ac-adversary.md +20 -0
- package/prompts/ac-format.md +34 -0
- package/prompts/aditi.md +29 -0
- package/prompts/blast-radius.md +29 -0
- package/prompts/coding-principles.md +34 -0
- package/prompts/container-rebuild.md +29 -0
- package/prompts/container-verify.md +28 -0
- package/prompts/discovery.md +27 -0
- package/prompts/environment.md +28 -0
- package/prompts/escalation-decision-tree.md +38 -0
- package/prompts/evidence-hierarchy.md +31 -0
- package/prompts/evidence-validator.md +22 -0
- package/prompts/marcus.md +29 -0
- package/prompts/prevention.md +26 -0
- package/prompts/prove-reproducer.md +43 -0
- package/prompts/quinn-decision-tree.md +31 -0
- package/prompts/quinn-ui-brief.md +106 -0
- package/prompts/quinn.md +29 -0
- package/prompts/rca.md +28 -0
- package/prompts/read-before-write.md +30 -0
- package/prompts/regression.md +26 -0
- package/prompts/rook.md +29 -0
- package/prompts/serena.md +28 -0
- package/prompts/testing-strategy.md +34 -0
- package/reference/migration-manifest.json +554 -0
- package/reference/prompts/agent-aditi.md +38 -0
- package/reference/prompts/agent-marcus.md +38 -0
- package/reference/prompts/agent-quinn.md +38 -0
- package/reference/prompts/agent-rook.md +37 -0
- package/reference/prompts/agent-serena.md +37 -0
- package/reference/prompts/council-skill.md +170 -0
- package/reference/prompts/goal-skill.md +235 -0
- package/reference/prompts/prove-skill.md +283 -0
- package/reference/prompts/quinn-standard.md +246 -0
- package/reference/prompts/ship-convergence.md +65 -0
- package/reference/prompts/ship-escalation.md +43 -0
- package/reference/prompts/ship-iteration.md +382 -0
- package/reference/prompts/ship-pending-patches.md +246 -0
- package/reference/prompts/ship-reference.md +250 -0
- package/reference/prompts/ship-skill.md +347 -0
- package/reference/prompts/ship-templates.md +215 -0
- package/reference/scripts/batch-status.sh +68 -0
- package/reference/scripts/chain-compat-check.sh +67 -0
- package/reference/scripts/context-audit.sh +314 -0
- package/reference/scripts/dead-export-check.sh +61 -0
- package/reference/scripts/generate-qc-tables.sh +92 -0
- package/reference/scripts/harness-init.sh +111 -0
- package/reference/scripts/pai-054-baseline-check.sh +40 -0
- package/reference/scripts/prove-smoke-test.sh +112 -0
- package/reference/scripts/setup-keychain-salt.sh +12 -0
- package/reference/scripts/skill-optimizer-friday.sh +88 -0
- package/reference/scripts/skill-qc-test.sh +351 -0
- package/reference/scripts/skill-synthesize-tonight.sh +66 -0
- package/reference/specs/HARNESS-EXTRACTION-SPEC.md +263 -0
- package/reference/specs/harness-v3-bash-deletion-plan.md +65 -0
- package/reference/specs/harness-v3-migration-plan.md +188 -0
- package/scripts/decision-reconcile.sh +74 -0
- package/scripts/decision-reconcile.ts +152 -0
- package/scripts/extract-constraints.ts +260 -0
- package/scripts/generate-code-map.ts +339 -0
- package/scripts/git-hooks/pre-commit +76 -0
- package/scripts/git-hooks/pre-push +271 -0
- package/scripts/harness-skill-check.sh +52 -0
- package/scripts/lib/gap-emit.sh +65 -0
- package/scripts/lib/jq-update.sh +30 -0
- package/scripts/lib/resolve-slug.sh +30 -0
- package/scripts/lib/verify-spec-compliance.sh +102 -0
- package/scripts/prove-backfill.sh +87 -0
- package/scripts/scaffold-project.ts +1488 -0
- package/scripts/scaffold-rungate-config.ts +91 -0
- package/scripts/self-containment-check.sh +109 -0
- package/scripts/skill-qc-validator.sh +402 -0
- package/scripts/slug-resolver.sh +14 -0
- package/scripts/sync-spec-tests.ts +236 -0
- package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +2493 -0
- package/specs/BOOTSTRAP-TEST-PLAN.md +571 -0
- package/specs/HARNESS-GATES.md +256 -0
- package/specs/HARNESS-SKILL-CHAIN.md +286 -0
- package/specs/HARNESS-SKILL-CONTRACT.md +93 -0
- package/specs/HARNESS-STANDARD.md +488 -0
- package/specs/SPEC-TEMPLATE.md +53 -0
- package/specs/bootstrap-data-flow.png +0 -0
- package/specs/harness-automation-matrix.md +314 -0
- package/specs/harness-automation-matrix.rules.json +64 -0
- package/test/anti.test.ts +80 -0
- package/test/contract-negative.test.ts +86 -0
- package/test/contract.test.ts +158 -0
- package/test/external-deps.test.ts +48 -0
- package/test/fixtures/golden-project/.env.example +3 -0
- package/test/fixtures/golden-project/Makefile +7 -0
- package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +23 -0
- package/test/fixtures/golden-project/docs/setup-guide.md +10 -0
- package/test/fixtures/golden-project/package.json +10 -0
- package/test/fixtures/golden-project/specs/api-spec.md +20 -0
- package/test/fixtures/golden-project/src/auth.ts +8 -0
- package/test/fixtures/golden-project/src/db.ts +11 -0
- package/test/fixtures/golden-project/src/index.ts +19 -0
- package/test/fixtures/golden-project/src/utils/format.ts +3 -0
- package/test/fixtures/golden-project/test/auth.test.ts +15 -0
- package/test/fixtures/golden-project/tsconfig.json +12 -0
- package/test/meta-sc-coverage.test.ts +159 -0
- package/test/phase-0.test.ts +846 -0
- package/test/phase-1.test.ts +617 -0
- package/test/phase-2.test.ts +620 -0
- package/test/phase-3.test.ts +99 -0
- package/test/scaffold-conformity.test.ts +8 -0
- package/test/schema-canary.test.ts +51 -0
- package/test/spec-compliance-auto.test.ts +64 -0
- package/test/spec-compliance.test.ts +295 -0
- package/test/spec-discovery.test.ts +68 -0
- package/test/spec-drift.test.ts +5 -0
- package/test/structure.test.ts +165 -0
- package/test/unit/agent-brief-guard-ship.test.ts +293 -0
- package/test/unit/extract-constraints.test.ts +145 -0
- package/test/unit/issue-close-guard.test.ts +204 -0
- package/test/unit/scaffold-project.test.ts +417 -0
- package/test/unit/skill-cooccurrence.test.ts +177 -0
- package/test/unit/skill-sequence-logger.test.ts +244 -0
- package/test/unit/workflow-state-guard.test.ts +95 -0
- package/workflows/batch-ship.js +295 -0
- package/workflows/council.js +481 -0
- package/workflows/prove.js +560 -0
- package/workflows/ship.js +789 -0
- package/workflows/verify.js +236 -0
|
@@ -0,0 +1,1053 @@
|
|
|
1
|
+
import { test, expect, describe } from "bun:test";
|
|
2
|
+
import { readFileSync, existsSync, readdirSync } from "fs";
|
|
3
|
+
import { execSync } from "child_process";
|
|
4
|
+
import { join } from "path";
|
|
5
|
+
import { WorkflowStateSchema } from "./schema";
|
|
6
|
+
|
|
7
|
+
const STANDALONE = !process.env.TEST_WORK_DIR;
|
|
8
|
+
const TEST_DIR = process.env.TEST_WORK_DIR || "/tmp/gate-viability-test";
|
|
9
|
+
const SF = `${TEST_DIR}/workflow-state.json`;
|
|
10
|
+
|
|
11
|
+
function loadRaw(): any | null {
|
|
12
|
+
if (!existsSync(SF)) return null;
|
|
13
|
+
return JSON.parse(readFileSync(SF, "utf-8"));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function sf(path: string): any {
|
|
17
|
+
const wf = loadRaw();
|
|
18
|
+
if (!wf) return undefined;
|
|
19
|
+
const parts = path.replace(/^\./, "").split(".").filter(Boolean);
|
|
20
|
+
let val: any = wf;
|
|
21
|
+
for (const p of parts) {
|
|
22
|
+
if (val == null) return undefined;
|
|
23
|
+
val = val[p];
|
|
24
|
+
}
|
|
25
|
+
return val;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function exec(cmd: string, cwd?: string): { ok: boolean; output: string } {
|
|
29
|
+
try {
|
|
30
|
+
const output = execSync(cmd, { cwd, encoding: "utf-8", timeout: 30000 }).trim();
|
|
31
|
+
return { ok: true, output };
|
|
32
|
+
} catch (e: any) {
|
|
33
|
+
return { ok: false, output: (e.stdout ?? "").trim() };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function loadProjectHarness(): any {
|
|
38
|
+
const root = sf("projectRoot");
|
|
39
|
+
if (!root) return null;
|
|
40
|
+
const p = join(root, ".claude", "rungate.json");
|
|
41
|
+
if (!existsSync(p)) return null;
|
|
42
|
+
return JSON.parse(readFileSync(p, "utf-8"));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const isVerifyPlus = () => { const p = sf("phase"); return ["VERIFY", "SHIP", "DONE"].includes(p); };
|
|
46
|
+
const isShipPlus = () => { const p = sf("phase"); return ["SHIP", "DONE"].includes(p); };
|
|
47
|
+
const isDone = () => sf("phase") === "DONE";
|
|
48
|
+
|
|
49
|
+
// ═══ SCHEMA VALIDATION ════════════════════════════════════════════════
|
|
50
|
+
describe("schema validation", () => {
|
|
51
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
52
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
test("workflow-state.json validates (includes threshold + behavioral superRefine)", () => {
|
|
56
|
+
const raw = loadRaw();
|
|
57
|
+
const result = WorkflowStateSchema.passthrough().safeParse(raw);
|
|
58
|
+
if (!result.success) {
|
|
59
|
+
const issues = result.error.issues.map(i => `${i.path.join(".")}: ${i.message}`);
|
|
60
|
+
expect(result.success, issues.join("\n")).toBe(true);
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// ═══ SCOPE CHECKS (11 functions) ══════════════════════════════════════
|
|
66
|
+
describe("scope checks", () => {
|
|
67
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
68
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
// 1. acs-exist
|
|
72
|
+
test("acs-exist: at least one AC", () => {
|
|
73
|
+
expect(sf("acs")?.length ?? 0).toBeGreaterThan(0);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// 4. acs-have-evidence-method
|
|
77
|
+
test("acs-have-evidence-method", () => {
|
|
78
|
+
const missing = (sf("acs") || []).filter((ac: any) => !ac.evidenceMethod);
|
|
79
|
+
expect(missing.map((ac: any) => ac.id)).toEqual([]);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
// 5. acs-measurable
|
|
83
|
+
test("acs-measurable: thresholds have op and value", () => {
|
|
84
|
+
const bad = (sf("acs") || []).filter((ac: any) => ac.threshold && (!ac.threshold.op || ac.threshold.value == null));
|
|
85
|
+
expect(bad.map((ac: any) => ac.id)).toEqual([]);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
// 6. source-spec-cited (GAP → PORTED)
|
|
89
|
+
test("source-spec-cited: all sourceSpecs have citedInDiscovery=true", () => {
|
|
90
|
+
const specs = sf("sourceSpecs") || [];
|
|
91
|
+
const uncited = specs.filter((s: any) => s.citedInDiscovery === false);
|
|
92
|
+
expect(uncited.map((s: any) => s.path)).toEqual([]);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// 7. spec-elements-in-acs (GAP → PORTED)
|
|
96
|
+
test("spec-elements-in-acs: if specElements exist, at least one AC references them", () => {
|
|
97
|
+
const specs = sf("sourceSpecs") || [];
|
|
98
|
+
const elements = specs.flatMap((s: any) => s.specElements || []);
|
|
99
|
+
if (elements.length === 0) return;
|
|
100
|
+
const acsWithSpec = (sf("acs") || []).filter((ac: any) => ac.specElement);
|
|
101
|
+
expect(acsWithSpec.length).toBeGreaterThan(0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// 8. sizing-declared
|
|
105
|
+
test("sizing-declared", () => {
|
|
106
|
+
expect(sf("sizing")?.predicted).toBeDefined();
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// 9. issue-goal-captured
|
|
110
|
+
test("issue-goal-captured", () => {
|
|
111
|
+
expect((sf("issueGoal") || "").length).toBeGreaterThan(0);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// 10. acs-no-skillmd-enforcement (GAP → PORTED)
|
|
115
|
+
test("acs-no-skillmd-enforcement: no ACs target SKILL.md", () => {
|
|
116
|
+
const pattern = /SKILL\.md/i;
|
|
117
|
+
const hits = (sf("acs") || []).filter((ac: any) =>
|
|
118
|
+
pattern.test(ac.statement || "") || pattern.test(ac.evidenceMethod?.command || "")
|
|
119
|
+
);
|
|
120
|
+
expect(hits.map((ac: any) => ac.id)).toEqual([]);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
// ac-issue-alignment: ACs cover issueGoal keywords (ADR-009 C4)
|
|
124
|
+
test("ac-issue-alignment: ACs cover goal keywords", () => {
|
|
125
|
+
const goal = sf("issueGoal");
|
|
126
|
+
if (!goal || typeof goal !== "string") return;
|
|
127
|
+
const STOPWORDS = new Set(["the","and","for","that","this","with","from","have","been","will","must","when","into","also","each","than"]);
|
|
128
|
+
const keywords = goal
|
|
129
|
+
.toLowerCase()
|
|
130
|
+
.replace(/[^a-z0-9\s]/g, " ")
|
|
131
|
+
.split(/\s+/)
|
|
132
|
+
.filter(w => w.length > 4 && !STOPWORDS.has(w));
|
|
133
|
+
if (keywords.length === 0) return;
|
|
134
|
+
const acText = (sf("acs") || []).map((ac: any) => (ac.statement || "").toLowerCase()).join(" ");
|
|
135
|
+
const covered = keywords.filter(kw => acText.includes(kw));
|
|
136
|
+
const coverage = covered.length / keywords.length;
|
|
137
|
+
if (coverage < 0.5) {
|
|
138
|
+
console.warn(`WARN: ac-issue-alignment: only ${Math.round(coverage * 100)}% goal keyword coverage (${covered.length}/${keywords.length})`);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
// 11. sourceSpecs-required (#473)
|
|
143
|
+
test("sourceSpecs-required: non-LIGHT tier must have sourceSpecs", () => {
|
|
144
|
+
const tier = sf("sizing.ceremonyTier");
|
|
145
|
+
if (tier === "LIGHT") return;
|
|
146
|
+
const specs = sf("sourceSpecs") || [];
|
|
147
|
+
expect(specs.length, "STANDARD/THOROUGH tier must have at least one sourceSpec").toBeGreaterThanOrEqual(1);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
// 11b. before-state-captured (#1411)
|
|
151
|
+
test("before-state-captured: STANDARD+ must have beforeState in workflow-state", () => {
|
|
152
|
+
const tier = sf("sizing.ceremonyTier");
|
|
153
|
+
const VALID_TYPES = ["text", "screenshot", "api", "none"];
|
|
154
|
+
const bs = sf("beforeState");
|
|
155
|
+
if (!bs) {
|
|
156
|
+
if (tier === "LIGHT") return;
|
|
157
|
+
console.warn("WARN: before-state-captured: beforeState missing for STANDARD+ tier — goal skill should capture before-state (#1411)");
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
expect(VALID_TYPES.includes(bs.type), `beforeState.type must be text|screenshot|api|none, got "${bs.type}"`).toBe(true);
|
|
161
|
+
if (tier !== "LIGHT" && bs.type === "none") {
|
|
162
|
+
console.warn('WARN: before-state-captured: beforeState.type is "none" for STANDARD+ tier — consider capturing actual before-state');
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// 12. discovery-evidence (#473)
|
|
167
|
+
test("discovery-evidence: non-LIGHT tiers have cited sourceSpecs", () => {
|
|
168
|
+
const tier = sf("sizing.ceremonyTier");
|
|
169
|
+
if (tier === "LIGHT") return;
|
|
170
|
+
const specs = sf("sourceSpecs") || [];
|
|
171
|
+
const cited = specs.filter((s: any) => s.citedInDiscovery);
|
|
172
|
+
expect(cited.length, "No sourceSpecs with citedInDiscovery — DISCOVERY may have been skipped").toBeGreaterThanOrEqual(1);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
// 13. governing-spec-exists (#473)
|
|
176
|
+
test("governing-spec-exists: spec file exists on disk", () => {
|
|
177
|
+
const tier = sf("sizing.ceremonyTier");
|
|
178
|
+
if (tier === "LIGHT") return;
|
|
179
|
+
const specs = sf("sourceSpecs") || [];
|
|
180
|
+
if (specs.length === 0) return; // caught by sourceSpecs-required
|
|
181
|
+
const specPath = specs[0].path;
|
|
182
|
+
const projectRoot = sf("projectRoot") || ".";
|
|
183
|
+
const resolved = specPath.startsWith("/") ? specPath : join(projectRoot, specPath);
|
|
184
|
+
expect(existsSync(resolved), `Governing spec not found: ${resolved}`).toBe(true);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
// 14. harness-port-validation: rungate.json ports match Makefile
|
|
188
|
+
test("harness-port-validation: dev/test/prod ports match Makefile", () => {
|
|
189
|
+
const projectRoot = sf("projectRoot");
|
|
190
|
+
if (!projectRoot) return;
|
|
191
|
+
const harnessPath = join(projectRoot, ".claude", "rungate.json");
|
|
192
|
+
const makefilePath = join(projectRoot, "Makefile");
|
|
193
|
+
if (!existsSync(harnessPath) || !existsSync(makefilePath)) return;
|
|
194
|
+
const harness = JSON.parse(readFileSync(harnessPath, "utf-8"));
|
|
195
|
+
const makefile = readFileSync(makefilePath, "utf-8");
|
|
196
|
+
|
|
197
|
+
const warnings: string[] = [];
|
|
198
|
+
|
|
199
|
+
// Check dev port — match PORT= in the dev-all/dev section of Makefile
|
|
200
|
+
const devPort = harness.dev?.apiBase?.match(/:(\d+)/)?.[1];
|
|
201
|
+
if (devPort) {
|
|
202
|
+
const devSection = makefile.match(/dev.*environment[^]*?PORT=(\d+)/i);
|
|
203
|
+
const makeDevPort = devSection?.[1];
|
|
204
|
+
if (makeDevPort && devPort !== makeDevPort) {
|
|
205
|
+
warnings.push(`dev.apiBase port ${devPort} != Makefile dev PORT ${makeDevPort}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Check prod port — match PORT= in container run commands
|
|
210
|
+
const prodPort = harness.prod?.apiBase?.match(/:(\d+)/)?.[1];
|
|
211
|
+
if (prodPort) {
|
|
212
|
+
const makeProdPort = makefile.match(/-e PORT=(\d+)/)?.[1];
|
|
213
|
+
if (makeProdPort && prodPort !== makeProdPort) {
|
|
214
|
+
warnings.push(`prod.apiBase port ${prodPort} != Makefile container PORT ${makeProdPort}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (warnings.length > 0) {
|
|
219
|
+
console.warn(`WARN: harness-port-validation: ${warnings.join("; ")}`);
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
// ═══ VERIFY CHECKS (16 functions) ═════════════════════════════════════
|
|
225
|
+
describe("verify checks", () => {
|
|
226
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
227
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
// 12. local-before-prod
|
|
231
|
+
test("local-before-prod: local validated before prod rebuild", () => {
|
|
232
|
+
const prodRebuild = sf("environments")?.prod?.rebuild;
|
|
233
|
+
if (!prodRebuild || prodRebuild === "SKIP") return;
|
|
234
|
+
const localApi = sf("environments")?.local?.api;
|
|
235
|
+
const localTests = sf("environments")?.local?.tests;
|
|
236
|
+
if (localApi !== "PASS" && localTests !== "PASS") {
|
|
237
|
+
expect("prod deployed without local validation").toBe("local.api or local.tests = PASS first");
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
// 13. container-has-fix — OCI label SHA comparison (#437)
|
|
242
|
+
test("container-has-fix: container image SHA matches HEAD", () => {
|
|
243
|
+
if (!isShipPlus()) return;
|
|
244
|
+
const harness = loadProjectHarness();
|
|
245
|
+
if (!harness?.prod?.rebuild) return;
|
|
246
|
+
const prodRebuild = sf("environments")?.prod?.rebuild;
|
|
247
|
+
const root = sf("projectRoot");
|
|
248
|
+
if (!root) return;
|
|
249
|
+
const containerName = harness.prod.containerName || harness.prod.imageName;
|
|
250
|
+
if (!containerName) return;
|
|
251
|
+
const runtime = exec("command -v podman") .ok ? "podman" : "docker";
|
|
252
|
+
const inspect = exec(`${runtime} inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' ${containerName}`);
|
|
253
|
+
if (!inspect.ok || !inspect.output) return;
|
|
254
|
+
const containerSha = inspect.output.replace(/'/g, "").trim();
|
|
255
|
+
const head = exec("git rev-parse HEAD", root);
|
|
256
|
+
if (!head.ok) return;
|
|
257
|
+
if (containerSha !== head.output) {
|
|
258
|
+
if (prodRebuild !== "PASS") {
|
|
259
|
+
expect(`container SHA ${containerSha.slice(0,8)} behind HEAD ${head.output.slice(0,8)} and rebuild not PASS`).toBe("container SHA matches HEAD or rebuild PASS");
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
// 14. all-acs-have-evidence (GAP → PORTED)
|
|
265
|
+
// OUTCOME ACs may not have mechanical evidence at ship time — verified at prove via Quinn
|
|
266
|
+
test("all-acs-have-evidence: every AC has evidence in verify+", () => {
|
|
267
|
+
if (!isVerifyPlus()) return;
|
|
268
|
+
const missing = (sf("acs") || []).filter((ac: any) => ac.evidence == null && ac.type !== "OUTCOME");
|
|
269
|
+
expect(missing.map((ac: any) => ac.id)).toEqual([]);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
// 15. all-acs-pass
|
|
273
|
+
// OUTCOME ACs may be PENDING at ship time — they require Quinn/prove for verification
|
|
274
|
+
test("all-acs-pass: every AC has verdict in ship+", () => {
|
|
275
|
+
if (!isShipPlus()) return;
|
|
276
|
+
const noVerdict = (sf("acs") || []).filter((ac: any) => (!ac.verdict || ac.verdict === "PENDING") && ac.type !== "OUTCOME");
|
|
277
|
+
expect(noVerdict.map((ac: any) => ac.id)).toEqual([]);
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
// 15b. outcome-acs-no-skip: OUTCOME ACs must be verified, not skipped (#90 RCA)
|
|
281
|
+
test("outcome-acs-no-skip: OUTCOME ACs cannot have SKIP verdict at ship+", () => {
|
|
282
|
+
if (!isShipPlus()) return;
|
|
283
|
+
const skipped = (sf("acs") || []).filter((ac: any) => ac.type === "OUTCOME" && ac.verdict === "SKIP");
|
|
284
|
+
expect(skipped.map((ac: any) => ac.id), "OUTCOME ACs require live verification — start dev server and run the feature (#90 RCA)").toEqual([]);
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
// 15c. outcome-evidence-behavioral: OUTCOME evidence is not just a grep count (#90 RCA)
|
|
288
|
+
test("outcome-evidence-behavioral: OUTCOME AC evidence is not just a number", () => {
|
|
289
|
+
if (!isShipPlus()) return;
|
|
290
|
+
const outcomes = (sf("acs") || []).filter((ac: any) => ac.type === "OUTCOME" && ac.verdict === "PASS");
|
|
291
|
+
const numericOnly = outcomes.filter((ac: any) => {
|
|
292
|
+
const content = ac.evidence?.content || "";
|
|
293
|
+
return /^\d+$/.test(content.trim());
|
|
294
|
+
});
|
|
295
|
+
if (numericOnly.length > 0) {
|
|
296
|
+
console.warn(`WARN: ${numericOnly.length} OUTCOME AC(s) have numeric-only evidence — behavioral evidence expected`);
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test("no FAIL verdicts in DONE", () => {
|
|
301
|
+
if (!isDone()) return;
|
|
302
|
+
const fails = (sf("acs") || []).filter((ac: any) => ac.verdict === "FAIL");
|
|
303
|
+
expect(fails.map((ac: any) => ac.id)).toEqual([]);
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
// 16. evidence-type-valid (GAP → PORTED)
|
|
307
|
+
test("evidence-type-valid: evidence.type matches allowed enum", () => {
|
|
308
|
+
if (!isVerifyPlus()) return;
|
|
309
|
+
const VALID = ["file-citation", "grep-output", "api-response", "screenshot", "test-output", "command-output", "manual-attestation", "grep"];
|
|
310
|
+
const bad = (sf("acs") || []).filter((ac: any) => {
|
|
311
|
+
if (!ac.evidence || typeof ac.evidence !== "object") return false;
|
|
312
|
+
return !VALID.includes(ac.evidence.type);
|
|
313
|
+
});
|
|
314
|
+
expect(bad.map((ac: any) => `${ac.id}: ${ac.evidence?.type}`)).toEqual([]);
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// 17. evidence-type-ratio (GAP → PORTED, ADR-009 A5: >= 50% behavioral evidence required)
|
|
318
|
+
test("evidence-type-ratio: <50% pattern-only evidence (ADR-009 A5)", () => {
|
|
319
|
+
if (!isVerifyPlus()) return;
|
|
320
|
+
const acs = (sf("acs") || []).filter((ac: any) => ac.evidenceMethod);
|
|
321
|
+
if (acs.length === 0) return;
|
|
322
|
+
const patternOnly = acs.filter((ac: any) => {
|
|
323
|
+
const t = ac.evidenceMethod?.type;
|
|
324
|
+
return !t || t === "pattern" || t === "GREP_CHECK" || t === "grep";
|
|
325
|
+
});
|
|
326
|
+
const ratio = patternOnly.length / acs.length;
|
|
327
|
+
// ADR-009 A5: at least 50% behavioral evidence required
|
|
328
|
+
expect(ratio, `${patternOnly.length}/${acs.length} (${Math.round(ratio * 100)}%) pattern-only — must be < 50% (ADR-009 A5)`).toBeLessThanOrEqual(0.5);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
// 18. evidence-method-executed (GAP → PORTED)
|
|
332
|
+
test("evidence-method-executed: COMMAND ACs have evidence content", () => {
|
|
333
|
+
if (!isVerifyPlus()) return;
|
|
334
|
+
const missing = (sf("acs") || []).filter((ac: any) => {
|
|
335
|
+
const t = ac.evidenceMethod?.type;
|
|
336
|
+
if (t !== "COMMAND" && t !== "command") return false;
|
|
337
|
+
return !ac.evidence || (typeof ac.evidence === "object" && !ac.evidence.content);
|
|
338
|
+
});
|
|
339
|
+
expect(missing.map((ac: any) => ac.id)).toEqual([]);
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
// 19. threshold-crosscheck (GAP → PORTED)
|
|
343
|
+
test("threshold-crosscheck: evidence values meet threshold", () => {
|
|
344
|
+
if (!isVerifyPlus()) return;
|
|
345
|
+
const violations: string[] = [];
|
|
346
|
+
for (const ac of sf("acs") || []) {
|
|
347
|
+
if (!ac.threshold || !ac.evidence) continue;
|
|
348
|
+
const val = typeof ac.evidence === "object" ? ac.evidence.value : null;
|
|
349
|
+
if (val == null) continue;
|
|
350
|
+
const exp = parseFloat(String(ac.threshold.value));
|
|
351
|
+
if (isNaN(exp) || isNaN(val)) continue;
|
|
352
|
+
let pass = true;
|
|
353
|
+
switch (ac.threshold.op) {
|
|
354
|
+
case ">=": pass = val >= exp; break;
|
|
355
|
+
case "<=": pass = val <= exp; break;
|
|
356
|
+
case ">": pass = val > exp; break;
|
|
357
|
+
case "<": pass = val < exp; break;
|
|
358
|
+
case "==": pass = val === exp; break;
|
|
359
|
+
case "!=": pass = val !== exp; break;
|
|
360
|
+
}
|
|
361
|
+
if (!pass) violations.push(`${ac.id}: ${val} ${ac.threshold.op} ${exp} = false`);
|
|
362
|
+
}
|
|
363
|
+
expect(violations).toEqual([]);
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
// 20. tests-pass (GAP → PORTED)
|
|
367
|
+
test("tests-pass: test suite passes", () => {
|
|
368
|
+
if (!isVerifyPlus()) return;
|
|
369
|
+
const harness = loadProjectHarness();
|
|
370
|
+
if (!harness?.dev?.testCmd) return;
|
|
371
|
+
const val = sf("environments")?.local?.tests;
|
|
372
|
+
if (!val) {
|
|
373
|
+
expect("local.tests not set").toBe("PASS or SKIP — required when dev.testCmd is configured");
|
|
374
|
+
} else if (val === "SKIP") {
|
|
375
|
+
// SKIP is allowed
|
|
376
|
+
} else if (val !== "PASS") {
|
|
377
|
+
expect(`tests: ${val}`).toBe("PASS");
|
|
378
|
+
}
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
// 21. tsc-pass (GAP → PORTED)
|
|
382
|
+
test("tsc-pass: no new TypeScript errors above baseline", () => {
|
|
383
|
+
if (!isVerifyPlus()) return;
|
|
384
|
+
const root = sf("projectRoot");
|
|
385
|
+
if (!root || !existsSync(join(root, "tsconfig.json"))) return;
|
|
386
|
+
// Check if Marcus recorded it
|
|
387
|
+
const tests = sf("environments")?.local?.tests;
|
|
388
|
+
if (tests === "PASS" || tests === "SKIP") return; // trust Marcus
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
// 22. code-committed (GAP → PORTED)
|
|
392
|
+
test("code-committed: no uncommitted changes in project", () => {
|
|
393
|
+
if (!isShipPlus()) return;
|
|
394
|
+
const root = sf("projectRoot");
|
|
395
|
+
if (!root) return;
|
|
396
|
+
const harness = loadProjectHarness();
|
|
397
|
+
const paths = harness?.codeCommittedPaths || [];
|
|
398
|
+
if (paths.length === 0) return;
|
|
399
|
+
const result = exec(`git status --porcelain -- ${paths.join(" ")}`, root);
|
|
400
|
+
const dirty = result.output.split("\n").filter(Boolean).length;
|
|
401
|
+
if (dirty > 0) {
|
|
402
|
+
expect(`${dirty} uncommitted files`).toBe("0 uncommitted files");
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
// 23. code-pushed (GAP → PORTED)
|
|
407
|
+
test("code-pushed: no unpushed commits", () => {
|
|
408
|
+
if (!isShipPlus()) return;
|
|
409
|
+
const root = sf("projectRoot");
|
|
410
|
+
if (!root) return;
|
|
411
|
+
const result = exec("git rev-list --count @{upstream}..HEAD", root);
|
|
412
|
+
if (!result.ok) return; // no upstream
|
|
413
|
+
if (result.output !== "0") {
|
|
414
|
+
expect(`${result.output} commits ahead`).toBe("0 commits ahead");
|
|
415
|
+
}
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
// 25. local-api-validated (GAP → PORTED)
|
|
419
|
+
test("local-api-validated: environments.local.api set", () => {
|
|
420
|
+
if (!isVerifyPlus()) return;
|
|
421
|
+
const harness = loadProjectHarness();
|
|
422
|
+
if (!harness?.dev?.apiBase) return;
|
|
423
|
+
const val = sf("environments")?.local?.api;
|
|
424
|
+
if (!val) {
|
|
425
|
+
expect("local.api not set").toBe("PASS or SKIP — required when dev.apiBase is configured");
|
|
426
|
+
} else if (val === "SKIP") {
|
|
427
|
+
// SKIP is allowed
|
|
428
|
+
} else if (val !== "PASS") {
|
|
429
|
+
expect(`api: ${val}`).toBe("PASS");
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
// 26. local-ui-validated (GAP → PORTED)
|
|
434
|
+
test("local-ui-validated: environments.local.ui set with skipReason", () => {
|
|
435
|
+
if (!isVerifyPlus()) return;
|
|
436
|
+
const harness = loadProjectHarness();
|
|
437
|
+
if (!harness?.dev?.uiBase) return;
|
|
438
|
+
const val = sf("environments")?.local?.ui;
|
|
439
|
+
if (!val) {
|
|
440
|
+
expect("local.ui not set").toBe("PASS or SKIP — required when dev.uiBase is configured");
|
|
441
|
+
} else if (val === "SKIP") {
|
|
442
|
+
const reason = sf("environments")?.local?.uiSkipReason;
|
|
443
|
+
if (!reason) {
|
|
444
|
+
expect("UI SKIP without reason").toBe("uiSkipReason required when SKIP");
|
|
445
|
+
}
|
|
446
|
+
} else if (val !== "PASS") {
|
|
447
|
+
expect(`ui: ${val}`).toBe("PASS");
|
|
448
|
+
}
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
// 29. quinn-on-ui-change
|
|
452
|
+
test("quinn-on-ui-change: quinn spawned when UI files change", () => {
|
|
453
|
+
if (!isShipPlus()) return;
|
|
454
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
455
|
+
const UI_SVC = /template|campaign-html|brief-pipeline|email-template|dashboard/i;
|
|
456
|
+
const allPaths: string[] = [];
|
|
457
|
+
for (const ac of sf("acs") || []) {
|
|
458
|
+
if (ac.statement) allPaths.push(ac.statement);
|
|
459
|
+
if (ac.evidenceMethod?.command) allPaths.push(ac.evidenceMethod.command);
|
|
460
|
+
for (const cf of ac.contextFiles || []) {
|
|
461
|
+
if (cf?.path) allPaths.push(cf.path);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
// Strip absolute project paths to avoid false positives (e.g. /DailyBriefDashboard/ matching "dashboard")
|
|
465
|
+
const projectRoot = sf("projectRoot") || "";
|
|
466
|
+
const joined = allPaths.join(" ").replaceAll(projectRoot, "");
|
|
467
|
+
if (!UI_EXT.test(joined) && !UI_SVC.test(joined)) return;
|
|
468
|
+
const quinn = sf("agents")?.quinn;
|
|
469
|
+
if (!quinn?.spawned) {
|
|
470
|
+
expect("quinn not spawned for UI change").toBe("quinn.spawned required");
|
|
471
|
+
}
|
|
472
|
+
// Quinn depth check: spawned but no testedPaths = code review only (ADR-009 A3)
|
|
473
|
+
// If Quinn has verdict PASS, accept that as sufficient evidence (ship.js doesn't write testedPaths yet)
|
|
474
|
+
if (quinn?.spawned && quinn.verdict !== 'PASS') {
|
|
475
|
+
const testedPaths = quinn.testedPaths;
|
|
476
|
+
expect(Array.isArray(testedPaths) && testedPaths.length > 0, "quinn.testedPaths must not be empty — code review only does not satisfy UI testing (ADR-009 A3)").toBe(true);
|
|
477
|
+
}
|
|
478
|
+
// Quinn screenshot check: WARN when verdict=PASS but no screenshots (ship.js Quinn doesn't capture yet)
|
|
479
|
+
if (quinn?.spawned && quinn.verdict !== 'PASS') {
|
|
480
|
+
const screenshots = quinn.screenshots || [];
|
|
481
|
+
const hasRealScreenshots = screenshots.length > 0 && screenshots.some((s: string) => existsSync(s));
|
|
482
|
+
if (!hasRealScreenshots) {
|
|
483
|
+
expect("quinn.screenshots empty or files missing").toBe("at least 1 screenshot file on disk — browser verification required for UI changes (#90 RCA)");
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
// quinn-container-port: Quinn must test on container :7776, not dev :7778 (#1371)
|
|
489
|
+
test("quinn-container-port: Quinn tested on container port 7776", () => {
|
|
490
|
+
const quinn = sf("agents")?.quinn;
|
|
491
|
+
if (!quinn?.spawned) return;
|
|
492
|
+
const agentPort = quinn.port;
|
|
493
|
+
const envPort = sf("environments")?.local?.quinn?.port;
|
|
494
|
+
const port = agentPort || envPort;
|
|
495
|
+
if (!port) {
|
|
496
|
+
console.warn("WARN: quinn-container-port: no port recorded in quinn agent data — cannot verify container testing");
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
if (port !== 7776) {
|
|
500
|
+
console.warn(`WARN: quinn-container-port: Quinn tested on port ${port}, expected 7776 (container). Testing on dev (:7778) does not validate container behavior.`);
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
// brief-ac-alignment: Marcus brief contains all AC ids (ADR-009 C2)
|
|
505
|
+
test("brief-ac-alignment: brief covers all ACs", () => {
|
|
506
|
+
if (!isVerifyPlus()) return;
|
|
507
|
+
const slug = sf("slug");
|
|
508
|
+
if (!slug) return;
|
|
509
|
+
const briefPath = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "marcus-brief.md");
|
|
510
|
+
if (!existsSync(briefPath)) {
|
|
511
|
+
// Brief file must exist for STANDARD+ tiers when marcus was spawned
|
|
512
|
+
const marcus = sf("agents")?.marcus;
|
|
513
|
+
if (marcus?.spawned) {
|
|
514
|
+
expect("marcus-brief.md missing").toBe("brief file required when Marcus spawned (ADR-009 C2)");
|
|
515
|
+
}
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
const brief = readFileSync(briefPath, "utf-8");
|
|
519
|
+
const acs = sf("acs") || [];
|
|
520
|
+
const missingIds = acs
|
|
521
|
+
.map((ac: any) => ac.id)
|
|
522
|
+
.filter((id: string) => !brief.includes(id));
|
|
523
|
+
expect(missingIds, `brief-ac-alignment: brief missing AC ids: ${missingIds.join(", ")}`).toEqual([]);
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// ac-spec-traceability (#440)
|
|
527
|
+
test("ac-spec-traceability: AC specElements exist in governing spec", () => {
|
|
528
|
+
if (!isVerifyPlus()) return;
|
|
529
|
+
const specPath = sf("governingSpec")?.path;
|
|
530
|
+
if (!specPath || !existsSync(specPath)) return;
|
|
531
|
+
const specContent = readFileSync(specPath, "utf-8");
|
|
532
|
+
const missing: string[] = [];
|
|
533
|
+
for (const ac of sf("acs") || []) {
|
|
534
|
+
if (!ac.specElement) continue;
|
|
535
|
+
if (!specContent.includes(ac.specElement)) {
|
|
536
|
+
missing.push(`${ac.id}: specElement "${ac.specElement}" not found in spec`);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
expect(missing).toEqual([]);
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
// 28. marcus-committed (GAP → PORTED)
|
|
543
|
+
test("marcus-committed: no uncommitted Marcus changes", () => {
|
|
544
|
+
if (!isVerifyPlus()) return;
|
|
545
|
+
const signalDir = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), "signals");
|
|
546
|
+
const slug = sf("slug") || "";
|
|
547
|
+
const signalFile = join(signalDir, `marcus-uncommitted-${slug.replace(/\//g, "-")}`);
|
|
548
|
+
if (existsSync(signalFile)) {
|
|
549
|
+
expect("marcus left uncommitted files").toBe("marcus committed all changes");
|
|
550
|
+
}
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
// prove-evidence-type: UI changes require screenshot/api-response evidence (ADR-009 A4)
|
|
554
|
+
test("prove-evidence-type: UI changes require live evidence", () => {
|
|
555
|
+
if (!isVerifyPlus()) return;
|
|
556
|
+
const root = sf("projectRoot");
|
|
557
|
+
if (!root) return;
|
|
558
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
559
|
+
if (!scopeSha) return;
|
|
560
|
+
const diff = exec(`git diff --name-only ${scopeSha}..HEAD`, root);
|
|
561
|
+
if (!diff.ok || !diff.output) return;
|
|
562
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
563
|
+
const uiChanged = diff.output.split("\n").filter(f => UI_EXT.test(f));
|
|
564
|
+
if (uiChanged.length === 0) return;
|
|
565
|
+
const slug = sf("slug");
|
|
566
|
+
if (!slug) return;
|
|
567
|
+
const evidencePath = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "prove-evidence.json");
|
|
568
|
+
if (!existsSync(evidencePath)) return;
|
|
569
|
+
const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
|
|
570
|
+
const afterType = evidence.afterEvidence?.type;
|
|
571
|
+
const VALID_UI_EVIDENCE = ["screenshot", "api-response"];
|
|
572
|
+
expect(afterType, "afterEvidence.type required for UI changes (ADR-009 A4)").toBeTruthy();
|
|
573
|
+
expect(VALID_UI_EVIDENCE.includes(afterType), `afterEvidence.type must be screenshot or api-response for UI changes, got "${afterType}" (ADR-009 A4)`).toBe(true);
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
// acHash consistency: verify gate checks scope's AC hash (ADR-009)
|
|
577
|
+
test("acHash-consistency-verify: ACs unchanged since scope", () => {
|
|
578
|
+
if (!isVerifyPlus()) return;
|
|
579
|
+
const acHash = sf("gates")?.scope?.acHash;
|
|
580
|
+
if (!acHash) return;
|
|
581
|
+
const acDefs = (sf("acs") || []).map((ac: any) => ({
|
|
582
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
583
|
+
specElement: ac.specElement, threshold: ac.threshold,
|
|
584
|
+
evidenceMethod: ac.evidenceMethod,
|
|
585
|
+
}));
|
|
586
|
+
const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
|
|
587
|
+
expect(currentHash, "ACs modified after scope approval — re-run scope gate (ADR-009 acHash)").toBe(acHash);
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
// spec-decision-compliance (#409, Decision #9: WARN tier — console.warn only, no expect failures)
|
|
591
|
+
test("spec-decision-compliance: AC language aligns with spec decisions", () => {
|
|
592
|
+
if (!isVerifyPlus()) return;
|
|
593
|
+
const localPolicies = join(__dirname, "spec-policies.json");
|
|
594
|
+
const homePolicies = join(process.env.HOME || "", ".claude", "skills", "ship", "spec-policies.json");
|
|
595
|
+
const policiesPath = existsSync(localPolicies) ? localPolicies : homePolicies;
|
|
596
|
+
if (!existsSync(policiesPath)) return;
|
|
597
|
+
const policies = JSON.parse(readFileSync(policiesPath, "utf-8"));
|
|
598
|
+
const patterns = policies.implementationPatterns;
|
|
599
|
+
if (!patterns) return;
|
|
600
|
+
const acs = sf("acs") || [];
|
|
601
|
+
let warnCount = 0;
|
|
602
|
+
for (const [key, rule] of Object.entries(patterns) as [string, any][]) {
|
|
603
|
+
if (!rule.antiPatterns || !rule.decision) continue;
|
|
604
|
+
const scopes: string[] = rule.scope || ["statement"];
|
|
605
|
+
for (const ac of acs) {
|
|
606
|
+
for (const scope of scopes) {
|
|
607
|
+
const text = scope === "statement" ? (ac.statement || "") : (ac.evidenceMethod?.command || "");
|
|
608
|
+
if (!text) continue;
|
|
609
|
+
for (const ap of rule.antiPatterns) {
|
|
610
|
+
if (new RegExp(ap, "i").test(text)) {
|
|
611
|
+
console.warn(`WARN: spec-decision-compliance [${key}]: AC "${ac.id}" matches anti-pattern "${ap}" — ${rule.message}`);
|
|
612
|
+
warnCount++;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
if (warnCount > 0) {
|
|
619
|
+
console.warn(`WARN: spec-decision-compliance: ${warnCount} anti-pattern match(es) found — review recommended`);
|
|
620
|
+
}
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
// contextual-skip-validation (#91 → ADR-009 A1: SKIP → FAIL for file-matched gates)
|
|
624
|
+
test("contextual-skip-validation: SKIP blocked when changed files require gate", () => {
|
|
625
|
+
if (!isVerifyPlus()) return;
|
|
626
|
+
const root = sf("projectRoot");
|
|
627
|
+
if (!root) return;
|
|
628
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
629
|
+
if (!scopeSha) return;
|
|
630
|
+
const diff = exec(`git diff --name-only ${scopeSha}..HEAD`, root);
|
|
631
|
+
if (!diff.ok || !diff.output) return; // no changed files — silently return
|
|
632
|
+
|
|
633
|
+
const FILE_GATE_MAP: Record<string, string[]> = {
|
|
634
|
+
"tsx|jsx": ["local-ui-validated", "quinn-on-ui-change"],
|
|
635
|
+
"css|scss|less|html|svg": ["local-ui-validated"],
|
|
636
|
+
"sql|migration": ["local-api-validated"],
|
|
637
|
+
"Dockerfile|docker-compose|Makefile": ["prod-rebuild-pass"],
|
|
638
|
+
};
|
|
639
|
+
|
|
640
|
+
const ENV_GATE_PATH: Record<string, string> = {
|
|
641
|
+
"local-ui-validated": "environments.local.ui",
|
|
642
|
+
"local-api-validated": "environments.local.api",
|
|
643
|
+
"quinn-on-ui-change": "agents.quinn.spawned",
|
|
644
|
+
"prod-rebuild-pass": "environments.prod.rebuild",
|
|
645
|
+
};
|
|
646
|
+
|
|
647
|
+
const changedFiles = diff.output.split("\n").filter(Boolean);
|
|
648
|
+
let warnCount = 0;
|
|
649
|
+
for (const file of changedFiles) {
|
|
650
|
+
for (const [extPattern, gates] of Object.entries(FILE_GATE_MAP)) {
|
|
651
|
+
const re = new RegExp(`\\.(${extPattern})$`, "i");
|
|
652
|
+
const nameRe = new RegExp(`(${extPattern})`, "i");
|
|
653
|
+
if (!re.test(file) && !nameRe.test(file.split("/").pop() || "")) continue;
|
|
654
|
+
for (const gate of gates) {
|
|
655
|
+
const envPath = ENV_GATE_PATH[gate];
|
|
656
|
+
if (!envPath) continue;
|
|
657
|
+
const val = sf(envPath);
|
|
658
|
+
if (val === "SKIP") {
|
|
659
|
+
expect(`contextual-skip-blocked: "${file}" changed but ${gate} is SKIP`).toBe("gate must not be SKIP when matched files changed (ADR-009 A1)");
|
|
660
|
+
warnCount++;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (warnCount > 0) {
|
|
666
|
+
expect(`contextual-skip-validation: ${warnCount} SKIP(s) on file-matched gates`).toBe("contextual-skip-validation: 0 SKIPs (ADR-009 A1)");
|
|
667
|
+
}
|
|
668
|
+
});
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
// ═══ B1/B2 AGENT RESULTS (#1406, #1407) ═════════════════════════════
|
|
672
|
+
describe("B1/B2 agent results", () => {
|
|
673
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
674
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
// B2: Evidence Validator prompt file exists
|
|
678
|
+
test("b2-prompt-exists: evidence-validator.md available", () => {
|
|
679
|
+
const promptPath = join(__dirname, "prompts", "evidence-validator.md");
|
|
680
|
+
expect(
|
|
681
|
+
existsSync(promptPath),
|
|
682
|
+
"evidence-validator.md prompt not found in gates/prompts/",
|
|
683
|
+
).toBe(true);
|
|
684
|
+
});
|
|
685
|
+
|
|
686
|
+
// B1: AC Adversary prompt file exists
|
|
687
|
+
test("b1-prompt-exists: ac-adversary.md available", () => {
|
|
688
|
+
const promptPath = join(__dirname, "prompts", "ac-adversary.md");
|
|
689
|
+
expect(
|
|
690
|
+
existsSync(promptPath),
|
|
691
|
+
"ac-adversary.md prompt not found in gates/prompts/",
|
|
692
|
+
).toBe(true);
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
// B2: Evidence Validator results structure (when present)
|
|
696
|
+
test("b2-evidence-validator-structure: results valid when populated", () => {
|
|
697
|
+
if (!isVerifyPlus()) return;
|
|
698
|
+
const ev = sf("gates.verify.evidenceValidator");
|
|
699
|
+
if (!ev) {
|
|
700
|
+
console.warn("WARN: gates.verify.evidenceValidator not populated — B2 agent may not have run");
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
703
|
+
expect(ev.ts, "evidenceValidator.ts must be an ISO timestamp").toBeTruthy();
|
|
704
|
+
expect(Array.isArray(ev.verdicts), "evidenceValidator.verdicts must be an array").toBe(true);
|
|
705
|
+
});
|
|
706
|
+
|
|
707
|
+
// B1: AC Adversary results structure (when present)
|
|
708
|
+
test("b1-adversary-verify-structure: results valid when populated", () => {
|
|
709
|
+
if (!isVerifyPlus()) return;
|
|
710
|
+
const adv = sf("gates.verify.adversary");
|
|
711
|
+
if (!adv) {
|
|
712
|
+
console.warn("WARN: gates.verify.adversary not populated — B1 verify agent may not have run");
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
expect(adv.ts, "adversary.ts must be an ISO timestamp").toBeTruthy();
|
|
716
|
+
expect(typeof adv.gameable, "adversary.gameable must be a number").toBe("number");
|
|
717
|
+
expect(typeof adv.approved, "adversary.approved must be a boolean").toBe("boolean");
|
|
718
|
+
});
|
|
719
|
+
});
|
|
720
|
+
|
|
721
|
+
// ═══ SPEC TRACEABILITY (2 functions) ══════════════════════════════════
|
|
722
|
+
describe("spec traceability", () => {
|
|
723
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
724
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
// 31. decision-id-valid (GAP → PORTED)
|
|
728
|
+
test("decision-id-valid: D-NNN IDs in ACs exist in governing spec", () => {
|
|
729
|
+
const specPath = sf("sourceSpecs")?.[0]?.path || sf("governingSpec")?.path;
|
|
730
|
+
if (!specPath || !existsSync(specPath)) return;
|
|
731
|
+
const spec = readFileSync(specPath, "utf-8");
|
|
732
|
+
const ids = new Set<string>();
|
|
733
|
+
for (const ac of sf("acs") || []) {
|
|
734
|
+
const matches = (ac.statement || "").match(/D-\d+/g) || [];
|
|
735
|
+
matches.forEach((id: string) => ids.add(id));
|
|
736
|
+
}
|
|
737
|
+
if (ids.size === 0) return;
|
|
738
|
+
const missing = [...ids].filter(id => !spec.includes(id));
|
|
739
|
+
expect(missing).toEqual([]);
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
// 32. evidence-target-valid (GAP → PORTED)
|
|
743
|
+
test("evidence-target-valid: evidence commands target files within scope", () => {
|
|
744
|
+
const harness = loadProjectHarness();
|
|
745
|
+
const paths = harness?.codeCommittedPaths || [];
|
|
746
|
+
if (paths.length === 0) return;
|
|
747
|
+
const commands = (sf("acs") || [])
|
|
748
|
+
.map((ac: any) => ac.evidenceMethod?.command || "")
|
|
749
|
+
.filter(Boolean);
|
|
750
|
+
if (commands.length === 0) return;
|
|
751
|
+
const fileRefs = commands.join("\n").match(/[^\s]+(\/[^\s]+|\.[a-z]+)/g) || [];
|
|
752
|
+
const outside = fileRefs.filter(ref => {
|
|
753
|
+
if (/^-/.test(ref) || /^\|/.test(ref)) return false;
|
|
754
|
+
return !paths.some((p: string) => ref.includes(p));
|
|
755
|
+
});
|
|
756
|
+
// Informational — outside refs are common (test fixtures, config)
|
|
757
|
+
});
|
|
758
|
+
});
|
|
759
|
+
|
|
760
|
+
// ═══ SHIP CHECKS (7 functions) ════════════════════════════════════════
|
|
761
|
+
describe("ship checks", () => {
|
|
762
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
763
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
// 35. all-gates-pass (GAP → PORTED)
|
|
767
|
+
test("all-gates-pass: scope + verify gates PASS", () => {
|
|
768
|
+
if (!isShipPlus()) return;
|
|
769
|
+
const scope = sf("gates")?.scope?.result;
|
|
770
|
+
const verify = sf("gates")?.verify?.result;
|
|
771
|
+
if (scope && scope !== "PASS") {
|
|
772
|
+
expect(`scope gate: ${scope}`).toBe("PASS");
|
|
773
|
+
}
|
|
774
|
+
if (verify && verify !== "PASS") {
|
|
775
|
+
expect(`verify gate: ${verify}`).toBe("PASS");
|
|
776
|
+
}
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
// 36. branch-merged / code-pushed (GAP → PORTED)
|
|
780
|
+
// Ship gate: code pushed + PR exists = PASS (merge is post-ship, verified at prove)
|
|
781
|
+
// Prove gate: merged to main = PASS
|
|
782
|
+
test("branch-merged: fix is on main", () => {
|
|
783
|
+
if (!isShipPlus()) return;
|
|
784
|
+
const root = sf("projectRoot");
|
|
785
|
+
if (!root) return;
|
|
786
|
+
const gate = process.env.GATE || "";
|
|
787
|
+
if (gate === "ship") {
|
|
788
|
+
// At ship gate: check code is pushed, not merged
|
|
789
|
+
const commit = sf("buildCommit");
|
|
790
|
+
const agents = sf("agents");
|
|
791
|
+
const pushed = agents?.marcus?.branch;
|
|
792
|
+
if (!commit && !pushed) {
|
|
793
|
+
expect("no commit or branch").toBe("buildCommit or marcus.branch required at ship gate");
|
|
794
|
+
}
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
// At prove gate or verify: check merged to main
|
|
798
|
+
const result = exec("git branch --contains HEAD --list main", root);
|
|
799
|
+
if (result.ok && !result.output.includes("main")) {
|
|
800
|
+
expect("not on main").toBe("merged to main");
|
|
801
|
+
}
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
// 38. prod-rebuild-pass (GAP → PORTED)
|
|
805
|
+
test("prod-rebuild-pass: prod rebuild required when configured", () => {
|
|
806
|
+
if (!isShipPlus()) return;
|
|
807
|
+
const harness = loadProjectHarness();
|
|
808
|
+
if (!harness?.prod?.rebuild) return;
|
|
809
|
+
const val = sf("environments")?.prod?.rebuild;
|
|
810
|
+
if (!val) {
|
|
811
|
+
expect("prod.rebuild not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
|
|
812
|
+
} else if (val === "SKIP") {
|
|
813
|
+
const reason = sf("environments")?.prod?.rebuildSkipReason;
|
|
814
|
+
if (!reason) expect("rebuild SKIP without reason").toBe("rebuildSkipReason required");
|
|
815
|
+
} else if (val !== "PASS") {
|
|
816
|
+
expect(`prod rebuild: ${val}`).toBe("PASS or SKIP with reason");
|
|
817
|
+
}
|
|
818
|
+
});
|
|
819
|
+
|
|
820
|
+
// 39. prod-smoke-pass (GAP → PORTED)
|
|
821
|
+
test("prod-smoke-pass: prod smoke required when configured", () => {
|
|
822
|
+
if (!isShipPlus()) return;
|
|
823
|
+
const harness = loadProjectHarness();
|
|
824
|
+
if (!harness?.prod?.smokeTest) return;
|
|
825
|
+
const val = sf("environments")?.prod?.smoke;
|
|
826
|
+
if (!val) {
|
|
827
|
+
expect("prod.smoke not set").toBe("PASS or SKIP with reason — required when prod.smokeTest is configured");
|
|
828
|
+
} else if (val === "SKIP") {
|
|
829
|
+
const reason = sf("environments")?.prod?.smokeSkipReason;
|
|
830
|
+
if (!reason) expect("smoke SKIP without reason").toBe("smokeSkipReason required");
|
|
831
|
+
} else if (val !== "PASS") {
|
|
832
|
+
expect(`prod smoke: ${val}`).toBe("PASS or SKIP with reason");
|
|
833
|
+
}
|
|
834
|
+
});
|
|
835
|
+
|
|
836
|
+
// 40. prod-quinn-pass (GAP → PORTED)
|
|
837
|
+
test("prod-quinn-pass: prod quinn required when prod configured", () => {
|
|
838
|
+
if (!isShipPlus()) return;
|
|
839
|
+
const harness = loadProjectHarness();
|
|
840
|
+
if (!harness?.prod?.rebuild) return;
|
|
841
|
+
const val = sf("environments")?.prod?.quinn;
|
|
842
|
+
if (!val) {
|
|
843
|
+
expect("prod.quinn not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
|
|
844
|
+
} else if (val === "SKIP") {
|
|
845
|
+
const reason = sf("environments")?.prod?.quinnSkipReason;
|
|
846
|
+
if (!reason) expect("quinn SKIP without reason").toBe("quinnSkipReason required");
|
|
847
|
+
} else if (val !== "PASS") {
|
|
848
|
+
expect(`prod quinn: ${val}`).toBe("PASS or SKIP with reason");
|
|
849
|
+
}
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
// quinn-tested-changed-files (#441)
|
|
853
|
+
test("quinn-tested-changed-files: Quinn tested paths cover UI diffs", () => {
|
|
854
|
+
if (!isShipPlus()) return;
|
|
855
|
+
const root = sf("projectRoot");
|
|
856
|
+
if (!root) return;
|
|
857
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
858
|
+
if (!scopeSha) return;
|
|
859
|
+
const diff = exec(`git diff --name-only ${scopeSha}..HEAD`, root);
|
|
860
|
+
if (!diff.ok) return;
|
|
861
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
862
|
+
const uiChanged = diff.output.split("\n").filter(f => UI_EXT.test(f));
|
|
863
|
+
if (uiChanged.length === 0) return;
|
|
864
|
+
const testedPaths = sf("agents")?.quinn?.testedPaths || [];
|
|
865
|
+
if (testedPaths.length === 0) {
|
|
866
|
+
expect(`${uiChanged.length} UI files changed but quinn.testedPaths empty`).toBe("quinn.testedPaths populated");
|
|
867
|
+
}
|
|
868
|
+
});
|
|
869
|
+
|
|
870
|
+
// 42. verify-blockers-cleared
|
|
871
|
+
test("verify-blockers-cleared: all Quinn-found bugs re-verified", () => {
|
|
872
|
+
if (!isShipPlus()) return;
|
|
873
|
+
const blockers = sf("verifyBlockers") || [];
|
|
874
|
+
const unresolved = blockers.filter((b: any) => !b.quinnReverified);
|
|
875
|
+
expect(unresolved.map((b: any) => b.id)).toEqual([]);
|
|
876
|
+
});
|
|
877
|
+
|
|
878
|
+
// 41. prod-quinn-spot (GAP → PORTED)
|
|
879
|
+
test("prod-quinn-spot: spot check PASS or inherited from full quinn", () => {
|
|
880
|
+
if (!isShipPlus()) return;
|
|
881
|
+
const harness = loadProjectHarness();
|
|
882
|
+
if (!harness?.prod?.rebuild) return;
|
|
883
|
+
const quinn = sf("environments")?.prod?.quinn;
|
|
884
|
+
const spot = sf("environments")?.prod?.quinnSpot;
|
|
885
|
+
if (quinn === "PASS" || spot === "PASS") return; // either is fine
|
|
886
|
+
if (spot === "SKIP") {
|
|
887
|
+
const reason = sf("environments")?.prod?.quinnSpotSkipReason;
|
|
888
|
+
if (!reason) expect("quinnSpot SKIP without reason").toBe("quinnSpotSkipReason required");
|
|
889
|
+
}
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
// 43. witness-chain (#473)
|
|
893
|
+
test("witness-chain: witness directory has gate artifacts", () => {
|
|
894
|
+
if (!isShipPlus()) return;
|
|
895
|
+
const slug = sf("slug");
|
|
896
|
+
if (!slug) return;
|
|
897
|
+
const witnessDir = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "witnesses");
|
|
898
|
+
if (!existsSync(witnessDir)) {
|
|
899
|
+
// WARN — witnesses are new, not all workflows have them yet
|
|
900
|
+
console.warn("WARN: No witness directory found — witness chain not verified");
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
const witnesses = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
|
|
904
|
+
expect(witnesses.length, "Witness directory exists but is empty").toBeGreaterThan(0);
|
|
905
|
+
});
|
|
906
|
+
|
|
907
|
+
// 44. witness-ac-verdict: AC PASS verdicts must have corresponding witness chain entry (#521)
|
|
908
|
+
test("witness-ac-verdict: AC PASS verdicts have matching witness with valid HMAC", () => {
|
|
909
|
+
if (!isShipPlus()) return;
|
|
910
|
+
const slug = sf("slug");
|
|
911
|
+
if (!slug) return;
|
|
912
|
+
const witnessDir = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "witnesses");
|
|
913
|
+
if (!existsSync(witnessDir)) {
|
|
914
|
+
// No witnesses = cannot verify AC verdicts were set by gate system
|
|
915
|
+
const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
|
|
916
|
+
if (passACs.length > 0) {
|
|
917
|
+
expect(
|
|
918
|
+
`${passACs.length} AC(s) have PASS verdict but no witness directory exists`,
|
|
919
|
+
).toBe("witness directory required when ACs have PASS verdicts (#521 — prevents manual workflow-state.json edits)");
|
|
920
|
+
}
|
|
921
|
+
return;
|
|
922
|
+
}
|
|
923
|
+
const witnessFiles = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
|
|
924
|
+
// Load all witness records
|
|
925
|
+
const witnesses: Array<{ gate: string; result: string; hmac: string; [k: string]: any }> = [];
|
|
926
|
+
for (const wf of witnessFiles) {
|
|
927
|
+
try {
|
|
928
|
+
const record = JSON.parse(readFileSync(join(witnessDir, wf), "utf-8"));
|
|
929
|
+
witnesses.push(record);
|
|
930
|
+
} catch { /* skip unparseable */ }
|
|
931
|
+
}
|
|
932
|
+
// Build set of gates that have PASS witnesses with HMAC
|
|
933
|
+
const passedGatesWithHmac = new Set(
|
|
934
|
+
witnesses
|
|
935
|
+
.filter(w => w.result === "PASS" && w.hmac && w.hmac.length > 0)
|
|
936
|
+
.map(w => w.gate),
|
|
937
|
+
);
|
|
938
|
+
// Each AC with PASS verdict must be backed by a prior gate witness
|
|
939
|
+
// The verify gate covers AC evidence checks; scope gate covers AC definition
|
|
940
|
+
// Ship witness is written AFTER this test suite — requiring it here is circular
|
|
941
|
+
const requiredGates = ["scope", "verify"];
|
|
942
|
+
const missingGates = requiredGates.filter(g => !passedGatesWithHmac.has(g));
|
|
943
|
+
const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
|
|
944
|
+
if (passACs.length > 0 && missingGates.length > 0) {
|
|
945
|
+
expect(
|
|
946
|
+
`AC PASS verdicts present but missing witness for: ${missingGates.join(", ")}`,
|
|
947
|
+
).toBe("PASS witnesses with valid HMAC required for verify + ship gates (#521 — manual edits detected)");
|
|
948
|
+
}
|
|
949
|
+
});
|
|
950
|
+
|
|
951
|
+
// pre-registration: ACs must predate fix (ADR-009 A2)
|
|
952
|
+
test("pre-registration: scope precedes ship", () => {
|
|
953
|
+
if (!isShipPlus()) return;
|
|
954
|
+
const scopeTs = sf("gates")?.scope?.ts;
|
|
955
|
+
if (!scopeTs) return; // LIGHT tier or pre-ADR-009 — scope gate not run
|
|
956
|
+
const scopeTime = new Date(scopeTs).getTime();
|
|
957
|
+
expect(scopeTime, "scope timestamp must be in the past (ADR-009 A2: pre-registration)").toBeLessThan(Date.now());
|
|
958
|
+
const shipTs = sf("gates")?.ship?.ts;
|
|
959
|
+
if (shipTs) {
|
|
960
|
+
expect(scopeTime, "scope must predate ship (ADR-009 A2: pre-registration)").toBeLessThanOrEqual(new Date(shipTs).getTime());
|
|
961
|
+
}
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
// acHash consistency: ship gate checks scope's AC hash (ADR-009)
|
|
965
|
+
test("acHash-consistency-ship: ACs unchanged since scope", () => {
|
|
966
|
+
if (!isShipPlus()) return;
|
|
967
|
+
const acHash = sf("gates")?.scope?.acHash;
|
|
968
|
+
if (!acHash) return;
|
|
969
|
+
const acDefs = (sf("acs") || []).map((ac: any) => ({
|
|
970
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
971
|
+
specElement: ac.specElement, threshold: ac.threshold,
|
|
972
|
+
evidenceMethod: ac.evidenceMethod,
|
|
973
|
+
}));
|
|
974
|
+
const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
|
|
975
|
+
expect(currentHash, "ACs modified after scope approval — re-run scope gate (ADR-009 acHash)").toBe(acHash);
|
|
976
|
+
});
|
|
977
|
+
|
|
978
|
+
// prove-label-enforcement: needs-prove label must have matching proven label (#1390)
|
|
979
|
+
test("prove-label-enforcement: needs-prove requires proven label", () => {
|
|
980
|
+
if (!isShipPlus()) return;
|
|
981
|
+
const issueNumber = sf("issue");
|
|
982
|
+
const issueRepo = sf("issueRepo") || sf("repo");
|
|
983
|
+
if (!issueNumber || !issueRepo) return;
|
|
984
|
+
// Check GitHub labels on the issue
|
|
985
|
+
let labels: string[] = [];
|
|
986
|
+
try {
|
|
987
|
+
const ghOut = execSync(
|
|
988
|
+
`gh issue view ${issueNumber} --repo ${issueRepo} --json labels -q '.labels[].name'`,
|
|
989
|
+
{ encoding: "utf-8", timeout: 15000 }
|
|
990
|
+
);
|
|
991
|
+
labels = ghOut.trim().split("\n").filter(Boolean);
|
|
992
|
+
} catch {
|
|
993
|
+
// If we can't read labels, skip — don't block on GitHub API failure
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
const hasNeedsProve = labels.includes("needs-prove");
|
|
997
|
+
const hasProven = labels.includes("proven");
|
|
998
|
+
if (hasNeedsProve && !hasProven) {
|
|
999
|
+
expect(
|
|
1000
|
+
"needs-prove label present without proven label",
|
|
1001
|
+
"Close gate blocked: run /prove to add proven label before closing (#1390)"
|
|
1002
|
+
).toBe("proven label required when needs-prove is set");
|
|
1003
|
+
}
|
|
1004
|
+
});
|
|
1005
|
+
});
|
|
1006
|
+
|
|
1007
|
+
// ═══ PROMPT IMMUTABILITY (#1409, ADR-009) ════════════════════════════
|
|
1008
|
+
describe("prompt immutability", () => {
|
|
1009
|
+
test("prompt-immutability: prompt files have uchg flag set", () => {
|
|
1010
|
+
const promptDir = join(__dirname, "prompts");
|
|
1011
|
+
if (!existsSync(promptDir)) return;
|
|
1012
|
+
const prompts = readdirSync(promptDir).filter(f => f.endsWith(".md"));
|
|
1013
|
+
if (prompts.length === 0) return;
|
|
1014
|
+
const unlocked: string[] = [];
|
|
1015
|
+
for (const f of prompts) {
|
|
1016
|
+
const result = exec(`ls -lO "${join(promptDir, f)}"`);
|
|
1017
|
+
if (result.ok && !result.output.includes("uchg")) {
|
|
1018
|
+
unlocked.push(f);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
expect(unlocked, "Prompt files must have uchg flag — run gates/lock-prompts.sh (#1409, ADR-009)").toEqual([]);
|
|
1022
|
+
});
|
|
1023
|
+
});
|
|
1024
|
+
|
|
1025
|
+
// ═══ PROVE CHECKS (ADR-009 B3) ═══════════════════════════════════════
|
|
1026
|
+
describe("prove checks", () => {
|
|
1027
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
1028
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
// prove-evidence exists when prove gate runs
|
|
1032
|
+
test("prove-evidence-exists: prove gate produces evidence file", () => {
|
|
1033
|
+
const currentGate = process.env.GATE;
|
|
1034
|
+
if (currentGate !== "prove") return;
|
|
1035
|
+
const slug = sf("slug");
|
|
1036
|
+
if (!slug) return;
|
|
1037
|
+
const evidencePath = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "prove-evidence.json");
|
|
1038
|
+
expect(existsSync(evidencePath), "prove-evidence.json must exist after prove gate runs (ADR-009 B3)").toBe(true);
|
|
1039
|
+
});
|
|
1040
|
+
|
|
1041
|
+
// prove-evidence has valid verdict enum
|
|
1042
|
+
test("prove-evidence-verdict: verdict is PROVEN|UNPROVEN|INCONCLUSIVE", () => {
|
|
1043
|
+
const currentGate = process.env.GATE;
|
|
1044
|
+
if (currentGate !== "prove") return;
|
|
1045
|
+
const slug = sf("slug");
|
|
1046
|
+
if (!slug) return;
|
|
1047
|
+
const evidencePath = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), slug, "prove-evidence.json");
|
|
1048
|
+
if (!existsSync(evidencePath)) return;
|
|
1049
|
+
const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
|
|
1050
|
+
const VALID_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
1051
|
+
expect(VALID_VERDICTS.includes(evidence.verdict), `prove-evidence verdict must be PROVEN|UNPROVEN|INCONCLUSIVE, got "${evidence.verdict}" (ADR-009 B3)`).toBe(true);
|
|
1052
|
+
});
|
|
1053
|
+
});
|