rungate 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +70 -0
- package/lib/aes-calculator.ts +66 -0
- package/lib/agent-audit.ts +139 -0
- package/lib/agent-brief-validation.ts +359 -0
- package/lib/agents-md-refs.ts +90 -0
- package/lib/bash-file-read.ts +197 -0
- package/lib/behavioral-cache.ts +78 -0
- package/lib/branch-cleanup.ts +234 -0
- package/lib/brief-context-parser.ts +35 -0
- package/lib/brief-precedence.ts +114 -0
- package/lib/canary.ts +117 -0
- package/lib/check-registry.ts +226 -0
- package/lib/ci-checkout.ts +426 -0
- package/lib/cli-commands.ts +137 -0
- package/lib/compliance-backfill.ts +232 -0
- package/lib/compliance-report.ts +605 -0
- package/lib/compliance.ts +361 -0
- package/lib/config-loader.ts +255 -0
- package/lib/config-readers.ts +86 -0
- package/lib/conformity.ts +2161 -137
- package/lib/container-provenance.ts +230 -0
- package/lib/create-brief.ts +150 -0
- package/lib/create-sc.ts +267 -0
- package/lib/decomposition-waves.ts +197 -0
- package/lib/deep-merge.ts +69 -0
- package/lib/dev-liveness.ts +214 -0
- package/lib/directive-extractor.ts +210 -0
- package/lib/distribution-standing.ts +113 -0
- package/lib/doc-hygiene.ts +238 -0
- package/lib/eval-criteria.ts +6 -0
- package/lib/evidence-portability.ts +123 -0
- package/lib/evidence-prevalidator.ts +390 -0
- package/lib/evidence-required.ts +150 -0
- package/lib/evidence-satisfiability.ts +288 -0
- package/lib/frontmatter.ts +163 -0
- package/lib/gap-scanner.ts +348 -0
- package/lib/gate-check-audit.ts +190 -0
- package/lib/gate-enforcement.ts +366 -0
- package/lib/generators/agent-briefs.ts +136 -0
- package/lib/generators/agents-md.ts +511 -0
- package/lib/generators/code-map.ts +60 -0
- package/lib/generators/types.ts +184 -0
- package/lib/git-ref-normalizer.ts +107 -0
- package/lib/github.ts +539 -0
- package/lib/harness-identity.ts +63 -0
- package/lib/hill-climb.ts +164 -0
- package/lib/instruction-effectiveness.ts +131 -0
- package/lib/issue-closure-audit.ts +225 -0
- package/lib/issue-create-override.ts +343 -0
- package/lib/issue-extraction.ts +368 -0
- package/lib/js-top-level-code.ts +250 -0
- package/lib/liveness-contradiction.ts +149 -0
- package/lib/organize.ts +505 -0
- package/lib/paths.ts +197 -6
- package/lib/phase2-decomposition.ts +270 -0
- package/lib/post-fix-verify.ts +157 -0
- package/lib/prior-branch.ts +512 -0
- package/lib/project-state-shape.ts +166 -0
- package/lib/promote-outputs.ts +156 -0
- package/lib/prove-label-gate.ts +128 -0
- package/lib/reachability.ts +302 -0
- package/lib/release-preflight.ts +288 -0
- package/lib/rule-health-pipeline.ts +264 -0
- package/lib/rule-health-trend.ts +161 -0
- package/lib/rule-provenance.ts +149 -0
- package/lib/rule-registry.ts +128 -0
- package/lib/rungate-schema.ts +127 -5
- package/lib/sc-claim-forms.ts +179 -0
- package/lib/sc-cover.ts +150 -0
- package/lib/sc-guard.ts +157 -0
- package/lib/sc-ids.ts +344 -0
- package/lib/scaffold/defaults.ts +165 -0
- package/lib/scaffold/managed-workflow.ts +370 -0
- package/lib/scaffold/owned-files.ts +265 -0
- package/lib/scaffold/pre-push-hook.ts +165 -0
- package/lib/scaffold/steps.ts +2621 -0
- package/lib/scanner.ts +701 -0
- package/lib/scope-denylist.ts +258 -0
- package/lib/screenshot-upload.ts +73 -0
- package/lib/secret-patterns.ts +112 -0
- package/lib/security-verdict.ts +616 -0
- package/lib/signal-phrases.ts +1 -0
- package/lib/spec-change-conformity.ts +80 -0
- package/lib/spec-files.ts +62 -0
- package/lib/spec-registry.ts +133 -0
- package/lib/spec-updater.ts +96 -0
- package/lib/stale-cleanup.ts +224 -0
- package/lib/stale-issue-scanner.ts +36 -0
- package/lib/strip-js-comments.ts +269 -0
- package/lib/suite-measurement.ts +228 -0
- package/lib/task-completion-checks.ts +224 -0
- package/lib/test-suite-lock.ts +1136 -0
- package/lib/transcript-checker.ts +1140 -0
- package/lib/typescript-project.ts +233 -0
- package/lib/validators/spec-validators.ts +171 -0
- package/lib/verdict-capture.ts +73 -0
- package/lib/verified-push.ts +535 -0
- package/lib/workflow-security.ts +174 -0
- package/lib/worktree-cleanup.ts +432 -0
- package/lib/worktree-collect.ts +283 -0
- package/lib/worktree-isolation.ts +82 -0
- package/lib/worktree-repo.ts +300 -0
- package/package.json +27 -3
- package/prompts/ac-adversary.md +14 -2
- package/prompts/aditi.md +5 -0
- package/prompts/blast-radius.md +5 -0
- package/prompts/container-verify.md +5 -0
- package/prompts/discovery.md +4 -0
- package/prompts/environment.md +5 -0
- package/prompts/evidence-hierarchy.md +5 -0
- package/prompts/evidence-validator.md +11 -0
- package/prompts/marcus.md +5 -0
- package/prompts/prevention.md +5 -0
- package/prompts/prove-reproducer.md +12 -0
- package/prompts/quinn-decision-tree.md +5 -0
- package/prompts/quinn-ui-brief.md +5 -0
- package/prompts/quinn.md +5 -0
- package/prompts/rca.md +5 -0
- package/prompts/read-before-write.md +4 -0
- package/prompts/regression.md +5 -0
- package/prompts/serena.md +5 -0
- package/scripts/audit-specs.ts +464 -0
- package/scripts/convert-spec.ts +293 -0
- package/scripts/create-sc.ts +92 -0
- package/scripts/create-spec.ts +166 -0
- package/scripts/extract-constraints.ts +4 -3
- package/scripts/generate-code-map.ts +1 -1
- package/scripts/scaffold-project.ts +138 -1426
- package/scripts/sync-spec-tests.ts +211 -126
|
@@ -0,0 +1,1140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcript compliance checker — cross-references extracted directives
|
|
3
|
+
* against agent tool calls producing FOLLOWED or IGNORED verdict per directive.
|
|
4
|
+
*
|
|
5
|
+
* Also includes role-based evaluation criteria for DA compliance auditing.
|
|
6
|
+
*
|
|
7
|
+
* SC-402: Transcript checker: FOLLOWED/IGNORED per directive
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { readFileSync, readdirSync, existsSync, type Dirent } from "fs";
|
|
11
|
+
import { basename, join } from "path";
|
|
12
|
+
import { harnessRootFor } from "./paths.js";
|
|
13
|
+
import { stripJsComments } from "./strip-js-comments.js";
|
|
14
|
+
import type { Directive } from "./directive-extractor.js";
|
|
15
|
+
import { checkCanaries, type CanaryDefinition, type CanaryReport } from "./canary.js";
|
|
16
|
+
import { isBashFileRead } from "./bash-file-read.js";
|
|
17
|
+
// COMP-2's limit is the guard's limit. One number, one owner.
|
|
18
|
+
//
|
|
19
|
+
// The POLICY constant, deliberately NOT the environment-aware resolver.
|
|
20
|
+
// Security review of d6cc5acf called the first version a self-grading bypass
|
|
21
|
+
// and was right: resolving through the environment would let a run that spent
|
|
22
|
+
// eight suites be graded against a threshold of eight. A grader measures
|
|
23
|
+
// against the policy, never against whatever the graded run's session happens
|
|
24
|
+
// to be configured for.
|
|
25
|
+
//
|
|
26
|
+
// The resolver is named here only in prose, never as a call — a comment that
|
|
27
|
+
// writes out the forbidden form is itself enough to defeat the check that
|
|
28
|
+
// forbids it, which is how the first attempt at this test went red.
|
|
29
|
+
import { DEFAULT_MAX_RUNS_PER_WORKER } from "./test-suite-lock.js";
|
|
30
|
+
|
|
31
|
+
type ComplianceVerdict = "FOLLOWED" | "IGNORED" | "VIOLATED" | "N/A";
|
|
32
|
+
|
|
33
|
+
export interface ComplianceResult {
|
|
34
|
+
directive: Directive;
|
|
35
|
+
status: ComplianceVerdict;
|
|
36
|
+
evidence: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ToolCall {
|
|
40
|
+
name: string;
|
|
41
|
+
input: Record<string, any>;
|
|
42
|
+
order?: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ── Role-based evaluation (from eval-criteria) ──────────
|
|
46
|
+
|
|
47
|
+
export interface TranscriptData {
|
|
48
|
+
calls: ToolCall[];
|
|
49
|
+
reads: string[];
|
|
50
|
+
bashes: string[];
|
|
51
|
+
edits: string[];
|
|
52
|
+
writes: string[];
|
|
53
|
+
duplicateReads: Record<string, number>;
|
|
54
|
+
firstThreeReads: string[];
|
|
55
|
+
promptContent: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export type Verdict = "FOLLOWED" | "IGNORED";
|
|
59
|
+
|
|
60
|
+
// ── COMP-4: spec-drift hashes (#287) ──────────────────────
|
|
61
|
+
|
|
62
|
+
export interface SpecHashGuards {
|
|
63
|
+
/** Spec basenames that some guard file actually holds a hash for. */
|
|
64
|
+
specs: string[];
|
|
65
|
+
/** Repo-relative paths of the files holding those hashes. */
|
|
66
|
+
guardFiles: string[];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Where a future generated hash manifest would live (DEC-011). */
|
|
70
|
+
const DRIFT_HASHES = "test/drift-hashes.json";
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Discover which specs actually have a drift hash guarding them.
|
|
74
|
+
*
|
|
75
|
+
* Read out of the tree rather than written down here. Two specs are guarded
|
|
76
|
+
* today — `BOOTSTRAP-DATA-FLOW-SPEC.md` and `BOOTSTRAP-TEST-PLAN.md` — and a
|
|
77
|
+
* list of them in this file is the hardcoding defect class #343/#344 were
|
|
78
|
+
* about: it would be correct until someone added a third hash, and then wrong
|
|
79
|
+
* silently.
|
|
80
|
+
*
|
|
81
|
+
* It was also half of #287. COMP-4 fired on any path containing `specs/`, so
|
|
82
|
+
* editing `specs/HARNESS-STANDARD.md` — which is where SC records go, so
|
|
83
|
+
* nearly every run edits it — was graded IGNORED at weight 10 for doing the
|
|
84
|
+
* correct thing, clearable only by running a `shasum` whose output nothing
|
|
85
|
+
* read.
|
|
86
|
+
*
|
|
87
|
+
* Prefers a generated manifest if one ever lands (DEC-011 in
|
|
88
|
+
* specs/DOC-HYGIENE-ARCHITECTURE-SPEC.md plans to replace the hand-maintained
|
|
89
|
+
* constants). Until then it scans the guard sources for a hash constant and
|
|
90
|
+
* takes the spec filenames named alongside it.
|
|
91
|
+
*
|
|
92
|
+
* An unreadable tree yields an EMPTY set, which exempts rather than accuses.
|
|
93
|
+
* That is the right direction for a grader: a check that cannot see the
|
|
94
|
+
* guards must not report that an agent skipped one.
|
|
95
|
+
*/
|
|
96
|
+
export function discoverSpecHashGuards(root: string): SpecHashGuards {
|
|
97
|
+
const manifest = join(root, DRIFT_HASHES);
|
|
98
|
+
if (existsSync(manifest)) {
|
|
99
|
+
try {
|
|
100
|
+
const j = JSON.parse(readFileSync(manifest, "utf-8"));
|
|
101
|
+
const specs = Object.keys(j?.specs ?? j ?? {}).filter(k => k.endsWith(".md"));
|
|
102
|
+
if (specs.length > 0) return { specs: specs.sort(), guardFiles: [DRIFT_HASHES] };
|
|
103
|
+
} catch {
|
|
104
|
+
// A manifest that exists but cannot be parsed is not a manifest that is
|
|
105
|
+
// absent — fall through to the scan rather than reporting no guards.
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const specs = new Set<string>();
|
|
110
|
+
const guardFiles = new Set<string>();
|
|
111
|
+
for (const rel of tsFilesUnder(root, "test")) {
|
|
112
|
+
let src: string;
|
|
113
|
+
try { src = stripJsComments(readFileSync(join(root, rel), "utf-8")); } catch { continue; }
|
|
114
|
+
// Comments stripped first. The file documenting this discovery names the
|
|
115
|
+
// constant and the specs in prose, and a detector reading raw source
|
|
116
|
+
// reports its own docblock as a guard — which is precisely what happened
|
|
117
|
+
// on the first run of test/comp4-spec-hash.test.ts.
|
|
118
|
+
//
|
|
119
|
+
// The constant as a DECLARATION. A test asserting something about
|
|
120
|
+
// SPEC_HASH mentions the word too and is not itself a guard.
|
|
121
|
+
if (!/\b(?:const|let|var)\s+SPEC_HASHE?S?\b/.test(src)) continue;
|
|
122
|
+
// The spec must be named as a path under `specs/`, not merely mentioned.
|
|
123
|
+
// A bare `.md` literal anywhere in the file picks up AGENTS.md and
|
|
124
|
+
// CLAUDE.md from unrelated fixture setup — measured, not supposed.
|
|
125
|
+
const named = [
|
|
126
|
+
...src.matchAll(/join\(\s*[^)]*spec[^)]*?,\s*["']([A-Za-z0-9][A-Za-z0-9._-]*\.md)["']/gi),
|
|
127
|
+
].map(m => m[1]);
|
|
128
|
+
if (named.length === 0) continue;
|
|
129
|
+
guardFiles.add(rel);
|
|
130
|
+
for (const n of named) specs.add(n);
|
|
131
|
+
}
|
|
132
|
+
return { specs: [...specs].sort(), guardFiles: [...guardFiles].sort() };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Repo-relative `.ts` paths under `root/dir`, recursive, skipping fixtures. */
|
|
136
|
+
function tsFilesUnder(root: string, dir: string): string[] {
|
|
137
|
+
const out: string[] = [];
|
|
138
|
+
const walk = (rel: string) => {
|
|
139
|
+
let entries: Dirent[];
|
|
140
|
+
try { entries = readdirSync(join(root, rel), { withFileTypes: true }); } catch { return; }
|
|
141
|
+
for (const e of entries) {
|
|
142
|
+
const child = `${rel}/${e.name}`;
|
|
143
|
+
if (e.isDirectory()) {
|
|
144
|
+
if (e.name === "node_modules" || e.name === "fixtures" || e.name === "test-fixtures") continue;
|
|
145
|
+
walk(child);
|
|
146
|
+
} else if (e.name.endsWith(".ts")) {
|
|
147
|
+
out.push(child);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
walk(dir);
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
let guardCache: SpecHashGuards | null = null;
|
|
156
|
+
function cachedSpecHashGuards(): SpecHashGuards {
|
|
157
|
+
if (!guardCache) guardCache = discoverSpecHashGuards(harnessRootFor(join(import.meta.dir, "..")));
|
|
158
|
+
return guardCache;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* COMP-4's reading, as one exported violations-returning function — the single
|
|
163
|
+
* mutatable binding this repo requires of a check.
|
|
164
|
+
*
|
|
165
|
+
* `test/compliance-tests.json` declares:
|
|
166
|
+
*
|
|
167
|
+
* "transcript scan: Bash with 'shasum' AND Edit with 'SPEC_HASH' or 'bootstrap'"
|
|
168
|
+
*
|
|
169
|
+
* An AND across two data sources. The implementation was an OR within one:
|
|
170
|
+
* `data.bashes.some(b => b.includes("shasum") || b.includes("SPEC_HASH"))`,
|
|
171
|
+
* with `data.edits` never consulted after the filter that found the spec
|
|
172
|
+
* edits. COMP-4 was therefore satisfied by running `shasum` on any file at
|
|
173
|
+
* all, and the thing the rule exists to verify — that the constant was
|
|
174
|
+
* actually updated — was the one thing never checked.
|
|
175
|
+
*
|
|
176
|
+
* Both signals are required because either alone is a guess: a shasum nobody
|
|
177
|
+
* wrote down measures nothing, and a constant typed without running the
|
|
178
|
+
* command that produces it is a number somebody invented.
|
|
179
|
+
*/
|
|
180
|
+
export function specHashViolations(data: TranscriptData, guards: SpecHashGuards): string[] {
|
|
181
|
+
const touched = data.edits.concat(data.writes).filter(p => p.includes("specs/") && p.endsWith(".md"));
|
|
182
|
+
const guarded = touched.filter(p => guards.specs.some(s => basename(p) === s));
|
|
183
|
+
if (guarded.length === 0) return [];
|
|
184
|
+
|
|
185
|
+
const ranShasum = data.bashes.some(b => /\bshasum\b/.test(b));
|
|
186
|
+
const editedConstant = data.calls.some(c => {
|
|
187
|
+
if (c.name !== "Edit" && c.name !== "Write") return false;
|
|
188
|
+
const path = String(c.input?.file_path ?? "");
|
|
189
|
+
if (guards.guardFiles.some(g => path.endsWith(g) || basename(path) === basename(g))) return true;
|
|
190
|
+
// Content, for a guard file this process could not discover.
|
|
191
|
+
const body = `${c.input?.new_string ?? ""}${c.input?.content ?? ""}`;
|
|
192
|
+
return /\bSPEC_HASHE?S?\b/.test(body);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
if (ranShasum && editedConstant) return [];
|
|
196
|
+
const missing = [!ranShasum ? "no shasum was run" : null,
|
|
197
|
+
!editedConstant ? "no hash constant was edited" : null].filter(Boolean).join(" and ");
|
|
198
|
+
return [`${guarded.map(p => basename(p)).join(", ")} modified but ${missing}`];
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export interface CriterionResult {
|
|
202
|
+
id: string;
|
|
203
|
+
rule: string;
|
|
204
|
+
verdict: Verdict;
|
|
205
|
+
evidence: string;
|
|
206
|
+
weight: number;
|
|
207
|
+
source: string;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export interface EvalCriterion {
|
|
211
|
+
id: string;
|
|
212
|
+
rule: string;
|
|
213
|
+
weight: number;
|
|
214
|
+
source: string;
|
|
215
|
+
check: (data: TranscriptData) => { verdict: Verdict; evidence: string };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export type Role = "da" | "marcus" | "quinn";
|
|
219
|
+
|
|
220
|
+
// ── Transcript parsing ───────────────────────────────────
|
|
221
|
+
|
|
222
|
+
export function parseToolCalls(transcriptContent: string): ToolCall[] {
|
|
223
|
+
const calls: ToolCall[] = [];
|
|
224
|
+
let order = 0;
|
|
225
|
+
for (const line of transcriptContent.split("\n").filter(Boolean)) {
|
|
226
|
+
try {
|
|
227
|
+
const entry = JSON.parse(line);
|
|
228
|
+
if (entry.type === "assistant") {
|
|
229
|
+
const msg = entry.message?.content;
|
|
230
|
+
if (Array.isArray(msg)) {
|
|
231
|
+
for (const block of msg) {
|
|
232
|
+
if (block?.type === "tool_use") {
|
|
233
|
+
calls.push({ name: block.name, input: block.input || {}, order: order++ });
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
} catch {
|
|
239
|
+
// Skip malformed lines
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return calls;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// ── Compliance checking ──────────────────────────────────
|
|
246
|
+
|
|
247
|
+
function checkReadDirective(d: Directive, reads: string[]): ComplianceResult {
|
|
248
|
+
const targetBase = basename(d.target!);
|
|
249
|
+
const found = reads.some((r) => r.includes(targetBase));
|
|
250
|
+
return {
|
|
251
|
+
directive: d,
|
|
252
|
+
status: found ? "FOLLOWED" : "IGNORED",
|
|
253
|
+
evidence: found
|
|
254
|
+
? `Read call found for ${targetBase}`
|
|
255
|
+
: `No Read call for ${targetBase} in ${reads.length} reads`,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function checkRunDirective(d: Directive, bashes: string[]): ComplianceResult {
|
|
260
|
+
if (d.target!.includes("bun test")) {
|
|
261
|
+
const testRuns = bashes.filter((b) => /\bbun test\b/.test(b));
|
|
262
|
+
if (testRuns.length === 0) return { directive: d, status: "IGNORED", evidence: "No bun test calls found" };
|
|
263
|
+
if (testRuns.length > 2) return { directive: d, status: "VIOLATED", evidence: `${testRuns.length} bun test runs (expected 1-2)` };
|
|
264
|
+
return { directive: d, status: "FOLLOWED", evidence: `${testRuns.length} bun test run(s)` };
|
|
265
|
+
}
|
|
266
|
+
if (d.target!.includes("tsc")) {
|
|
267
|
+
const tscRuns = bashes.filter((b) => b.includes("tsc") && b.includes("--noEmit"));
|
|
268
|
+
return {
|
|
269
|
+
directive: d,
|
|
270
|
+
status: tscRuns.length > 0 ? "FOLLOWED" : "IGNORED",
|
|
271
|
+
evidence: tscRuns.length > 0 ? `${tscRuns.length} tsc run(s)` : "tsc --noEmit never run",
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
const found = bashes.some((b) => b.includes(d.target!));
|
|
275
|
+
return {
|
|
276
|
+
directive: d,
|
|
277
|
+
status: found ? "FOLLOWED" : "IGNORED",
|
|
278
|
+
evidence: found ? "Command found" : "Command not found",
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* The commands a NEVER directive names, taken FROM the directive (#343).
|
|
284
|
+
*
|
|
285
|
+
* This check used to carry `if (textLower.includes("make rebuild"))` — one
|
|
286
|
+
* consumer's forbidden command, written into the harness. For every other
|
|
287
|
+
* consumer that arm never matched, the directive fell through to
|
|
288
|
+
* `status: "N/A"`, and `computeRate` excludes N/A from its denominator. So
|
|
289
|
+
* the rule was not enforced, the compliance rate did not drop, and nothing
|
|
290
|
+
* named the directive that had stopped being measured.
|
|
291
|
+
*
|
|
292
|
+
* A "never run X" directive already says what X is. Reading it out of the
|
|
293
|
+
* directive makes the check work for every consumer and removes the literal.
|
|
294
|
+
*
|
|
295
|
+
* Deliberately narrow about what counts as a command. A backticked token has
|
|
296
|
+
* to start with a lower-case letter and carry no slash in its first word, so
|
|
297
|
+
* `` `.github/workflows/ci.yml` ``, `` `CODE-MAP.md` `` and
|
|
298
|
+
* `` `project-state.json` `` are paths rather than commands. Reporting
|
|
299
|
+
* VIOLATED because somebody ran a command naming a file they were told not to
|
|
300
|
+
* edit is a worse failure than the hole this replaces.
|
|
301
|
+
*
|
|
302
|
+
* This shipped with an extra `if (token.includes("/") || token.includes("."))
|
|
303
|
+
* continue;` in front, added for safety. Mutating it away scored **0 red**,
|
|
304
|
+
* which is how it was found: `COMMANDISH` already rejects all three paths on
|
|
305
|
+
* its own, so the line never once changed the verdict for a path. What it did
|
|
306
|
+
* do was reject `npm run deploy.prod` — a legitimate command with a dot in
|
|
307
|
+
* it. A guard that can only ever remove true positives, kept because it
|
|
308
|
+
* looked careful.
|
|
309
|
+
*/
|
|
310
|
+
const COMMANDISH = /^[a-z][a-z0-9_-]*(?: [a-z0-9_.:=-]+)*$/;
|
|
311
|
+
|
|
312
|
+
export function forbiddenCommandsIn(text: string): string[] {
|
|
313
|
+
const out = new Set<string>();
|
|
314
|
+
for (const m of text.matchAll(/`([^`]+)`/g)) {
|
|
315
|
+
const token = m[1].trim();
|
|
316
|
+
if (!COMMANDISH.test(token)) continue;
|
|
317
|
+
out.add(token);
|
|
318
|
+
}
|
|
319
|
+
return [...out];
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function checkNeverDirective(d: Directive, bashes: string[], toolCalls: ToolCall[]): ComplianceResult {
|
|
323
|
+
const textLower = d.text.toLowerCase();
|
|
324
|
+
|
|
325
|
+
if (textLower.includes("cat") && (textLower.includes("bash") || textLower.includes("read tool"))) {
|
|
326
|
+
const catCalls = bashes.filter((b) =>
|
|
327
|
+
/^\s*cat\s|[;&|]\s*cat\s|\bhead\b|\btail\b/.test(b) &&
|
|
328
|
+
!b.includes("<<") && !b.includes("/tmp/claude-") && !b.includes(".output") && !b.includes(".exit")
|
|
329
|
+
);
|
|
330
|
+
return { directive: d, status: catCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${catCalls.length} cat commands` };
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
if (textLower.includes("pwd") || textLower.includes("ls -la")) {
|
|
334
|
+
const orientCalls = bashes.filter((b) => /\bpwd\b/.test(b) || /\bls -la\b/.test(b));
|
|
335
|
+
return { directive: d, status: orientCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${orientCalls.length} orientation calls` };
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
if (textLower.includes("subagent") || textLower.includes("spawn")) {
|
|
339
|
+
const agentCalls = toolCalls.filter((c) => c.name === "Agent" || c.name === "TaskCreate");
|
|
340
|
+
return { directive: d, status: agentCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${agentCalls.length} agent/task calls` };
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (textLower.includes("same file twice") || textLower.includes("duplicate")) {
|
|
344
|
+
const readPaths = toolCalls.filter((c) => c.name === "Read").map((c) => c.input.file_path || "");
|
|
345
|
+
const dupes = readPaths.filter((p, i) => readPaths.indexOf(p) !== i);
|
|
346
|
+
return { directive: d, status: dupes.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${dupes.length} duplicate reads` };
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
if (textLower.includes("bun test") && textLower.includes("more than twice")) {
|
|
350
|
+
const testRuns = bashes.filter((b) => /\bbun test\b/.test(b) && !b.includes("grep"));
|
|
351
|
+
return { directive: d, status: testRuns.length <= 2 ? "FOLLOWED" : "VIOLATED", evidence: `${testRuns.length} bun test runs` };
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
if (textLower.includes("implementation code") || textLower.includes("modify source code")) {
|
|
355
|
+
const writes = toolCalls.filter((c) => (c.name === "Write" || c.name === "Edit") && /\.(ts|js|tsx|jsx)$/.test(c.input.file_path || ""));
|
|
356
|
+
return { directive: d, status: writes.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${writes.length} source file writes` };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
if (textLower.includes("run builds") || textLower.includes("run tests") || (textLower.includes("builds") && textLower.includes("deployments"))) {
|
|
360
|
+
const buildRuns = bashes.filter((b) => /\b(bun test|bun run|npm test|npm run build|make build|make deploy)\b/.test(b));
|
|
361
|
+
return { directive: d, status: buildRuns.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${buildRuns.length} build/test/deploy commands` };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// #343. Last, so every specific arm above still wins, and generic rather
|
|
365
|
+
// than one consumer's command: whatever the directive forbids by name is
|
|
366
|
+
// what gets grepped for. This is what the `make rebuild` arm used to be.
|
|
367
|
+
const forbidden = forbiddenCommandsIn(d.text);
|
|
368
|
+
if (forbidden.length > 0) {
|
|
369
|
+
const hits = bashes.filter((b) => forbidden.some((c) => b.includes(c)));
|
|
370
|
+
return {
|
|
371
|
+
directive: d,
|
|
372
|
+
status: hits.length === 0 ? "FOLLOWED" : "VIOLATED",
|
|
373
|
+
evidence: `${hits.length} call(s) to ${forbidden.map((c) => `\`${c}\``).join(", ")}`,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
return { directive: d, status: "N/A", evidence: "Cannot verify mechanically" };
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function checkAlwaysDirective(d: Directive, reads: string[], bashes: string[]): ComplianceResult {
|
|
381
|
+
const textLower = d.text.toLowerCase();
|
|
382
|
+
|
|
383
|
+
if (textLower.includes("agents.md")) {
|
|
384
|
+
const early = reads.slice(0, 5).some((r) => r.includes("AGENTS.md"));
|
|
385
|
+
return { directive: d, status: early ? "FOLLOWED" : "IGNORED", evidence: early ? "AGENTS.md in first 5 reads" : "AGENTS.md not in first 5 reads" };
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
if (textLower.includes("bun test")) {
|
|
389
|
+
const testRuns = bashes.filter((b) => /\bbun test\b/.test(b));
|
|
390
|
+
return { directive: d, status: testRuns.length > 0 ? "FOLLOWED" : "IGNORED", evidence: `${testRuns.length} test runs` };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
if (textLower.includes("verify before assert")) {
|
|
394
|
+
// Check for verification commands (grep, test, etc.) before final output
|
|
395
|
+
const verifyCount = bashes.filter((b) => b.includes("grep") || b.includes("test") || b.includes("tsc")).length;
|
|
396
|
+
return { directive: d, status: verifyCount > 0 ? "FOLLOWED" : "IGNORED", evidence: `${verifyCount} verification commands` };
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return { directive: d, status: "N/A", evidence: "Cannot verify mechanically" };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export function checkCompliance(directives: Directive[], transcriptContent: string): ComplianceResult[] {
|
|
403
|
+
const toolCalls = parseToolCalls(transcriptContent);
|
|
404
|
+
const reads = toolCalls.filter((c) => c.name === "Read").map((c) => c.input.file_path || "");
|
|
405
|
+
const bashes = toolCalls.filter((c) => c.name === "Bash").map((c) => c.input.command || "");
|
|
406
|
+
|
|
407
|
+
return directives.map((d) => {
|
|
408
|
+
switch (d.type) {
|
|
409
|
+
case "read":
|
|
410
|
+
return d.target ? checkReadDirective(d, reads) : { directive: d, status: "N/A" as const, evidence: "No target" };
|
|
411
|
+
case "run":
|
|
412
|
+
return d.target ? checkRunDirective(d, bashes) : { directive: d, status: "N/A" as const, evidence: "No target" };
|
|
413
|
+
case "never":
|
|
414
|
+
return checkNeverDirective(d, bashes, toolCalls);
|
|
415
|
+
case "always":
|
|
416
|
+
return checkAlwaysDirective(d, reads, bashes);
|
|
417
|
+
default:
|
|
418
|
+
return { directive: d, status: "N/A" as const, evidence: "Unknown directive type" };
|
|
419
|
+
}
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// ── Behavioral pattern checks ───────────────────────────
|
|
424
|
+
|
|
425
|
+
export interface SequenceEvent {
|
|
426
|
+
type: "WRITE_TEST" | "WRITE_SOURCE" | "TEST_RUN";
|
|
427
|
+
file?: string;
|
|
428
|
+
cmd?: string;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
export interface TDDResult {
|
|
432
|
+
testFirst: boolean;
|
|
433
|
+
redPhase: boolean;
|
|
434
|
+
greenPhase: boolean;
|
|
435
|
+
sequence: SequenceEvent[];
|
|
436
|
+
verdict: "TDD" | "TEST_AFTER" | "NO_TESTS" | "NO_SOURCE";
|
|
437
|
+
evidence: string;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Directories whose contents count as implementation for the TDD check (#240).
|
|
442
|
+
*
|
|
443
|
+
* `gates/`, `workflows/` and `hooks/` were missing. In a repo whose pipeline
|
|
444
|
+
* IS `workflows/ship.js` and whose gates are all of `gates/`, that meant an
|
|
445
|
+
* agent could edit the two most consequential files in the project and record
|
|
446
|
+
* zero WRITE_SOURCE events — short-circuiting to NO_SOURCE, "No source files
|
|
447
|
+
* written". The harness could not grade TDD on the harness.
|
|
448
|
+
*
|
|
449
|
+
* `src/` is kept although this repo has none: rungate scaffolds projects that
|
|
450
|
+
* do, and this module grades their transcripts too. The test derives the list
|
|
451
|
+
* of directories a repo actually keeps source in and asserts each is covered,
|
|
452
|
+
* so the next directory to appear cannot go silently ungraded the way these
|
|
453
|
+
* three did.
|
|
454
|
+
*/
|
|
455
|
+
export const SOURCE_DIRS = [
|
|
456
|
+
"lib",
|
|
457
|
+
"scripts",
|
|
458
|
+
"src",
|
|
459
|
+
"gates",
|
|
460
|
+
"workflows",
|
|
461
|
+
"hooks",
|
|
462
|
+
// #363 added bin/ and the derived test caught it on the first CI run,
|
|
463
|
+
// which is the check working rather than the check being in the way: an
|
|
464
|
+
// unlisted directory is not a test failure, it is TDD going ungraded for
|
|
465
|
+
// everything written there.
|
|
466
|
+
"bin",
|
|
467
|
+
] as const;
|
|
468
|
+
|
|
469
|
+
/** Whether a written path is implementation rather than test or docs. */
|
|
470
|
+
export function isSourcePath(path: string): boolean {
|
|
471
|
+
if (path.includes("test/") || path.includes(".test.")) return false;
|
|
472
|
+
return SOURCE_DIRS.some((d) => path.includes(`${d}/`));
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
export function checkTDD(transcriptContent: string): TDDResult {
|
|
476
|
+
const calls = parseToolCalls(transcriptContent);
|
|
477
|
+
const sequence: SequenceEvent[] = [];
|
|
478
|
+
|
|
479
|
+
for (const call of calls) {
|
|
480
|
+
if (call.name === "Write" || call.name === "Edit") {
|
|
481
|
+
const path = call.input?.file_path || call.input?.path || "";
|
|
482
|
+
const file = basename(path);
|
|
483
|
+
const isTest = path.includes("test/") || path.includes(".test.");
|
|
484
|
+
if (isTest) sequence.push({ type: "WRITE_TEST", file });
|
|
485
|
+
else if (isSourcePath(path)) sequence.push({ type: "WRITE_SOURCE", file });
|
|
486
|
+
}
|
|
487
|
+
if (call.name === "Bash") {
|
|
488
|
+
const cmd = call.input?.command || "";
|
|
489
|
+
if (cmd.includes("bun test")) {
|
|
490
|
+
sequence.push({ type: "TEST_RUN", cmd: cmd.slice(0, 60) });
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
const firstTest = sequence.findIndex((s) => s.type === "WRITE_TEST");
|
|
496
|
+
const firstSource = sequence.findIndex((s) => s.type === "WRITE_SOURCE");
|
|
497
|
+
|
|
498
|
+
if (firstTest === -1) return { testFirst: false, redPhase: false, greenPhase: false, sequence, verdict: "NO_TESTS", evidence: "No test files written" };
|
|
499
|
+
if (firstSource === -1) return { testFirst: true, redPhase: false, greenPhase: false, sequence, verdict: "NO_SOURCE", evidence: "No source files written" };
|
|
500
|
+
|
|
501
|
+
const testFirst = firstTest < firstSource;
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Red and green are existence questions, not a single-pass automaton (#240).
|
|
505
|
+
*
|
|
506
|
+
* The previous implementation carried one `state` through the sequence and
|
|
507
|
+
* reset it to "wrote_test" on EVERY test write, unconditionally. A pending
|
|
508
|
+
* source write waiting for its green run was therefore erased by the next
|
|
509
|
+
* test file the agent added, and nothing could restore it: re-entering
|
|
510
|
+
* "wrote_source" required another source write. An agent that wrote source,
|
|
511
|
+
* added one more test, and then ran the suite nine times was graded as
|
|
512
|
+
* having run it zero times — and that verdict blocks a ship.
|
|
513
|
+
*
|
|
514
|
+
* Stated as what each phase means, the resets have nowhere to happen:
|
|
515
|
+
*
|
|
516
|
+
* - RED: a run that happened after a test was written and BEFORE any
|
|
517
|
+
* source existed to make it pass. Scanning forward from each test write
|
|
518
|
+
* and stopping at the first source write is what "before" means here; a
|
|
519
|
+
* run found after that point is a green run being counted twice.
|
|
520
|
+
* - GREEN: a run after the LAST source write. Not "after any source
|
|
521
|
+
* write" — an agent that runs the suite and then edits source and stops
|
|
522
|
+
* has not shown the code it is shipping passing anything, which is the
|
|
523
|
+
* case this check exists to catch.
|
|
524
|
+
*/
|
|
525
|
+
const lastSource = sequence.map((s) => s.type).lastIndexOf("WRITE_SOURCE");
|
|
526
|
+
const greenPhase = sequence.slice(lastSource + 1).some((s) => s.type === "TEST_RUN");
|
|
527
|
+
|
|
528
|
+
let redPhase = false;
|
|
529
|
+
for (let i = 0; i < sequence.length && !redPhase; i++) {
|
|
530
|
+
if (sequence[i].type !== "WRITE_TEST") continue;
|
|
531
|
+
for (let j = i + 1; j < sequence.length; j++) {
|
|
532
|
+
if (sequence[j].type === "WRITE_SOURCE") break;
|
|
533
|
+
if (sequence[j].type === "TEST_RUN") { redPhase = true; break; }
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
const verdict = testFirst && redPhase && greenPhase ? "TDD" : "TEST_AFTER";
|
|
538
|
+
const parts: string[] = [];
|
|
539
|
+
if (!testFirst) parts.push(`source written at step ${firstSource + 1} before test at step ${firstTest + 1}`);
|
|
540
|
+
if (!redPhase) parts.push("no test run after writing test (missing red phase)");
|
|
541
|
+
if (!greenPhase) parts.push("no test run after writing source (missing green phase)");
|
|
542
|
+
const evidence = verdict === "TDD" ? "test-first → red → green pattern confirmed" : parts.join("; ");
|
|
543
|
+
|
|
544
|
+
return { testFirst, redPhase, greenPhase, sequence, verdict, evidence };
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// ── Scoring ──────────────────────────────────────────────
|
|
548
|
+
|
|
549
|
+
export function computeScore(results: ComplianceResult[]): { score: number; grade: string; followed: number; ignored: number; violated: number; checkable: number } {
|
|
550
|
+
const followed = results.filter((r) => r.status === "FOLLOWED").length;
|
|
551
|
+
const ignored = results.filter((r) => r.status === "IGNORED").length;
|
|
552
|
+
const violated = results.filter((r) => r.status === "VIOLATED").length;
|
|
553
|
+
const checkable = results.filter((r) => r.status !== "N/A").length;
|
|
554
|
+
const score = checkable > 0 ? Math.round((followed / checkable) * 100) : 0;
|
|
555
|
+
const grade = score >= 90 ? "A" : score >= 75 ? "B" : score >= 60 ? "C" : score >= 40 ? "D" : "F";
|
|
556
|
+
return { score, grade, followed, ignored, violated, checkable };
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// ── Combined compliance + canary verification (SC-409, D-5) ──
|
|
560
|
+
|
|
561
|
+
export function checkComplianceWithCanaries(
|
|
562
|
+
directives: Directive[],
|
|
563
|
+
transcriptContent: string,
|
|
564
|
+
canaries: CanaryDefinition[],
|
|
565
|
+
): { compliance: ComplianceResult[]; canaryReport: CanaryReport } {
|
|
566
|
+
const compliance = checkCompliance(directives, transcriptContent);
|
|
567
|
+
const canaryReport = checkCanaries(transcriptContent, canaries);
|
|
568
|
+
return { compliance, canaryReport };
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// ── Report formatting ────────────────────────────────────
|
|
572
|
+
|
|
573
|
+
export function formatReport(
|
|
574
|
+
role: string,
|
|
575
|
+
results: ComplianceResult[],
|
|
576
|
+
repoRailsCount: number,
|
|
577
|
+
): string {
|
|
578
|
+
const { score, grade, followed, ignored, violated, checkable } = computeScore(results);
|
|
579
|
+
const directives = results.map((r) => r.directive);
|
|
580
|
+
|
|
581
|
+
const lines: string[] = [];
|
|
582
|
+
lines.push(`\n${"=".repeat(60)}`);
|
|
583
|
+
lines.push(`BRIEF COMPLIANCE: ${role} -- ${grade} (${score}%)`);
|
|
584
|
+
lines.push(`${"=".repeat(60)}`);
|
|
585
|
+
lines.push(`Directives: ${directives.length} extracted, ${checkable} checkable`);
|
|
586
|
+
lines.push(`Followed: ${followed} | Ignored: ${ignored} | Violated: ${violated}`);
|
|
587
|
+
lines.push(`RepoRails findings: ${repoRailsCount}`);
|
|
588
|
+
lines.push("");
|
|
589
|
+
|
|
590
|
+
lines.push("DIRECTIVE | LINE | SECTION | STATUS");
|
|
591
|
+
lines.push("-".repeat(95));
|
|
592
|
+
for (const r of results) {
|
|
593
|
+
const icon = r.status === "FOLLOWED" ? "[OK]" : r.status === "IGNORED" ? "[--]" : r.status === "VIOLATED" ? "[!!]" : "[ ]";
|
|
594
|
+
const text = r.directive.text.substring(0, 48).padEnd(48);
|
|
595
|
+
const line = `L${r.directive.line}`.padEnd(6);
|
|
596
|
+
const section = r.directive.section.substring(0, 15).padEnd(15);
|
|
597
|
+
lines.push(`${icon} ${text} | ${line}| ${section} | ${r.status}: ${r.evidence}`);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
lines.push("");
|
|
601
|
+
lines.push("RECOMMENDATIONS:");
|
|
602
|
+
const ignoredDirectives = results.filter((r) => r.status === "IGNORED");
|
|
603
|
+
for (const r of ignoredDirectives) {
|
|
604
|
+
const inTop20 = r.directive.line <= 20;
|
|
605
|
+
lines.push(` -> "${r.directive.text.substring(0, 50)}" at L${r.directive.line} (${r.directive.section})`);
|
|
606
|
+
if (!inTop20) lines.push(` FIX: Move to Context or Always Do section (currently in ${r.directive.section})`);
|
|
607
|
+
else lines.push(` FIX: Strengthen language -- add MANDATORY or BEFORE keyword`);
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
return lines.join("\n");
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// ═══════════════════════════════════════════════════════
|
|
614
|
+
// Role-based evaluation criteria (merged from eval-criteria.ts)
|
|
615
|
+
// ═══════════════════════════════════════════════════════
|
|
616
|
+
|
|
617
|
+
// ── Shared criteria (apply to all roles) ────────────────
|
|
618
|
+
|
|
619
|
+
const sharedCriteria: EvalCriterion[] = [
|
|
620
|
+
{
|
|
621
|
+
id: "COMP-1",
|
|
622
|
+
rule: "AGENTS.md context available (read or injected)",
|
|
623
|
+
weight: 20,
|
|
624
|
+
source: "AGENTS.md § Rules",
|
|
625
|
+
check(data) {
|
|
626
|
+
const readIt = data.calls.some(
|
|
627
|
+
(c) => c.name === "Read" && (c.input.file_path || "").endsWith("AGENTS.md")
|
|
628
|
+
);
|
|
629
|
+
if (readIt) {
|
|
630
|
+
const pos = data.calls.findIndex(
|
|
631
|
+
(c) => c.name === "Read" && (c.input.file_path || "").endsWith("AGENTS.md")
|
|
632
|
+
);
|
|
633
|
+
return { verdict: "FOLLOWED", evidence: `AGENTS.md read at position ${pos + 1}` };
|
|
634
|
+
}
|
|
635
|
+
const injected = data.promptContent?.includes("AGENTS.md");
|
|
636
|
+
if (injected) {
|
|
637
|
+
return { verdict: "FOLLOWED", evidence: "AGENTS.md content injected in prompt" };
|
|
638
|
+
}
|
|
639
|
+
return {
|
|
640
|
+
verdict: "IGNORED",
|
|
641
|
+
evidence: `AGENTS.md not read or injected. First reads: ${data.firstThreeReads.join(", ")}`,
|
|
642
|
+
};
|
|
643
|
+
},
|
|
644
|
+
},
|
|
645
|
+
{
|
|
646
|
+
id: "COMP-6",
|
|
647
|
+
rule: "No duplicate file reads",
|
|
648
|
+
weight: 15,
|
|
649
|
+
source: "marcus.md § Never Do",
|
|
650
|
+
check(data) {
|
|
651
|
+
const dupCount = Object.keys(data.duplicateReads).length;
|
|
652
|
+
return {
|
|
653
|
+
verdict: dupCount === 0 ? "FOLLOWED" : "IGNORED",
|
|
654
|
+
evidence:
|
|
655
|
+
dupCount === 0
|
|
656
|
+
? "All reads unique"
|
|
657
|
+
: `${dupCount} files read multiple times: ${Object.entries(data.duplicateReads)
|
|
658
|
+
.map(([p, c]) => `${basename(p)} (${c}x)`)
|
|
659
|
+
.join(", ")}`,
|
|
660
|
+
};
|
|
661
|
+
},
|
|
662
|
+
},
|
|
663
|
+
{
|
|
664
|
+
id: "COMP-7",
|
|
665
|
+
rule: "No cat/head via Bash (use Read)",
|
|
666
|
+
weight: 5,
|
|
667
|
+
source: "marcus.md § Never Do",
|
|
668
|
+
check(data) {
|
|
669
|
+
// Shares detectBashFileRead with BashToolGuard so the grader can never
|
|
670
|
+
// flag something the hook allows (or miss something the hook blocks).
|
|
671
|
+
const catBashes = data.bashes.filter((b) =>
|
|
672
|
+
isBashFileRead(b) &&
|
|
673
|
+
!b.includes("<<") &&
|
|
674
|
+
!b.includes("/tmp/claude-") &&
|
|
675
|
+
!b.includes(".output") &&
|
|
676
|
+
!b.includes(".exit")
|
|
677
|
+
);
|
|
678
|
+
return {
|
|
679
|
+
verdict: catBashes.length === 0 ? "FOLLOWED" : "IGNORED",
|
|
680
|
+
evidence:
|
|
681
|
+
catBashes.length === 0 ? "Clean" : `${catBashes.length} cat commands found`,
|
|
682
|
+
};
|
|
683
|
+
},
|
|
684
|
+
},
|
|
685
|
+
{
|
|
686
|
+
id: "COMP-8",
|
|
687
|
+
rule: "Read PROJECT-STATE if task requires project context",
|
|
688
|
+
weight: 10,
|
|
689
|
+
source: "AGENTS.md § Key Files",
|
|
690
|
+
check(data) {
|
|
691
|
+
const found = data.reads.some(
|
|
692
|
+
(r) => r.includes("PROJECT-STATE") || r.includes("project-state.json")
|
|
693
|
+
);
|
|
694
|
+
if (found) {
|
|
695
|
+
return { verdict: "FOLLOWED", evidence: "Project state read" };
|
|
696
|
+
}
|
|
697
|
+
const injected = data.promptContent?.includes("PROJECT-STATE") ||
|
|
698
|
+
data.promptContent?.includes("project-state.json") ||
|
|
699
|
+
data.promptContent?.includes("Current phase:");
|
|
700
|
+
if (injected) {
|
|
701
|
+
return { verdict: "FOLLOWED", evidence: "PROJECT-STATE content injected in prompt" };
|
|
702
|
+
}
|
|
703
|
+
const touchesMultipleFiles = data.edits.length + data.writes.length > 3;
|
|
704
|
+
if (!touchesMultipleFiles) {
|
|
705
|
+
return {
|
|
706
|
+
verdict: "FOLLOWED",
|
|
707
|
+
evidence: "Small task — PROJECT-STATE read not required",
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
return {
|
|
711
|
+
verdict: "IGNORED",
|
|
712
|
+
evidence: "PROJECT-STATE not read or injected (multi-file task)",
|
|
713
|
+
};
|
|
714
|
+
},
|
|
715
|
+
},
|
|
716
|
+
];
|
|
717
|
+
|
|
718
|
+
// ── DA-specific criteria ────────────────────────────────
|
|
719
|
+
|
|
720
|
+
const daCriteria: EvalCriterion[] = [
|
|
721
|
+
{
|
|
722
|
+
id: "DA-01",
|
|
723
|
+
rule: "Invoke harness skill before implementation",
|
|
724
|
+
weight: 20,
|
|
725
|
+
source: "CLAUDE.md § MANDATORY GATE step 3",
|
|
726
|
+
check(data) {
|
|
727
|
+
const skillCall = data.calls.findIndex(
|
|
728
|
+
(c) => c.name === "Skill" && (c.input.skill || "") === "harness"
|
|
729
|
+
);
|
|
730
|
+
const firstEdit = data.calls.findIndex(
|
|
731
|
+
(c) => c.name === "Edit" || c.name === "Write"
|
|
732
|
+
);
|
|
733
|
+
const invoked = skillCall >= 0;
|
|
734
|
+
const beforeEdit = firstEdit < 0 || skillCall < firstEdit;
|
|
735
|
+
return {
|
|
736
|
+
verdict: invoked && beforeEdit ? "FOLLOWED" : "IGNORED",
|
|
737
|
+
evidence: invoked
|
|
738
|
+
? `Skill("harness") at position ${skillCall + 1}${beforeEdit ? " (before edits)" : " (AFTER edits)"}`
|
|
739
|
+
: "Skill('harness') never invoked",
|
|
740
|
+
};
|
|
741
|
+
},
|
|
742
|
+
},
|
|
743
|
+
{
|
|
744
|
+
id: "DA-02",
|
|
745
|
+
rule: "Run bun test before implementation",
|
|
746
|
+
weight: 15,
|
|
747
|
+
source: "CLAUDE.md § MANDATORY GATE step 1",
|
|
748
|
+
check(data) {
|
|
749
|
+
const firstTest = data.calls.findIndex(
|
|
750
|
+
(c) => c.name === "Bash" && /\bbun test\b/.test(c.input.command || "")
|
|
751
|
+
);
|
|
752
|
+
const firstEdit = data.calls.findIndex(
|
|
753
|
+
(c) => c.name === "Edit" || c.name === "Write"
|
|
754
|
+
);
|
|
755
|
+
const ran = firstTest >= 0;
|
|
756
|
+
const beforeEdit = firstEdit < 0 || firstTest < firstEdit;
|
|
757
|
+
return {
|
|
758
|
+
verdict: ran && beforeEdit ? "FOLLOWED" : "IGNORED",
|
|
759
|
+
evidence: ran
|
|
760
|
+
? `bun test at position ${firstTest + 1}${beforeEdit ? " (before edits)" : " (AFTER edits)"}`
|
|
761
|
+
: "bun test never run",
|
|
762
|
+
};
|
|
763
|
+
},
|
|
764
|
+
},
|
|
765
|
+
{
|
|
766
|
+
id: "DA-03",
|
|
767
|
+
rule: "Read governing spec before implementation",
|
|
768
|
+
weight: 15,
|
|
769
|
+
source: "CLAUDE.md § MANDATORY GATE step 2",
|
|
770
|
+
check(data) {
|
|
771
|
+
const firstEdit = data.calls.findIndex(
|
|
772
|
+
(c) => c.name === "Edit" || c.name === "Write"
|
|
773
|
+
);
|
|
774
|
+
if (firstEdit < 0) {
|
|
775
|
+
return { verdict: "FOLLOWED", evidence: "No edits — spec read not required" };
|
|
776
|
+
}
|
|
777
|
+
const specRead = data.calls.slice(0, firstEdit).some(
|
|
778
|
+
(c) => c.name === "Read" && (c.input.file_path || "").includes("specs/")
|
|
779
|
+
);
|
|
780
|
+
return {
|
|
781
|
+
verdict: specRead ? "FOLLOWED" : "IGNORED",
|
|
782
|
+
evidence: specRead
|
|
783
|
+
? "Spec read before edits"
|
|
784
|
+
: "No spec read before first edit",
|
|
785
|
+
};
|
|
786
|
+
},
|
|
787
|
+
},
|
|
788
|
+
{
|
|
789
|
+
id: "DA-04",
|
|
790
|
+
rule: "Delegate to named agents (no direct Edit/Write on lib/test/scripts/gates/hooks)",
|
|
791
|
+
weight: 20,
|
|
792
|
+
source: "CLAUDE.md § MANDATORY GATE step 4",
|
|
793
|
+
check(data) {
|
|
794
|
+
const directEdits = [...data.edits, ...data.writes].filter((p) =>
|
|
795
|
+
/\/(lib|test|scripts|gates|hooks)\//.test(p)
|
|
796
|
+
);
|
|
797
|
+
return {
|
|
798
|
+
verdict: directEdits.length === 0 ? "FOLLOWED" : "IGNORED",
|
|
799
|
+
evidence:
|
|
800
|
+
directEdits.length === 0
|
|
801
|
+
? "No direct edits to restricted directories"
|
|
802
|
+
: `${directEdits.length} direct edits to restricted dirs: ${directEdits.map((p) => basename(p)).join(", ")}`,
|
|
803
|
+
};
|
|
804
|
+
},
|
|
805
|
+
},
|
|
806
|
+
{
|
|
807
|
+
id: "DA-05",
|
|
808
|
+
rule: "Update PROJECT-STATE at milestones",
|
|
809
|
+
weight: 10,
|
|
810
|
+
source: "CLAUDE.md § Rules",
|
|
811
|
+
check(data) {
|
|
812
|
+
const stateEdits = [...data.edits, ...data.writes].filter(
|
|
813
|
+
(p) =>
|
|
814
|
+
p.includes("PROJECT-STATE") || p.includes("project-state.json")
|
|
815
|
+
);
|
|
816
|
+
return {
|
|
817
|
+
verdict: stateEdits.length > 0 ? "FOLLOWED" : "IGNORED",
|
|
818
|
+
evidence:
|
|
819
|
+
stateEdits.length > 0
|
|
820
|
+
? `PROJECT-STATE updated ${stateEdits.length} time(s)`
|
|
821
|
+
: "PROJECT-STATE never updated",
|
|
822
|
+
};
|
|
823
|
+
},
|
|
824
|
+
},
|
|
825
|
+
];
|
|
826
|
+
|
|
827
|
+
// ── Marcus-specific criteria ────────────────────────────
|
|
828
|
+
|
|
829
|
+
const marcusCriteria: EvalCriterion[] = [
|
|
830
|
+
{
|
|
831
|
+
id: "COMP-9",
|
|
832
|
+
rule: "Total tool calls <= 40",
|
|
833
|
+
weight: 10,
|
|
834
|
+
source: "marcus.md § Efficiency Rules",
|
|
835
|
+
check(data) {
|
|
836
|
+
const total = data.calls.length;
|
|
837
|
+
return {
|
|
838
|
+
verdict: total <= 40 ? "FOLLOWED" : "IGNORED",
|
|
839
|
+
evidence: `${total} calls`,
|
|
840
|
+
};
|
|
841
|
+
},
|
|
842
|
+
},
|
|
843
|
+
{
|
|
844
|
+
id: "COMP-10",
|
|
845
|
+
rule: "Grep:Read ratio <= 2:1",
|
|
846
|
+
weight: 10,
|
|
847
|
+
source: "prompts/coding-principles.md",
|
|
848
|
+
check(data) {
|
|
849
|
+
const grepBashes = data.bashes.filter((b) => b.includes("grep"));
|
|
850
|
+
const ratio = data.reads.length > 0 ? grepBashes.length / data.reads.length : 0;
|
|
851
|
+
return {
|
|
852
|
+
verdict: ratio <= 2 ? "FOLLOWED" : "IGNORED",
|
|
853
|
+
evidence: `${grepBashes.length} greps, ${data.reads.length} reads (ratio: ${ratio.toFixed(1)})`,
|
|
854
|
+
};
|
|
855
|
+
},
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
id: "COMP-2",
|
|
859
|
+
// The number comes from the guard that enforces it, not from a copy here.
|
|
860
|
+
// Two independent statements of one limit is a drift waiting to happen,
|
|
861
|
+
// and the direction it drifts is the dangerous one: a grader still saying
|
|
862
|
+
// "2" after the guard moved to 4 marks a compliant run as IGNORED, and
|
|
863
|
+
// the run that reads worst is the one that obeyed.
|
|
864
|
+
rule: `<= ${DEFAULT_MAX_RUNS_PER_WORKER} full suite runs (targeted runs are unlimited)`,
|
|
865
|
+
weight: 10,
|
|
866
|
+
source: "marcus.md § Testing Rules",
|
|
867
|
+
check(data) {
|
|
868
|
+
const fullSuiteRuns = data.bashes.filter(
|
|
869
|
+
(b) => /\bbun test\s*$/.test(b.trim()) || (/\bbun test\b/.test(b) && !b.includes("test/") && !b.includes(".test.") && !b.includes("grep"))
|
|
870
|
+
);
|
|
871
|
+
const targetedRuns = data.bashes.filter(
|
|
872
|
+
(b) => /\bbun test\b/.test(b) && (b.includes("test/") || b.includes(".test.")) && !b.includes("grep")
|
|
873
|
+
);
|
|
874
|
+
return {
|
|
875
|
+
verdict: fullSuiteRuns.length <= DEFAULT_MAX_RUNS_PER_WORKER ? "FOLLOWED" : "IGNORED",
|
|
876
|
+
evidence: `${fullSuiteRuns.length} full suite runs, ${targetedRuns.length} targeted runs`,
|
|
877
|
+
};
|
|
878
|
+
},
|
|
879
|
+
},
|
|
880
|
+
{
|
|
881
|
+
id: "COMP-3",
|
|
882
|
+
rule: "Use golden fixture pattern for test files",
|
|
883
|
+
weight: 10,
|
|
884
|
+
source: "INSTRUCTION-COMPLIANCE-SPEC § COMP-3",
|
|
885
|
+
check(data) {
|
|
886
|
+
const testWrites = data.edits.concat(data.writes).filter(
|
|
887
|
+
(p) => p.includes("test/") && p.includes(".test.")
|
|
888
|
+
);
|
|
889
|
+
if (testWrites.length === 0) {
|
|
890
|
+
return { verdict: "FOLLOWED", evidence: "No test files written — exempt" };
|
|
891
|
+
}
|
|
892
|
+
return { verdict: "FOLLOWED", evidence: `${testWrites.length} test file(s) written` };
|
|
893
|
+
},
|
|
894
|
+
},
|
|
895
|
+
{
|
|
896
|
+
id: "COMP-4",
|
|
897
|
+
rule: "Update spec-drift hash after modifying spec",
|
|
898
|
+
weight: 10,
|
|
899
|
+
source: "INSTRUCTION-COMPLIANCE-SPEC § COMP-4",
|
|
900
|
+
check(data) {
|
|
901
|
+
const violations = specHashViolations(data, cachedSpecHashGuards());
|
|
902
|
+
return violations.length === 0
|
|
903
|
+
? { verdict: "FOLLOWED", evidence: "No hash-guarded spec modified without its hash being updated" }
|
|
904
|
+
: { verdict: "IGNORED", evidence: violations.join("; ") };
|
|
905
|
+
},
|
|
906
|
+
},
|
|
907
|
+
{
|
|
908
|
+
id: "COMP-5",
|
|
909
|
+
rule: "Governing spec context available (read or injected) when touching spec'd area",
|
|
910
|
+
weight: 15,
|
|
911
|
+
source: "marcus.md § Context",
|
|
912
|
+
check(data) {
|
|
913
|
+
const firstEdit = data.calls.findIndex(
|
|
914
|
+
(c) => c.name === "Edit" || c.name === "Write"
|
|
915
|
+
);
|
|
916
|
+
if (firstEdit < 0) {
|
|
917
|
+
return { verdict: "FOLLOWED", evidence: "No edits — spec read not required" };
|
|
918
|
+
}
|
|
919
|
+
const specRead = data.calls.slice(0, firstEdit).some(
|
|
920
|
+
(c) => c.name === "Read" && (c.input.file_path || "").includes("specs/")
|
|
921
|
+
);
|
|
922
|
+
if (specRead) {
|
|
923
|
+
return { verdict: "FOLLOWED", evidence: "Spec read before edits" };
|
|
924
|
+
}
|
|
925
|
+
const specInjected = data.promptContent?.includes("specs/") || data.promptContent?.includes("Governing spec");
|
|
926
|
+
if (specInjected) {
|
|
927
|
+
return { verdict: "FOLLOWED", evidence: "Spec content injected in prompt" };
|
|
928
|
+
}
|
|
929
|
+
const editPaths = data.edits.concat(data.writes);
|
|
930
|
+
const touchesSpecArea = editPaths.some(
|
|
931
|
+
(p) => p.includes("lib/") || p.includes("gates/") || p.includes("hooks/") || p.includes("workflows/")
|
|
932
|
+
);
|
|
933
|
+
if (!touchesSpecArea) {
|
|
934
|
+
return { verdict: "FOLLOWED", evidence: "Task does not touch spec'd area — spec read not required" };
|
|
935
|
+
}
|
|
936
|
+
return {
|
|
937
|
+
verdict: "IGNORED",
|
|
938
|
+
evidence: "No spec read or injection before first edit (touches spec'd area)",
|
|
939
|
+
};
|
|
940
|
+
},
|
|
941
|
+
},
|
|
942
|
+
{
|
|
943
|
+
id: "COMP-11",
|
|
944
|
+
rule: "Coding/testing principles available (read or injected) for core changes",
|
|
945
|
+
weight: 15,
|
|
946
|
+
source: "marcus.md § Context",
|
|
947
|
+
check(data) {
|
|
948
|
+
if (data.edits.length === 0 && data.writes.length === 0) {
|
|
949
|
+
return { verdict: "FOLLOWED", evidence: "No code written — exempt" };
|
|
950
|
+
}
|
|
951
|
+
const promptReads = data.reads.filter((r) => r.includes("prompts/"));
|
|
952
|
+
if (promptReads.length > 0) {
|
|
953
|
+
return {
|
|
954
|
+
verdict: "FOLLOWED",
|
|
955
|
+
evidence: `${promptReads.length} prompt(s) read: ${promptReads.map((r) => basename(r)).join(", ")}`,
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
const principlesInjected = data.promptContent?.includes("prompts/") ||
|
|
959
|
+
data.promptContent?.includes("Coding Principles") ||
|
|
960
|
+
data.promptContent?.includes("coding-principles");
|
|
961
|
+
if (principlesInjected) {
|
|
962
|
+
return { verdict: "FOLLOWED", evidence: "Coding/testing principles injected in prompt" };
|
|
963
|
+
}
|
|
964
|
+
const editPaths = data.edits.concat(data.writes);
|
|
965
|
+
const touchesCoreLib = editPaths.some(
|
|
966
|
+
(p) => p.includes("lib/") || p.includes("src/")
|
|
967
|
+
);
|
|
968
|
+
const multipleFiles = editPaths.length > 3;
|
|
969
|
+
if (!touchesCoreLib && !multipleFiles) {
|
|
970
|
+
return { verdict: "FOLLOWED", evidence: "Simple task — prompts/ read not required" };
|
|
971
|
+
}
|
|
972
|
+
return {
|
|
973
|
+
verdict: "IGNORED",
|
|
974
|
+
evidence: "No coding principles read or injected (touches core lib or multi-file change)",
|
|
975
|
+
};
|
|
976
|
+
},
|
|
977
|
+
},
|
|
978
|
+
{
|
|
979
|
+
id: "COMP-12",
|
|
980
|
+
rule: "Grep before Read for non-key files",
|
|
981
|
+
weight: 5,
|
|
982
|
+
source: "memory: Grep Before Read",
|
|
983
|
+
check(data) {
|
|
984
|
+
const writtenFiles = new Set(
|
|
985
|
+
data.calls.filter((c) => c.name === "Write" || c.name === "Edit")
|
|
986
|
+
.map((c) => c.input.file_path || "")
|
|
987
|
+
);
|
|
988
|
+
const readWithoutGrep = data.reads.filter((r) => {
|
|
989
|
+
if (r.includes("/tmp/claude-") || r.includes(".output")) return false;
|
|
990
|
+
const readIdx = data.calls.findIndex(
|
|
991
|
+
(c) => c.name === "Read" && c.input.file_path === r
|
|
992
|
+
);
|
|
993
|
+
if (readIdx > 0 && writtenFiles.has(r)) return false;
|
|
994
|
+
const fn = basename(r);
|
|
995
|
+
if (
|
|
996
|
+
["AGENTS.md", "PROJECT-STATE.md", "CLAUDE.md", "rungate.json", "SCHEMA-GUIDE.md"].includes(fn)
|
|
997
|
+
)
|
|
998
|
+
return false;
|
|
999
|
+
const readCall = data.calls[readIdx];
|
|
1000
|
+
if (readCall?.input?.offset || readCall?.input?.limit) return false;
|
|
1001
|
+
const priorGrep = data.calls
|
|
1002
|
+
.slice(0, readIdx)
|
|
1003
|
+
.some((c) => c.name === "Bash" && (c.input.command || "").includes(basename(r)));
|
|
1004
|
+
return !priorGrep;
|
|
1005
|
+
});
|
|
1006
|
+
const ratio = data.reads.length > 0 ? readWithoutGrep.length / data.reads.length : 0;
|
|
1007
|
+
return {
|
|
1008
|
+
verdict: ratio <= 0.5 ? "FOLLOWED" : "IGNORED",
|
|
1009
|
+
evidence: `${readWithoutGrep.length}/${data.reads.length} reads without prior grep (${Math.round(ratio * 100)}%)`,
|
|
1010
|
+
};
|
|
1011
|
+
},
|
|
1012
|
+
},
|
|
1013
|
+
];
|
|
1014
|
+
|
|
1015
|
+
// ── Quinn-specific criteria ─────────────────────────────
|
|
1016
|
+
|
|
1017
|
+
const quinnCriteria: EvalCriterion[] = [
|
|
1018
|
+
{
|
|
1019
|
+
id: "Q-01",
|
|
1020
|
+
rule: "Run full test suite (bun test)",
|
|
1021
|
+
weight: 20,
|
|
1022
|
+
source: "quinn.md § Core Principles",
|
|
1023
|
+
check(data) {
|
|
1024
|
+
const bunTestRuns = data.bashes.filter((b) => /\bbun test\b/.test(b));
|
|
1025
|
+
return {
|
|
1026
|
+
verdict: bunTestRuns.length > 0 ? "FOLLOWED" : "IGNORED",
|
|
1027
|
+
evidence:
|
|
1028
|
+
bunTestRuns.length > 0
|
|
1029
|
+
? `${bunTestRuns.length} bun test run(s)`
|
|
1030
|
+
: "bun test never run",
|
|
1031
|
+
};
|
|
1032
|
+
},
|
|
1033
|
+
},
|
|
1034
|
+
{
|
|
1035
|
+
id: "Q-02",
|
|
1036
|
+
rule: "Run type check (tsc --noEmit)",
|
|
1037
|
+
weight: 15,
|
|
1038
|
+
source: "quinn.md § Before reporting done",
|
|
1039
|
+
check(data) {
|
|
1040
|
+
const tscRuns = data.bashes.filter((b) => b.includes("tsc") && b.includes("--noEmit"));
|
|
1041
|
+
return {
|
|
1042
|
+
verdict: tscRuns.length > 0 ? "FOLLOWED" : "IGNORED",
|
|
1043
|
+
evidence:
|
|
1044
|
+
tscRuns.length > 0
|
|
1045
|
+
? `${tscRuns.length} tsc --noEmit run(s)`
|
|
1046
|
+
: "tsc --noEmit never run",
|
|
1047
|
+
};
|
|
1048
|
+
},
|
|
1049
|
+
},
|
|
1050
|
+
{
|
|
1051
|
+
id: "Q-03",
|
|
1052
|
+
rule: "Verify AC evidence is not self-attested",
|
|
1053
|
+
weight: 15,
|
|
1054
|
+
source: "quinn.md § Never Do",
|
|
1055
|
+
check(data) {
|
|
1056
|
+
// Quinn should read test output or grep for evidence, not just assert it
|
|
1057
|
+
const evidenceBashes = data.bashes.filter(
|
|
1058
|
+
(b) => b.includes("grep") || b.includes("bun test") || b.includes("tsc")
|
|
1059
|
+
);
|
|
1060
|
+
return {
|
|
1061
|
+
verdict: evidenceBashes.length >= 2 ? "FOLLOWED" : "IGNORED",
|
|
1062
|
+
evidence: `${evidenceBashes.length} verification commands found`,
|
|
1063
|
+
};
|
|
1064
|
+
},
|
|
1065
|
+
},
|
|
1066
|
+
{
|
|
1067
|
+
id: "Q-04",
|
|
1068
|
+
rule: "No direct code edits (validation only)",
|
|
1069
|
+
weight: 20,
|
|
1070
|
+
source: "quinn.md § Core Principles",
|
|
1071
|
+
check(data) {
|
|
1072
|
+
const codeEdits = [...data.edits, ...data.writes].filter(
|
|
1073
|
+
(p) => /\.(ts|js|json)$/.test(p) && !p.includes("test")
|
|
1074
|
+
);
|
|
1075
|
+
return {
|
|
1076
|
+
verdict: codeEdits.length === 0 ? "FOLLOWED" : "IGNORED",
|
|
1077
|
+
evidence:
|
|
1078
|
+
codeEdits.length === 0
|
|
1079
|
+
? "No code edits — validation only"
|
|
1080
|
+
: `${codeEdits.length} code file(s) edited: ${codeEdits.map((p) => basename(p)).join(", ")}`,
|
|
1081
|
+
};
|
|
1082
|
+
},
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
id: "Q-05",
|
|
1086
|
+
rule: "Total tool calls <= 30",
|
|
1087
|
+
weight: 10,
|
|
1088
|
+
source: "quinn.md § Efficiency",
|
|
1089
|
+
check(data) {
|
|
1090
|
+
const total = data.calls.length;
|
|
1091
|
+
return {
|
|
1092
|
+
verdict: total <= 30 ? "FOLLOWED" : "IGNORED",
|
|
1093
|
+
evidence: `${total} calls`,
|
|
1094
|
+
};
|
|
1095
|
+
},
|
|
1096
|
+
},
|
|
1097
|
+
];
|
|
1098
|
+
|
|
1099
|
+
// ── Role registry ───────────────────────────────────────
|
|
1100
|
+
|
|
1101
|
+
const roleRegistry: Record<Role, EvalCriterion[]> = {
|
|
1102
|
+
da: [...sharedCriteria, ...daCriteria],
|
|
1103
|
+
marcus: [...sharedCriteria, ...marcusCriteria],
|
|
1104
|
+
quinn: [...sharedCriteria, ...quinnCriteria],
|
|
1105
|
+
};
|
|
1106
|
+
|
|
1107
|
+
/**
|
|
1108
|
+
* Get eval criteria for a given role.
|
|
1109
|
+
* Combines shared criteria with role-specific ones.
|
|
1110
|
+
*/
|
|
1111
|
+
export function getCriteria(role: Role): EvalCriterion[] {
|
|
1112
|
+
return roleRegistry[role] ?? [];
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* Evaluate transcript data against a role's criteria.
|
|
1117
|
+
* Returns per-criterion results with FOLLOWED/IGNORED verdicts.
|
|
1118
|
+
*/
|
|
1119
|
+
export function evaluateCriteria(
|
|
1120
|
+
role: Role,
|
|
1121
|
+
data: TranscriptData
|
|
1122
|
+
): CriterionResult[] {
|
|
1123
|
+
const criteria = getCriteria(role);
|
|
1124
|
+
return criteria.map((criterion) => {
|
|
1125
|
+
const { verdict, evidence } = criterion.check(data);
|
|
1126
|
+
return {
|
|
1127
|
+
id: criterion.id,
|
|
1128
|
+
rule: criterion.rule,
|
|
1129
|
+
verdict,
|
|
1130
|
+
evidence,
|
|
1131
|
+
weight: criterion.weight,
|
|
1132
|
+
source: criterion.source,
|
|
1133
|
+
};
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/**
|
|
1138
|
+
* All supported roles.
|
|
1139
|
+
*/
|
|
1140
|
+
export const ROLES: readonly Role[] = ["da", "marcus", "quinn"] as const;
|