rungate 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (282) hide show
  1. package/bin/rungate.ts +70 -0
  2. package/lib/aes-calculator.ts +66 -0
  3. package/lib/agent-audit.ts +139 -0
  4. package/lib/agent-brief-validation.ts +359 -0
  5. package/lib/agents-md-refs.ts +90 -0
  6. package/lib/bash-file-read.ts +197 -0
  7. package/lib/behavioral-cache.ts +78 -0
  8. package/lib/branch-cleanup.ts +234 -0
  9. package/lib/brief-context-parser.ts +35 -0
  10. package/lib/canary.ts +117 -0
  11. package/lib/check-registry.ts +226 -0
  12. package/lib/ci-checkout.ts +400 -0
  13. package/lib/cli-commands.ts +137 -0
  14. package/lib/compliance-backfill.ts +220 -0
  15. package/lib/compliance-report.ts +504 -0
  16. package/lib/compliance.ts +361 -0
  17. package/lib/config-loader.ts +255 -0
  18. package/lib/conformity.ts +1818 -131
  19. package/lib/container-provenance.ts +230 -0
  20. package/lib/create-brief.ts +150 -0
  21. package/lib/create-sc.ts +272 -0
  22. package/lib/deep-merge.ts +69 -0
  23. package/lib/dev-liveness.ts +214 -0
  24. package/lib/directive-extractor.ts +210 -0
  25. package/lib/distribution-standing.ts +113 -0
  26. package/lib/doc-hygiene.ts +238 -0
  27. package/lib/eval-criteria.ts +6 -0
  28. package/lib/evidence-portability.ts +123 -0
  29. package/lib/evidence-prevalidator.ts +390 -0
  30. package/lib/evidence-required.ts +150 -0
  31. package/lib/evidence-satisfiability.ts +288 -0
  32. package/lib/frontmatter.ts +163 -0
  33. package/lib/gap-scanner.ts +348 -0
  34. package/lib/gate-enforcement.ts +366 -0
  35. package/lib/generators/agent-briefs.ts +136 -0
  36. package/lib/generators/agents-md.ts +506 -0
  37. package/lib/generators/code-map.ts +60 -0
  38. package/lib/generators/types.ts +184 -0
  39. package/lib/git-ref-normalizer.ts +107 -0
  40. package/lib/github.ts +539 -0
  41. package/lib/harness-identity.ts +63 -0
  42. package/lib/hill-climb.ts +164 -0
  43. package/lib/instruction-effectiveness.ts +131 -0
  44. package/lib/issue-closure-audit.ts +225 -0
  45. package/lib/issue-create-override.ts +343 -0
  46. package/lib/js-top-level-code.ts +250 -0
  47. package/lib/liveness-contradiction.ts +149 -0
  48. package/lib/organize.ts +505 -0
  49. package/lib/paths.ts +197 -6
  50. package/lib/post-fix-verify.ts +157 -0
  51. package/lib/prior-branch.ts +489 -0
  52. package/lib/project-state-shape.ts +166 -0
  53. package/lib/promote-outputs.ts +156 -0
  54. package/lib/prove-label-gate.ts +35 -0
  55. package/lib/reachability.ts +302 -0
  56. package/lib/release-preflight.ts +186 -0
  57. package/lib/rule-health-pipeline.ts +264 -0
  58. package/lib/rule-health-trend.ts +161 -0
  59. package/lib/rule-provenance.ts +149 -0
  60. package/lib/rule-registry.ts +128 -0
  61. package/lib/rungate-schema.ts +114 -4
  62. package/lib/sc-claim-forms.ts +179 -0
  63. package/lib/sc-cover.ts +150 -0
  64. package/lib/sc-guard.ts +157 -0
  65. package/lib/sc-ids.ts +344 -0
  66. package/lib/scaffold/defaults.ts +165 -0
  67. package/lib/scaffold/managed-workflow.ts +370 -0
  68. package/lib/scaffold/owned-files.ts +265 -0
  69. package/lib/scaffold/steps.ts +2680 -0
  70. package/lib/scanner.ts +701 -0
  71. package/lib/screenshot-upload.ts +73 -0
  72. package/lib/secret-patterns.ts +112 -0
  73. package/lib/security-verdict.ts +616 -0
  74. package/lib/signal-phrases.ts +1 -0
  75. package/lib/spec-change-conformity.ts +80 -0
  76. package/lib/spec-files.ts +62 -0
  77. package/lib/spec-registry.ts +133 -0
  78. package/lib/spec-updater.ts +96 -0
  79. package/lib/stale-cleanup.ts +224 -0
  80. package/lib/stale-issue-scanner.ts +36 -0
  81. package/lib/strip-js-comments.ts +269 -0
  82. package/lib/suite-measurement.ts +228 -0
  83. package/lib/task-completion-checks.ts +177 -0
  84. package/lib/test-suite-lock.ts +1136 -0
  85. package/lib/transcript-checker.ts +1140 -0
  86. package/lib/validators/spec-validators.ts +171 -0
  87. package/lib/verdict-capture.ts +73 -0
  88. package/lib/verified-push.ts +358 -0
  89. package/lib/workflow-security.ts +174 -0
  90. package/lib/worktree-cleanup.ts +250 -0
  91. package/lib/worktree-collect.ts +283 -0
  92. package/lib/worktree-isolation.ts +82 -0
  93. package/package.json +36 -3
  94. package/prompts/ac-adversary.md +14 -2
  95. package/prompts/aditi.md +5 -0
  96. package/prompts/blast-radius.md +5 -0
  97. package/prompts/container-verify.md +5 -0
  98. package/prompts/discovery.md +4 -0
  99. package/prompts/environment.md +5 -0
  100. package/prompts/evidence-hierarchy.md +5 -0
  101. package/prompts/evidence-validator.md +11 -0
  102. package/prompts/marcus.md +5 -0
  103. package/prompts/prevention.md +5 -0
  104. package/prompts/prove-reproducer.md +12 -0
  105. package/prompts/quinn-decision-tree.md +5 -0
  106. package/prompts/quinn-ui-brief.md +5 -0
  107. package/prompts/quinn.md +5 -0
  108. package/prompts/rca.md +5 -0
  109. package/prompts/read-before-write.md +4 -0
  110. package/prompts/regression.md +5 -0
  111. package/prompts/serena.md +5 -0
  112. package/scripts/audit-specs.ts +464 -0
  113. package/scripts/convert-spec.ts +293 -0
  114. package/scripts/create-sc.ts +92 -0
  115. package/scripts/create-spec.ts +166 -0
  116. package/scripts/extract-constraints.ts +4 -3
  117. package/scripts/generate-code-map.ts +1 -1
  118. package/scripts/scaffold-project.ts +138 -1426
  119. package/scripts/sync-spec-tests.ts +211 -126
  120. package/.claude/agents/aditi.md +0 -47
  121. package/.claude/agents/marcus.md +0 -50
  122. package/.claude/agents/quinn.md +0 -58
  123. package/.claude/agents/rook.md +0 -47
  124. package/.claude/agents/serena.md +0 -46
  125. package/.claude/rungate.json +0 -11
  126. package/.fallowrc.json +0 -28
  127. package/.github/copilot-instructions.md +0 -20
  128. package/.rungate/conformity-findings.json +0 -648
  129. package/AGENTS.md +0 -106
  130. package/CODE-MAP.md +0 -50
  131. package/HARNESS.md +0 -88
  132. package/bun.lock +0 -26
  133. package/bunfig.toml +0 -2
  134. package/config/brief-policies.json +0 -28
  135. package/config/ceremony-profiles.json +0 -212
  136. package/config/consumer-cascade-map.json +0 -24
  137. package/config/doc-cascade-map.json +0 -24
  138. package/config/eval.json +0 -84
  139. package/config/profiles/account-plan.yaml +0 -36
  140. package/config/profiles/campaign.yaml +0 -94
  141. package/config/prove-evidence.schema.json +0 -65
  142. package/config/spec-policies.json +0 -36
  143. package/config/workflow-schema.json +0 -281
  144. package/gates/SCHEMA-GUIDE.md +0 -121
  145. package/gates/ac-quality.test.ts +0 -106
  146. package/gates/adversarial.test.ts +0 -274
  147. package/gates/brief-assembler.test.ts +0 -429
  148. package/gates/brief-assembler.ts +0 -334
  149. package/gates/bunfig.toml +0 -2
  150. package/gates/chain.test.ts +0 -91
  151. package/gates/e2e-smoke.test.ts +0 -234
  152. package/gates/error-classifier.test.ts +0 -249
  153. package/gates/error-classifier.ts +0 -89
  154. package/gates/orchestrator.test.ts +0 -950
  155. package/gates/orchestrator.ts +0 -569
  156. package/gates/preload.ts +0 -77
  157. package/gates/prompts/ac-adversary.md +0 -20
  158. package/gates/prompts/evidence-validator.md +0 -22
  159. package/gates/prompts/prove-reproducer.md +0 -53
  160. package/gates/prove.test.ts +0 -164
  161. package/gates/run-gate.ts +0 -936
  162. package/gates/schema-parity.test.ts +0 -197
  163. package/gates/schema.ts +0 -362
  164. package/gates/self-heal.test.ts +0 -511
  165. package/gates/self-heal.ts +0 -121
  166. package/gates/ship-orchestrator.test.ts +0 -379
  167. package/gates/ship-orchestrator.ts +0 -363
  168. package/gates/test-fixtures/broken-no-environments.json +0 -60
  169. package/gates/test-fixtures/valid-standard.json +0 -54
  170. package/gates/test-utils.ts +0 -43
  171. package/gates/witness.ts +0 -220
  172. package/gates/workflow.test.ts +0 -1053
  173. package/hooks/AgentBriefGuard.hook.ts +0 -586
  174. package/hooks/AgentVerdictCapture.hook.ts +0 -134
  175. package/hooks/AutoVerifyGate.hook.ts +0 -76
  176. package/hooks/GateEnforcement.hook.ts +0 -273
  177. package/hooks/IssueCloseGuard.hook.ts +0 -116
  178. package/hooks/MergeGuard.hook.ts +0 -55
  179. package/hooks/StaleTTLCleanup.hook.ts +0 -133
  180. package/hooks/VerifyPhaseLock.hook.ts +0 -81
  181. package/hooks/WorkflowStateGuard.hook.ts +0 -37
  182. package/hooks/lib/agentDetection.ts +0 -70
  183. package/hooks/lib/findWorkflow.ts +0 -101
  184. package/hooks/lib/parseStdin.ts +0 -37
  185. package/hooks/lib/paths.ts +0 -111
  186. package/hooks/lib/utils.ts +0 -81
  187. package/reference/migration-manifest.json +0 -554
  188. package/reference/prompts/agent-aditi.md +0 -38
  189. package/reference/prompts/agent-marcus.md +0 -38
  190. package/reference/prompts/agent-quinn.md +0 -38
  191. package/reference/prompts/agent-rook.md +0 -37
  192. package/reference/prompts/agent-serena.md +0 -37
  193. package/reference/prompts/council-skill.md +0 -170
  194. package/reference/prompts/goal-skill.md +0 -235
  195. package/reference/prompts/prove-skill.md +0 -283
  196. package/reference/prompts/quinn-standard.md +0 -246
  197. package/reference/prompts/ship-convergence.md +0 -65
  198. package/reference/prompts/ship-escalation.md +0 -43
  199. package/reference/prompts/ship-iteration.md +0 -382
  200. package/reference/prompts/ship-pending-patches.md +0 -246
  201. package/reference/prompts/ship-reference.md +0 -250
  202. package/reference/prompts/ship-skill.md +0 -347
  203. package/reference/prompts/ship-templates.md +0 -215
  204. package/reference/scripts/batch-status.sh +0 -68
  205. package/reference/scripts/chain-compat-check.sh +0 -67
  206. package/reference/scripts/context-audit.sh +0 -314
  207. package/reference/scripts/dead-export-check.sh +0 -61
  208. package/reference/scripts/generate-qc-tables.sh +0 -92
  209. package/reference/scripts/harness-init.sh +0 -111
  210. package/reference/scripts/pai-054-baseline-check.sh +0 -40
  211. package/reference/scripts/prove-smoke-test.sh +0 -112
  212. package/reference/scripts/setup-keychain-salt.sh +0 -12
  213. package/reference/scripts/skill-optimizer-friday.sh +0 -88
  214. package/reference/scripts/skill-qc-test.sh +0 -351
  215. package/reference/scripts/skill-synthesize-tonight.sh +0 -66
  216. package/reference/specs/HARNESS-EXTRACTION-SPEC.md +0 -263
  217. package/reference/specs/harness-v3-bash-deletion-plan.md +0 -65
  218. package/reference/specs/harness-v3-migration-plan.md +0 -188
  219. package/scripts/decision-reconcile.sh +0 -74
  220. package/scripts/decision-reconcile.ts +0 -152
  221. package/scripts/git-hooks/pre-commit +0 -76
  222. package/scripts/git-hooks/pre-push +0 -271
  223. package/scripts/harness-skill-check.sh +0 -52
  224. package/scripts/lib/gap-emit.sh +0 -65
  225. package/scripts/lib/jq-update.sh +0 -30
  226. package/scripts/lib/resolve-slug.sh +0 -30
  227. package/scripts/lib/verify-spec-compliance.sh +0 -102
  228. package/scripts/prove-backfill.sh +0 -87
  229. package/scripts/scaffold-rungate-config.ts +0 -91
  230. package/scripts/self-containment-check.sh +0 -109
  231. package/scripts/skill-qc-validator.sh +0 -402
  232. package/scripts/slug-resolver.sh +0 -14
  233. package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +0 -2493
  234. package/specs/BOOTSTRAP-TEST-PLAN.md +0 -571
  235. package/specs/HARNESS-GATES.md +0 -256
  236. package/specs/HARNESS-SKILL-CHAIN.md +0 -286
  237. package/specs/HARNESS-SKILL-CONTRACT.md +0 -93
  238. package/specs/HARNESS-STANDARD.md +0 -488
  239. package/specs/SPEC-TEMPLATE.md +0 -53
  240. package/specs/bootstrap-data-flow.png +0 -0
  241. package/specs/harness-automation-matrix.md +0 -314
  242. package/specs/harness-automation-matrix.rules.json +0 -64
  243. package/test/anti.test.ts +0 -80
  244. package/test/contract-negative.test.ts +0 -86
  245. package/test/contract.test.ts +0 -158
  246. package/test/external-deps.test.ts +0 -48
  247. package/test/fixtures/golden-project/.env.example +0 -3
  248. package/test/fixtures/golden-project/Makefile +0 -7
  249. package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +0 -23
  250. package/test/fixtures/golden-project/docs/setup-guide.md +0 -10
  251. package/test/fixtures/golden-project/package.json +0 -10
  252. package/test/fixtures/golden-project/specs/api-spec.md +0 -20
  253. package/test/fixtures/golden-project/src/auth.ts +0 -8
  254. package/test/fixtures/golden-project/src/db.ts +0 -11
  255. package/test/fixtures/golden-project/src/index.ts +0 -19
  256. package/test/fixtures/golden-project/src/utils/format.ts +0 -3
  257. package/test/fixtures/golden-project/test/auth.test.ts +0 -15
  258. package/test/fixtures/golden-project/tsconfig.json +0 -12
  259. package/test/meta-sc-coverage.test.ts +0 -159
  260. package/test/phase-0.test.ts +0 -846
  261. package/test/phase-1.test.ts +0 -617
  262. package/test/phase-2.test.ts +0 -620
  263. package/test/phase-3.test.ts +0 -99
  264. package/test/scaffold-conformity.test.ts +0 -8
  265. package/test/schema-canary.test.ts +0 -51
  266. package/test/spec-compliance-auto.test.ts +0 -64
  267. package/test/spec-compliance.test.ts +0 -295
  268. package/test/spec-discovery.test.ts +0 -68
  269. package/test/spec-drift.test.ts +0 -5
  270. package/test/structure.test.ts +0 -165
  271. package/test/unit/agent-brief-guard-ship.test.ts +0 -293
  272. package/test/unit/extract-constraints.test.ts +0 -145
  273. package/test/unit/issue-close-guard.test.ts +0 -204
  274. package/test/unit/scaffold-project.test.ts +0 -417
  275. package/test/unit/skill-cooccurrence.test.ts +0 -177
  276. package/test/unit/skill-sequence-logger.test.ts +0 -244
  277. package/test/unit/workflow-state-guard.test.ts +0 -95
  278. package/workflows/batch-ship.js +0 -295
  279. package/workflows/council.js +0 -481
  280. package/workflows/prove.js +0 -560
  281. package/workflows/ship.js +0 -789
  282. package/workflows/verify.js +0 -236
@@ -0,0 +1,1140 @@
1
+ /**
2
+ * Transcript compliance checker — cross-references extracted directives
3
+ * against agent tool calls producing FOLLOWED or IGNORED verdict per directive.
4
+ *
5
+ * Also includes role-based evaluation criteria for DA compliance auditing.
6
+ *
7
+ * SC-402: Transcript checker: FOLLOWED/IGNORED per directive
8
+ */
9
+
10
+ import { readFileSync, readdirSync, existsSync, type Dirent } from "fs";
11
+ import { basename, join } from "path";
12
+ import { harnessRootFor } from "./paths.js";
13
+ import { stripJsComments } from "./strip-js-comments.js";
14
+ import type { Directive } from "./directive-extractor.js";
15
+ import { checkCanaries, type CanaryDefinition, type CanaryReport } from "./canary.js";
16
+ import { isBashFileRead } from "./bash-file-read.js";
17
+ // COMP-2's limit is the guard's limit. One number, one owner.
18
+ //
19
+ // The POLICY constant, deliberately NOT the environment-aware resolver.
20
+ // Security review of d6cc5acf called the first version a self-grading bypass
21
+ // and was right: resolving through the environment would let a run that spent
22
+ // eight suites be graded against a threshold of eight. A grader measures
23
+ // against the policy, never against whatever the graded run's session happens
24
+ // to be configured for.
25
+ //
26
+ // The resolver is named here only in prose, never as a call — a comment that
27
+ // writes out the forbidden form is itself enough to defeat the check that
28
+ // forbids it, which is how the first attempt at this test went red.
29
+ import { DEFAULT_MAX_RUNS_PER_WORKER } from "./test-suite-lock.js";
30
+
31
+ type ComplianceVerdict = "FOLLOWED" | "IGNORED" | "VIOLATED" | "N/A";
32
+
33
+ export interface ComplianceResult {
34
+ directive: Directive;
35
+ status: ComplianceVerdict;
36
+ evidence: string;
37
+ }
38
+
39
+ export interface ToolCall {
40
+ name: string;
41
+ input: Record<string, any>;
42
+ order?: number;
43
+ }
44
+
45
+ // ── Role-based evaluation (from eval-criteria) ──────────
46
+
47
+ export interface TranscriptData {
48
+ calls: ToolCall[];
49
+ reads: string[];
50
+ bashes: string[];
51
+ edits: string[];
52
+ writes: string[];
53
+ duplicateReads: Record<string, number>;
54
+ firstThreeReads: string[];
55
+ promptContent: string;
56
+ }
57
+
58
+ export type Verdict = "FOLLOWED" | "IGNORED";
59
+
60
+ // ── COMP-4: spec-drift hashes (#287) ──────────────────────
61
+
62
+ export interface SpecHashGuards {
63
+ /** Spec basenames that some guard file actually holds a hash for. */
64
+ specs: string[];
65
+ /** Repo-relative paths of the files holding those hashes. */
66
+ guardFiles: string[];
67
+ }
68
+
69
+ /** Where a future generated hash manifest would live (DEC-011). */
70
+ const DRIFT_HASHES = "test/drift-hashes.json";
71
+
72
+ /**
73
+ * Discover which specs actually have a drift hash guarding them.
74
+ *
75
+ * Read out of the tree rather than written down here. Two specs are guarded
76
+ * today — `BOOTSTRAP-DATA-FLOW-SPEC.md` and `BOOTSTRAP-TEST-PLAN.md` — and a
77
+ * list of them in this file is the hardcoding defect class #343/#344 were
78
+ * about: it would be correct until someone added a third hash, and then wrong
79
+ * silently.
80
+ *
81
+ * It was also half of #287. COMP-4 fired on any path containing `specs/`, so
82
+ * editing `specs/HARNESS-STANDARD.md` — which is where SC records go, so
83
+ * nearly every run edits it — was graded IGNORED at weight 10 for doing the
84
+ * correct thing, clearable only by running a `shasum` whose output nothing
85
+ * read.
86
+ *
87
+ * Prefers a generated manifest if one ever lands (DEC-011 in
88
+ * specs/DOC-HYGIENE-ARCHITECTURE-SPEC.md plans to replace the hand-maintained
89
+ * constants). Until then it scans the guard sources for a hash constant and
90
+ * takes the spec filenames named alongside it.
91
+ *
92
+ * An unreadable tree yields an EMPTY set, which exempts rather than accuses.
93
+ * That is the right direction for a grader: a check that cannot see the
94
+ * guards must not report that an agent skipped one.
95
+ */
96
+ export function discoverSpecHashGuards(root: string): SpecHashGuards {
97
+ const manifest = join(root, DRIFT_HASHES);
98
+ if (existsSync(manifest)) {
99
+ try {
100
+ const j = JSON.parse(readFileSync(manifest, "utf-8"));
101
+ const specs = Object.keys(j?.specs ?? j ?? {}).filter(k => k.endsWith(".md"));
102
+ if (specs.length > 0) return { specs: specs.sort(), guardFiles: [DRIFT_HASHES] };
103
+ } catch {
104
+ // A manifest that exists but cannot be parsed is not a manifest that is
105
+ // absent — fall through to the scan rather than reporting no guards.
106
+ }
107
+ }
108
+
109
+ const specs = new Set<string>();
110
+ const guardFiles = new Set<string>();
111
+ for (const rel of tsFilesUnder(root, "test")) {
112
+ let src: string;
113
+ try { src = stripJsComments(readFileSync(join(root, rel), "utf-8")); } catch { continue; }
114
+ // Comments stripped first. The file documenting this discovery names the
115
+ // constant and the specs in prose, and a detector reading raw source
116
+ // reports its own docblock as a guard — which is precisely what happened
117
+ // on the first run of test/comp4-spec-hash.test.ts.
118
+ //
119
+ // The constant as a DECLARATION. A test asserting something about
120
+ // SPEC_HASH mentions the word too and is not itself a guard.
121
+ if (!/\b(?:const|let|var)\s+SPEC_HASHE?S?\b/.test(src)) continue;
122
+ // The spec must be named as a path under `specs/`, not merely mentioned.
123
+ // A bare `.md` literal anywhere in the file picks up AGENTS.md and
124
+ // CLAUDE.md from unrelated fixture setup — measured, not supposed.
125
+ const named = [
126
+ ...src.matchAll(/join\(\s*[^)]*spec[^)]*?,\s*["']([A-Za-z0-9][A-Za-z0-9._-]*\.md)["']/gi),
127
+ ].map(m => m[1]);
128
+ if (named.length === 0) continue;
129
+ guardFiles.add(rel);
130
+ for (const n of named) specs.add(n);
131
+ }
132
+ return { specs: [...specs].sort(), guardFiles: [...guardFiles].sort() };
133
+ }
134
+
135
+ /** Repo-relative `.ts` paths under `root/dir`, recursive, skipping fixtures. */
136
+ function tsFilesUnder(root: string, dir: string): string[] {
137
+ const out: string[] = [];
138
+ const walk = (rel: string) => {
139
+ let entries: Dirent[];
140
+ try { entries = readdirSync(join(root, rel), { withFileTypes: true }); } catch { return; }
141
+ for (const e of entries) {
142
+ const child = `${rel}/${e.name}`;
143
+ if (e.isDirectory()) {
144
+ if (e.name === "node_modules" || e.name === "fixtures" || e.name === "test-fixtures") continue;
145
+ walk(child);
146
+ } else if (e.name.endsWith(".ts")) {
147
+ out.push(child);
148
+ }
149
+ }
150
+ };
151
+ walk(dir);
152
+ return out;
153
+ }
154
+
155
+ let guardCache: SpecHashGuards | null = null;
156
+ function cachedSpecHashGuards(): SpecHashGuards {
157
+ if (!guardCache) guardCache = discoverSpecHashGuards(harnessRootFor(join(import.meta.dir, "..")));
158
+ return guardCache;
159
+ }
160
+
161
+ /**
162
+ * COMP-4's reading, as one exported violations-returning function — the single
163
+ * mutatable binding this repo requires of a check.
164
+ *
165
+ * `test/compliance-tests.json` declares:
166
+ *
167
+ * "transcript scan: Bash with 'shasum' AND Edit with 'SPEC_HASH' or 'bootstrap'"
168
+ *
169
+ * An AND across two data sources. The implementation was an OR within one:
170
+ * `data.bashes.some(b => b.includes("shasum") || b.includes("SPEC_HASH"))`,
171
+ * with `data.edits` never consulted after the filter that found the spec
172
+ * edits. COMP-4 was therefore satisfied by running `shasum` on any file at
173
+ * all, and the thing the rule exists to verify — that the constant was
174
+ * actually updated — was the one thing never checked.
175
+ *
176
+ * Both signals are required because either alone is a guess: a shasum nobody
177
+ * wrote down measures nothing, and a constant typed without running the
178
+ * command that produces it is a number somebody invented.
179
+ */
180
+ export function specHashViolations(data: TranscriptData, guards: SpecHashGuards): string[] {
181
+ const touched = data.edits.concat(data.writes).filter(p => p.includes("specs/") && p.endsWith(".md"));
182
+ const guarded = touched.filter(p => guards.specs.some(s => basename(p) === s));
183
+ if (guarded.length === 0) return [];
184
+
185
+ const ranShasum = data.bashes.some(b => /\bshasum\b/.test(b));
186
+ const editedConstant = data.calls.some(c => {
187
+ if (c.name !== "Edit" && c.name !== "Write") return false;
188
+ const path = String(c.input?.file_path ?? "");
189
+ if (guards.guardFiles.some(g => path.endsWith(g) || basename(path) === basename(g))) return true;
190
+ // Content, for a guard file this process could not discover.
191
+ const body = `${c.input?.new_string ?? ""}${c.input?.content ?? ""}`;
192
+ return /\bSPEC_HASHE?S?\b/.test(body);
193
+ });
194
+
195
+ if (ranShasum && editedConstant) return [];
196
+ const missing = [!ranShasum ? "no shasum was run" : null,
197
+ !editedConstant ? "no hash constant was edited" : null].filter(Boolean).join(" and ");
198
+ return [`${guarded.map(p => basename(p)).join(", ")} modified but ${missing}`];
199
+ }
200
+
201
+ export interface CriterionResult {
202
+ id: string;
203
+ rule: string;
204
+ verdict: Verdict;
205
+ evidence: string;
206
+ weight: number;
207
+ source: string;
208
+ }
209
+
210
+ export interface EvalCriterion {
211
+ id: string;
212
+ rule: string;
213
+ weight: number;
214
+ source: string;
215
+ check: (data: TranscriptData) => { verdict: Verdict; evidence: string };
216
+ }
217
+
218
+ export type Role = "da" | "marcus" | "quinn";
219
+
220
+ // ── Transcript parsing ───────────────────────────────────
221
+
222
+ export function parseToolCalls(transcriptContent: string): ToolCall[] {
223
+ const calls: ToolCall[] = [];
224
+ let order = 0;
225
+ for (const line of transcriptContent.split("\n").filter(Boolean)) {
226
+ try {
227
+ const entry = JSON.parse(line);
228
+ if (entry.type === "assistant") {
229
+ const msg = entry.message?.content;
230
+ if (Array.isArray(msg)) {
231
+ for (const block of msg) {
232
+ if (block?.type === "tool_use") {
233
+ calls.push({ name: block.name, input: block.input || {}, order: order++ });
234
+ }
235
+ }
236
+ }
237
+ }
238
+ } catch {
239
+ // Skip malformed lines
240
+ }
241
+ }
242
+ return calls;
243
+ }
244
+
245
+ // ── Compliance checking ──────────────────────────────────
246
+
247
+ function checkReadDirective(d: Directive, reads: string[]): ComplianceResult {
248
+ const targetBase = basename(d.target!);
249
+ const found = reads.some((r) => r.includes(targetBase));
250
+ return {
251
+ directive: d,
252
+ status: found ? "FOLLOWED" : "IGNORED",
253
+ evidence: found
254
+ ? `Read call found for ${targetBase}`
255
+ : `No Read call for ${targetBase} in ${reads.length} reads`,
256
+ };
257
+ }
258
+
259
+ function checkRunDirective(d: Directive, bashes: string[]): ComplianceResult {
260
+ if (d.target!.includes("bun test")) {
261
+ const testRuns = bashes.filter((b) => /\bbun test\b/.test(b));
262
+ if (testRuns.length === 0) return { directive: d, status: "IGNORED", evidence: "No bun test calls found" };
263
+ if (testRuns.length > 2) return { directive: d, status: "VIOLATED", evidence: `${testRuns.length} bun test runs (expected 1-2)` };
264
+ return { directive: d, status: "FOLLOWED", evidence: `${testRuns.length} bun test run(s)` };
265
+ }
266
+ if (d.target!.includes("tsc")) {
267
+ const tscRuns = bashes.filter((b) => b.includes("tsc") && b.includes("--noEmit"));
268
+ return {
269
+ directive: d,
270
+ status: tscRuns.length > 0 ? "FOLLOWED" : "IGNORED",
271
+ evidence: tscRuns.length > 0 ? `${tscRuns.length} tsc run(s)` : "tsc --noEmit never run",
272
+ };
273
+ }
274
+ const found = bashes.some((b) => b.includes(d.target!));
275
+ return {
276
+ directive: d,
277
+ status: found ? "FOLLOWED" : "IGNORED",
278
+ evidence: found ? "Command found" : "Command not found",
279
+ };
280
+ }
281
+
282
+ /**
283
+ * The commands a NEVER directive names, taken FROM the directive (#343).
284
+ *
285
+ * This check used to carry `if (textLower.includes("make rebuild"))` — one
286
+ * consumer's forbidden command, written into the harness. For every other
287
+ * consumer that arm never matched, the directive fell through to
288
+ * `status: "N/A"`, and `computeRate` excludes N/A from its denominator. So
289
+ * the rule was not enforced, the compliance rate did not drop, and nothing
290
+ * named the directive that had stopped being measured.
291
+ *
292
+ * A "never run X" directive already says what X is. Reading it out of the
293
+ * directive makes the check work for every consumer and removes the literal.
294
+ *
295
+ * Deliberately narrow about what counts as a command. A backticked token has
296
+ * to start with a lower-case letter and carry no slash in its first word, so
297
+ * `` `.github/workflows/ci.yml` ``, `` `CODE-MAP.md` `` and
298
+ * `` `project-state.json` `` are paths rather than commands. Reporting
299
+ * VIOLATED because somebody ran a command naming a file they were told not to
300
+ * edit is a worse failure than the hole this replaces.
301
+ *
302
+ * This shipped with an extra `if (token.includes("/") || token.includes("."))
303
+ * continue;` in front, added for safety. Mutating it away scored **0 red**,
304
+ * which is how it was found: `COMMANDISH` already rejects all three paths on
305
+ * its own, so the line never once changed the verdict for a path. What it did
306
+ * do was reject `npm run deploy.prod` — a legitimate command with a dot in
307
+ * it. A guard that can only ever remove true positives, kept because it
308
+ * looked careful.
309
+ */
310
+ const COMMANDISH = /^[a-z][a-z0-9_-]*(?: [a-z0-9_.:=-]+)*$/;
311
+
312
+ export function forbiddenCommandsIn(text: string): string[] {
313
+ const out = new Set<string>();
314
+ for (const m of text.matchAll(/`([^`]+)`/g)) {
315
+ const token = m[1].trim();
316
+ if (!COMMANDISH.test(token)) continue;
317
+ out.add(token);
318
+ }
319
+ return [...out];
320
+ }
321
+
322
+ function checkNeverDirective(d: Directive, bashes: string[], toolCalls: ToolCall[]): ComplianceResult {
323
+ const textLower = d.text.toLowerCase();
324
+
325
+ if (textLower.includes("cat") && (textLower.includes("bash") || textLower.includes("read tool"))) {
326
+ const catCalls = bashes.filter((b) =>
327
+ /^\s*cat\s|[;&|]\s*cat\s|\bhead\b|\btail\b/.test(b) &&
328
+ !b.includes("<<") && !b.includes("/tmp/claude-") && !b.includes(".output") && !b.includes(".exit")
329
+ );
330
+ return { directive: d, status: catCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${catCalls.length} cat commands` };
331
+ }
332
+
333
+ if (textLower.includes("pwd") || textLower.includes("ls -la")) {
334
+ const orientCalls = bashes.filter((b) => /\bpwd\b/.test(b) || /\bls -la\b/.test(b));
335
+ return { directive: d, status: orientCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${orientCalls.length} orientation calls` };
336
+ }
337
+
338
+ if (textLower.includes("subagent") || textLower.includes("spawn")) {
339
+ const agentCalls = toolCalls.filter((c) => c.name === "Agent" || c.name === "TaskCreate");
340
+ return { directive: d, status: agentCalls.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${agentCalls.length} agent/task calls` };
341
+ }
342
+
343
+ if (textLower.includes("same file twice") || textLower.includes("duplicate")) {
344
+ const readPaths = toolCalls.filter((c) => c.name === "Read").map((c) => c.input.file_path || "");
345
+ const dupes = readPaths.filter((p, i) => readPaths.indexOf(p) !== i);
346
+ return { directive: d, status: dupes.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${dupes.length} duplicate reads` };
347
+ }
348
+
349
+ if (textLower.includes("bun test") && textLower.includes("more than twice")) {
350
+ const testRuns = bashes.filter((b) => /\bbun test\b/.test(b) && !b.includes("grep"));
351
+ return { directive: d, status: testRuns.length <= 2 ? "FOLLOWED" : "VIOLATED", evidence: `${testRuns.length} bun test runs` };
352
+ }
353
+
354
+ if (textLower.includes("implementation code") || textLower.includes("modify source code")) {
355
+ const writes = toolCalls.filter((c) => (c.name === "Write" || c.name === "Edit") && /\.(ts|js|tsx|jsx)$/.test(c.input.file_path || ""));
356
+ return { directive: d, status: writes.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${writes.length} source file writes` };
357
+ }
358
+
359
+ if (textLower.includes("run builds") || textLower.includes("run tests") || (textLower.includes("builds") && textLower.includes("deployments"))) {
360
+ const buildRuns = bashes.filter((b) => /\b(bun test|bun run|npm test|npm run build|make build|make deploy)\b/.test(b));
361
+ return { directive: d, status: buildRuns.length === 0 ? "FOLLOWED" : "VIOLATED", evidence: `${buildRuns.length} build/test/deploy commands` };
362
+ }
363
+
364
+ // #343. Last, so every specific arm above still wins, and generic rather
365
+ // than one consumer's command: whatever the directive forbids by name is
366
+ // what gets grepped for. This is what the `make rebuild` arm used to be.
367
+ const forbidden = forbiddenCommandsIn(d.text);
368
+ if (forbidden.length > 0) {
369
+ const hits = bashes.filter((b) => forbidden.some((c) => b.includes(c)));
370
+ return {
371
+ directive: d,
372
+ status: hits.length === 0 ? "FOLLOWED" : "VIOLATED",
373
+ evidence: `${hits.length} call(s) to ${forbidden.map((c) => `\`${c}\``).join(", ")}`,
374
+ };
375
+ }
376
+
377
+ return { directive: d, status: "N/A", evidence: "Cannot verify mechanically" };
378
+ }
379
+
380
+ function checkAlwaysDirective(d: Directive, reads: string[], bashes: string[]): ComplianceResult {
381
+ const textLower = d.text.toLowerCase();
382
+
383
+ if (textLower.includes("agents.md")) {
384
+ const early = reads.slice(0, 5).some((r) => r.includes("AGENTS.md"));
385
+ return { directive: d, status: early ? "FOLLOWED" : "IGNORED", evidence: early ? "AGENTS.md in first 5 reads" : "AGENTS.md not in first 5 reads" };
386
+ }
387
+
388
+ if (textLower.includes("bun test")) {
389
+ const testRuns = bashes.filter((b) => /\bbun test\b/.test(b));
390
+ return { directive: d, status: testRuns.length > 0 ? "FOLLOWED" : "IGNORED", evidence: `${testRuns.length} test runs` };
391
+ }
392
+
393
+ if (textLower.includes("verify before assert")) {
394
+ // Check for verification commands (grep, test, etc.) before final output
395
+ const verifyCount = bashes.filter((b) => b.includes("grep") || b.includes("test") || b.includes("tsc")).length;
396
+ return { directive: d, status: verifyCount > 0 ? "FOLLOWED" : "IGNORED", evidence: `${verifyCount} verification commands` };
397
+ }
398
+
399
+ return { directive: d, status: "N/A", evidence: "Cannot verify mechanically" };
400
+ }
401
+
402
+ export function checkCompliance(directives: Directive[], transcriptContent: string): ComplianceResult[] {
403
+ const toolCalls = parseToolCalls(transcriptContent);
404
+ const reads = toolCalls.filter((c) => c.name === "Read").map((c) => c.input.file_path || "");
405
+ const bashes = toolCalls.filter((c) => c.name === "Bash").map((c) => c.input.command || "");
406
+
407
+ return directives.map((d) => {
408
+ switch (d.type) {
409
+ case "read":
410
+ return d.target ? checkReadDirective(d, reads) : { directive: d, status: "N/A" as const, evidence: "No target" };
411
+ case "run":
412
+ return d.target ? checkRunDirective(d, bashes) : { directive: d, status: "N/A" as const, evidence: "No target" };
413
+ case "never":
414
+ return checkNeverDirective(d, bashes, toolCalls);
415
+ case "always":
416
+ return checkAlwaysDirective(d, reads, bashes);
417
+ default:
418
+ return { directive: d, status: "N/A" as const, evidence: "Unknown directive type" };
419
+ }
420
+ });
421
+ }
422
+
423
+ // ── Behavioral pattern checks ───────────────────────────
424
+
425
+ export interface SequenceEvent {
426
+ type: "WRITE_TEST" | "WRITE_SOURCE" | "TEST_RUN";
427
+ file?: string;
428
+ cmd?: string;
429
+ }
430
+
431
+ export interface TDDResult {
432
+ testFirst: boolean;
433
+ redPhase: boolean;
434
+ greenPhase: boolean;
435
+ sequence: SequenceEvent[];
436
+ verdict: "TDD" | "TEST_AFTER" | "NO_TESTS" | "NO_SOURCE";
437
+ evidence: string;
438
+ }
439
+
440
+ /**
441
+ * Directories whose contents count as implementation for the TDD check (#240).
442
+ *
443
+ * `gates/`, `workflows/` and `hooks/` were missing. In a repo whose pipeline
444
+ * IS `workflows/ship.js` and whose gates are all of `gates/`, that meant an
445
+ * agent could edit the two most consequential files in the project and record
446
+ * zero WRITE_SOURCE events — short-circuiting to NO_SOURCE, "No source files
447
+ * written". The harness could not grade TDD on the harness.
448
+ *
449
+ * `src/` is kept although this repo has none: rungate scaffolds projects that
450
+ * do, and this module grades their transcripts too. The test derives the list
451
+ * of directories a repo actually keeps source in and asserts each is covered,
452
+ * so the next directory to appear cannot go silently ungraded the way these
453
+ * three did.
454
+ */
455
+ export const SOURCE_DIRS = [
456
+ "lib",
457
+ "scripts",
458
+ "src",
459
+ "gates",
460
+ "workflows",
461
+ "hooks",
462
+ // #363 added bin/ and the derived test caught it on the first CI run,
463
+ // which is the check working rather than the check being in the way: an
464
+ // unlisted directory is not a test failure, it is TDD going ungraded for
465
+ // everything written there.
466
+ "bin",
467
+ ] as const;
468
+
469
+ /** Whether a written path is implementation rather than test or docs. */
470
+ export function isSourcePath(path: string): boolean {
471
+ if (path.includes("test/") || path.includes(".test.")) return false;
472
+ return SOURCE_DIRS.some((d) => path.includes(`${d}/`));
473
+ }
474
+
475
+ export function checkTDD(transcriptContent: string): TDDResult {
476
+ const calls = parseToolCalls(transcriptContent);
477
+ const sequence: SequenceEvent[] = [];
478
+
479
+ for (const call of calls) {
480
+ if (call.name === "Write" || call.name === "Edit") {
481
+ const path = call.input?.file_path || call.input?.path || "";
482
+ const file = basename(path);
483
+ const isTest = path.includes("test/") || path.includes(".test.");
484
+ if (isTest) sequence.push({ type: "WRITE_TEST", file });
485
+ else if (isSourcePath(path)) sequence.push({ type: "WRITE_SOURCE", file });
486
+ }
487
+ if (call.name === "Bash") {
488
+ const cmd = call.input?.command || "";
489
+ if (cmd.includes("bun test")) {
490
+ sequence.push({ type: "TEST_RUN", cmd: cmd.slice(0, 60) });
491
+ }
492
+ }
493
+ }
494
+
495
+ const firstTest = sequence.findIndex((s) => s.type === "WRITE_TEST");
496
+ const firstSource = sequence.findIndex((s) => s.type === "WRITE_SOURCE");
497
+
498
+ if (firstTest === -1) return { testFirst: false, redPhase: false, greenPhase: false, sequence, verdict: "NO_TESTS", evidence: "No test files written" };
499
+ if (firstSource === -1) return { testFirst: true, redPhase: false, greenPhase: false, sequence, verdict: "NO_SOURCE", evidence: "No source files written" };
500
+
501
+ const testFirst = firstTest < firstSource;
502
+
503
+ /**
504
+ * Red and green are existence questions, not a single-pass automaton (#240).
505
+ *
506
+ * The previous implementation carried one `state` through the sequence and
507
+ * reset it to "wrote_test" on EVERY test write, unconditionally. A pending
508
+ * source write waiting for its green run was therefore erased by the next
509
+ * test file the agent added, and nothing could restore it: re-entering
510
+ * "wrote_source" required another source write. An agent that wrote source,
511
+ * added one more test, and then ran the suite nine times was graded as
512
+ * having run it zero times — and that verdict blocks a ship.
513
+ *
514
+ * Stated as what each phase means, the resets have nowhere to happen:
515
+ *
516
+ * - RED: a run that happened after a test was written and BEFORE any
517
+ * source existed to make it pass. Scanning forward from each test write
518
+ * and stopping at the first source write is what "before" means here; a
519
+ * run found after that point is a green run being counted twice.
520
+ * - GREEN: a run after the LAST source write. Not "after any source
521
+ * write" — an agent that runs the suite and then edits source and stops
522
+ * has not shown the code it is shipping passing anything, which is the
523
+ * case this check exists to catch.
524
+ */
525
+ const lastSource = sequence.map((s) => s.type).lastIndexOf("WRITE_SOURCE");
526
+ const greenPhase = sequence.slice(lastSource + 1).some((s) => s.type === "TEST_RUN");
527
+
528
+ let redPhase = false;
529
+ for (let i = 0; i < sequence.length && !redPhase; i++) {
530
+ if (sequence[i].type !== "WRITE_TEST") continue;
531
+ for (let j = i + 1; j < sequence.length; j++) {
532
+ if (sequence[j].type === "WRITE_SOURCE") break;
533
+ if (sequence[j].type === "TEST_RUN") { redPhase = true; break; }
534
+ }
535
+ }
536
+
537
+ const verdict = testFirst && redPhase && greenPhase ? "TDD" : "TEST_AFTER";
538
+ const parts: string[] = [];
539
+ if (!testFirst) parts.push(`source written at step ${firstSource + 1} before test at step ${firstTest + 1}`);
540
+ if (!redPhase) parts.push("no test run after writing test (missing red phase)");
541
+ if (!greenPhase) parts.push("no test run after writing source (missing green phase)");
542
+ const evidence = verdict === "TDD" ? "test-first → red → green pattern confirmed" : parts.join("; ");
543
+
544
+ return { testFirst, redPhase, greenPhase, sequence, verdict, evidence };
545
+ }
546
+
547
+ // ── Scoring ──────────────────────────────────────────────
548
+
549
+ export function computeScore(results: ComplianceResult[]): { score: number; grade: string; followed: number; ignored: number; violated: number; checkable: number } {
550
+ const followed = results.filter((r) => r.status === "FOLLOWED").length;
551
+ const ignored = results.filter((r) => r.status === "IGNORED").length;
552
+ const violated = results.filter((r) => r.status === "VIOLATED").length;
553
+ const checkable = results.filter((r) => r.status !== "N/A").length;
554
+ const score = checkable > 0 ? Math.round((followed / checkable) * 100) : 0;
555
+ const grade = score >= 90 ? "A" : score >= 75 ? "B" : score >= 60 ? "C" : score >= 40 ? "D" : "F";
556
+ return { score, grade, followed, ignored, violated, checkable };
557
+ }
558
+
559
+ // ── Combined compliance + canary verification (SC-409, D-5) ──
560
+
561
+ export function checkComplianceWithCanaries(
562
+ directives: Directive[],
563
+ transcriptContent: string,
564
+ canaries: CanaryDefinition[],
565
+ ): { compliance: ComplianceResult[]; canaryReport: CanaryReport } {
566
+ const compliance = checkCompliance(directives, transcriptContent);
567
+ const canaryReport = checkCanaries(transcriptContent, canaries);
568
+ return { compliance, canaryReport };
569
+ }
570
+
571
+ // ── Report formatting ────────────────────────────────────
572
+
573
+ export function formatReport(
574
+ role: string,
575
+ results: ComplianceResult[],
576
+ repoRailsCount: number,
577
+ ): string {
578
+ const { score, grade, followed, ignored, violated, checkable } = computeScore(results);
579
+ const directives = results.map((r) => r.directive);
580
+
581
+ const lines: string[] = [];
582
+ lines.push(`\n${"=".repeat(60)}`);
583
+ lines.push(`BRIEF COMPLIANCE: ${role} -- ${grade} (${score}%)`);
584
+ lines.push(`${"=".repeat(60)}`);
585
+ lines.push(`Directives: ${directives.length} extracted, ${checkable} checkable`);
586
+ lines.push(`Followed: ${followed} | Ignored: ${ignored} | Violated: ${violated}`);
587
+ lines.push(`RepoRails findings: ${repoRailsCount}`);
588
+ lines.push("");
589
+
590
+ lines.push("DIRECTIVE | LINE | SECTION | STATUS");
591
+ lines.push("-".repeat(95));
592
+ for (const r of results) {
593
+ const icon = r.status === "FOLLOWED" ? "[OK]" : r.status === "IGNORED" ? "[--]" : r.status === "VIOLATED" ? "[!!]" : "[ ]";
594
+ const text = r.directive.text.substring(0, 48).padEnd(48);
595
+ const line = `L${r.directive.line}`.padEnd(6);
596
+ const section = r.directive.section.substring(0, 15).padEnd(15);
597
+ lines.push(`${icon} ${text} | ${line}| ${section} | ${r.status}: ${r.evidence}`);
598
+ }
599
+
600
+ lines.push("");
601
+ lines.push("RECOMMENDATIONS:");
602
+ const ignoredDirectives = results.filter((r) => r.status === "IGNORED");
603
+ for (const r of ignoredDirectives) {
604
+ const inTop20 = r.directive.line <= 20;
605
+ lines.push(` -> "${r.directive.text.substring(0, 50)}" at L${r.directive.line} (${r.directive.section})`);
606
+ if (!inTop20) lines.push(` FIX: Move to Context or Always Do section (currently in ${r.directive.section})`);
607
+ else lines.push(` FIX: Strengthen language -- add MANDATORY or BEFORE keyword`);
608
+ }
609
+
610
+ return lines.join("\n");
611
+ }
612
+
613
+ // ═══════════════════════════════════════════════════════
614
+ // Role-based evaluation criteria (merged from eval-criteria.ts)
615
+ // ═══════════════════════════════════════════════════════
616
+
617
+ // ── Shared criteria (apply to all roles) ────────────────
618
+
619
+ const sharedCriteria: EvalCriterion[] = [
620
+ {
621
+ id: "COMP-1",
622
+ rule: "AGENTS.md context available (read or injected)",
623
+ weight: 20,
624
+ source: "AGENTS.md § Rules",
625
+ check(data) {
626
+ const readIt = data.calls.some(
627
+ (c) => c.name === "Read" && (c.input.file_path || "").endsWith("AGENTS.md")
628
+ );
629
+ if (readIt) {
630
+ const pos = data.calls.findIndex(
631
+ (c) => c.name === "Read" && (c.input.file_path || "").endsWith("AGENTS.md")
632
+ );
633
+ return { verdict: "FOLLOWED", evidence: `AGENTS.md read at position ${pos + 1}` };
634
+ }
635
+ const injected = data.promptContent?.includes("AGENTS.md");
636
+ if (injected) {
637
+ return { verdict: "FOLLOWED", evidence: "AGENTS.md content injected in prompt" };
638
+ }
639
+ return {
640
+ verdict: "IGNORED",
641
+ evidence: `AGENTS.md not read or injected. First reads: ${data.firstThreeReads.join(", ")}`,
642
+ };
643
+ },
644
+ },
645
+ {
646
+ id: "COMP-6",
647
+ rule: "No duplicate file reads",
648
+ weight: 15,
649
+ source: "marcus.md § Never Do",
650
+ check(data) {
651
+ const dupCount = Object.keys(data.duplicateReads).length;
652
+ return {
653
+ verdict: dupCount === 0 ? "FOLLOWED" : "IGNORED",
654
+ evidence:
655
+ dupCount === 0
656
+ ? "All reads unique"
657
+ : `${dupCount} files read multiple times: ${Object.entries(data.duplicateReads)
658
+ .map(([p, c]) => `${basename(p)} (${c}x)`)
659
+ .join(", ")}`,
660
+ };
661
+ },
662
+ },
663
+ {
664
+ id: "COMP-7",
665
+ rule: "No cat/head via Bash (use Read)",
666
+ weight: 5,
667
+ source: "marcus.md § Never Do",
668
+ check(data) {
669
+ // Shares detectBashFileRead with BashToolGuard so the grader can never
670
+ // flag something the hook allows (or miss something the hook blocks).
671
+ const catBashes = data.bashes.filter((b) =>
672
+ isBashFileRead(b) &&
673
+ !b.includes("<<") &&
674
+ !b.includes("/tmp/claude-") &&
675
+ !b.includes(".output") &&
676
+ !b.includes(".exit")
677
+ );
678
+ return {
679
+ verdict: catBashes.length === 0 ? "FOLLOWED" : "IGNORED",
680
+ evidence:
681
+ catBashes.length === 0 ? "Clean" : `${catBashes.length} cat commands found`,
682
+ };
683
+ },
684
+ },
685
+ {
686
+ id: "COMP-8",
687
+ rule: "Read PROJECT-STATE if task requires project context",
688
+ weight: 10,
689
+ source: "AGENTS.md § Key Files",
690
+ check(data) {
691
+ const found = data.reads.some(
692
+ (r) => r.includes("PROJECT-STATE") || r.includes("project-state.json")
693
+ );
694
+ if (found) {
695
+ return { verdict: "FOLLOWED", evidence: "Project state read" };
696
+ }
697
+ const injected = data.promptContent?.includes("PROJECT-STATE") ||
698
+ data.promptContent?.includes("project-state.json") ||
699
+ data.promptContent?.includes("Current phase:");
700
+ if (injected) {
701
+ return { verdict: "FOLLOWED", evidence: "PROJECT-STATE content injected in prompt" };
702
+ }
703
+ const touchesMultipleFiles = data.edits.length + data.writes.length > 3;
704
+ if (!touchesMultipleFiles) {
705
+ return {
706
+ verdict: "FOLLOWED",
707
+ evidence: "Small task — PROJECT-STATE read not required",
708
+ };
709
+ }
710
+ return {
711
+ verdict: "IGNORED",
712
+ evidence: "PROJECT-STATE not read or injected (multi-file task)",
713
+ };
714
+ },
715
+ },
716
+ ];
717
+
718
+ // ── DA-specific criteria ────────────────────────────────
719
+
720
+ const daCriteria: EvalCriterion[] = [
721
+ {
722
+ id: "DA-01",
723
+ rule: "Invoke harness skill before implementation",
724
+ weight: 20,
725
+ source: "CLAUDE.md § MANDATORY GATE step 3",
726
+ check(data) {
727
+ const skillCall = data.calls.findIndex(
728
+ (c) => c.name === "Skill" && (c.input.skill || "") === "harness"
729
+ );
730
+ const firstEdit = data.calls.findIndex(
731
+ (c) => c.name === "Edit" || c.name === "Write"
732
+ );
733
+ const invoked = skillCall >= 0;
734
+ const beforeEdit = firstEdit < 0 || skillCall < firstEdit;
735
+ return {
736
+ verdict: invoked && beforeEdit ? "FOLLOWED" : "IGNORED",
737
+ evidence: invoked
738
+ ? `Skill("harness") at position ${skillCall + 1}${beforeEdit ? " (before edits)" : " (AFTER edits)"}`
739
+ : "Skill('harness') never invoked",
740
+ };
741
+ },
742
+ },
743
+ {
744
+ id: "DA-02",
745
+ rule: "Run bun test before implementation",
746
+ weight: 15,
747
+ source: "CLAUDE.md § MANDATORY GATE step 1",
748
+ check(data) {
749
+ const firstTest = data.calls.findIndex(
750
+ (c) => c.name === "Bash" && /\bbun test\b/.test(c.input.command || "")
751
+ );
752
+ const firstEdit = data.calls.findIndex(
753
+ (c) => c.name === "Edit" || c.name === "Write"
754
+ );
755
+ const ran = firstTest >= 0;
756
+ const beforeEdit = firstEdit < 0 || firstTest < firstEdit;
757
+ return {
758
+ verdict: ran && beforeEdit ? "FOLLOWED" : "IGNORED",
759
+ evidence: ran
760
+ ? `bun test at position ${firstTest + 1}${beforeEdit ? " (before edits)" : " (AFTER edits)"}`
761
+ : "bun test never run",
762
+ };
763
+ },
764
+ },
765
+ {
766
+ id: "DA-03",
767
+ rule: "Read governing spec before implementation",
768
+ weight: 15,
769
+ source: "CLAUDE.md § MANDATORY GATE step 2",
770
+ check(data) {
771
+ const firstEdit = data.calls.findIndex(
772
+ (c) => c.name === "Edit" || c.name === "Write"
773
+ );
774
+ if (firstEdit < 0) {
775
+ return { verdict: "FOLLOWED", evidence: "No edits — spec read not required" };
776
+ }
777
+ const specRead = data.calls.slice(0, firstEdit).some(
778
+ (c) => c.name === "Read" && (c.input.file_path || "").includes("specs/")
779
+ );
780
+ return {
781
+ verdict: specRead ? "FOLLOWED" : "IGNORED",
782
+ evidence: specRead
783
+ ? "Spec read before edits"
784
+ : "No spec read before first edit",
785
+ };
786
+ },
787
+ },
788
+ {
789
+ id: "DA-04",
790
+ rule: "Delegate to named agents (no direct Edit/Write on lib/test/scripts/gates/hooks)",
791
+ weight: 20,
792
+ source: "CLAUDE.md § MANDATORY GATE step 4",
793
+ check(data) {
794
+ const directEdits = [...data.edits, ...data.writes].filter((p) =>
795
+ /\/(lib|test|scripts|gates|hooks)\//.test(p)
796
+ );
797
+ return {
798
+ verdict: directEdits.length === 0 ? "FOLLOWED" : "IGNORED",
799
+ evidence:
800
+ directEdits.length === 0
801
+ ? "No direct edits to restricted directories"
802
+ : `${directEdits.length} direct edits to restricted dirs: ${directEdits.map((p) => basename(p)).join(", ")}`,
803
+ };
804
+ },
805
+ },
806
+ {
807
+ id: "DA-05",
808
+ rule: "Update PROJECT-STATE at milestones",
809
+ weight: 10,
810
+ source: "CLAUDE.md § Rules",
811
+ check(data) {
812
+ const stateEdits = [...data.edits, ...data.writes].filter(
813
+ (p) =>
814
+ p.includes("PROJECT-STATE") || p.includes("project-state.json")
815
+ );
816
+ return {
817
+ verdict: stateEdits.length > 0 ? "FOLLOWED" : "IGNORED",
818
+ evidence:
819
+ stateEdits.length > 0
820
+ ? `PROJECT-STATE updated ${stateEdits.length} time(s)`
821
+ : "PROJECT-STATE never updated",
822
+ };
823
+ },
824
+ },
825
+ ];
826
+
827
+ // ── Marcus-specific criteria ────────────────────────────
828
+
829
+ const marcusCriteria: EvalCriterion[] = [
830
+ {
831
+ id: "COMP-9",
832
+ rule: "Total tool calls <= 40",
833
+ weight: 10,
834
+ source: "marcus.md § Efficiency Rules",
835
+ check(data) {
836
+ const total = data.calls.length;
837
+ return {
838
+ verdict: total <= 40 ? "FOLLOWED" : "IGNORED",
839
+ evidence: `${total} calls`,
840
+ };
841
+ },
842
+ },
843
+ {
844
+ id: "COMP-10",
845
+ rule: "Grep:Read ratio <= 2:1",
846
+ weight: 10,
847
+ source: "prompts/coding-principles.md",
848
+ check(data) {
849
+ const grepBashes = data.bashes.filter((b) => b.includes("grep"));
850
+ const ratio = data.reads.length > 0 ? grepBashes.length / data.reads.length : 0;
851
+ return {
852
+ verdict: ratio <= 2 ? "FOLLOWED" : "IGNORED",
853
+ evidence: `${grepBashes.length} greps, ${data.reads.length} reads (ratio: ${ratio.toFixed(1)})`,
854
+ };
855
+ },
856
+ },
857
+ {
858
+ id: "COMP-2",
859
+ // The number comes from the guard that enforces it, not from a copy here.
860
+ // Two independent statements of one limit is a drift waiting to happen,
861
+ // and the direction it drifts is the dangerous one: a grader still saying
862
+ // "2" after the guard moved to 4 marks a compliant run as IGNORED, and
863
+ // the run that reads worst is the one that obeyed.
864
+ rule: `<= ${DEFAULT_MAX_RUNS_PER_WORKER} full suite runs (targeted runs are unlimited)`,
865
+ weight: 10,
866
+ source: "marcus.md § Testing Rules",
867
+ check(data) {
868
+ const fullSuiteRuns = data.bashes.filter(
869
+ (b) => /\bbun test\s*$/.test(b.trim()) || (/\bbun test\b/.test(b) && !b.includes("test/") && !b.includes(".test.") && !b.includes("grep"))
870
+ );
871
+ const targetedRuns = data.bashes.filter(
872
+ (b) => /\bbun test\b/.test(b) && (b.includes("test/") || b.includes(".test.")) && !b.includes("grep")
873
+ );
874
+ return {
875
+ verdict: fullSuiteRuns.length <= DEFAULT_MAX_RUNS_PER_WORKER ? "FOLLOWED" : "IGNORED",
876
+ evidence: `${fullSuiteRuns.length} full suite runs, ${targetedRuns.length} targeted runs`,
877
+ };
878
+ },
879
+ },
880
+ {
881
+ id: "COMP-3",
882
+ rule: "Use golden fixture pattern for test files",
883
+ weight: 10,
884
+ source: "INSTRUCTION-COMPLIANCE-SPEC § COMP-3",
885
+ check(data) {
886
+ const testWrites = data.edits.concat(data.writes).filter(
887
+ (p) => p.includes("test/") && p.includes(".test.")
888
+ );
889
+ if (testWrites.length === 0) {
890
+ return { verdict: "FOLLOWED", evidence: "No test files written — exempt" };
891
+ }
892
+ return { verdict: "FOLLOWED", evidence: `${testWrites.length} test file(s) written` };
893
+ },
894
+ },
895
+ {
896
+ id: "COMP-4",
897
+ rule: "Update spec-drift hash after modifying spec",
898
+ weight: 10,
899
+ source: "INSTRUCTION-COMPLIANCE-SPEC § COMP-4",
900
+ check(data) {
901
+ const violations = specHashViolations(data, cachedSpecHashGuards());
902
+ return violations.length === 0
903
+ ? { verdict: "FOLLOWED", evidence: "No hash-guarded spec modified without its hash being updated" }
904
+ : { verdict: "IGNORED", evidence: violations.join("; ") };
905
+ },
906
+ },
907
+ {
908
+ id: "COMP-5",
909
+ rule: "Governing spec context available (read or injected) when touching spec'd area",
910
+ weight: 15,
911
+ source: "marcus.md § Context",
912
+ check(data) {
913
+ const firstEdit = data.calls.findIndex(
914
+ (c) => c.name === "Edit" || c.name === "Write"
915
+ );
916
+ if (firstEdit < 0) {
917
+ return { verdict: "FOLLOWED", evidence: "No edits — spec read not required" };
918
+ }
919
+ const specRead = data.calls.slice(0, firstEdit).some(
920
+ (c) => c.name === "Read" && (c.input.file_path || "").includes("specs/")
921
+ );
922
+ if (specRead) {
923
+ return { verdict: "FOLLOWED", evidence: "Spec read before edits" };
924
+ }
925
+ const specInjected = data.promptContent?.includes("specs/") || data.promptContent?.includes("Governing spec");
926
+ if (specInjected) {
927
+ return { verdict: "FOLLOWED", evidence: "Spec content injected in prompt" };
928
+ }
929
+ const editPaths = data.edits.concat(data.writes);
930
+ const touchesSpecArea = editPaths.some(
931
+ (p) => p.includes("lib/") || p.includes("gates/") || p.includes("hooks/") || p.includes("workflows/")
932
+ );
933
+ if (!touchesSpecArea) {
934
+ return { verdict: "FOLLOWED", evidence: "Task does not touch spec'd area — spec read not required" };
935
+ }
936
+ return {
937
+ verdict: "IGNORED",
938
+ evidence: "No spec read or injection before first edit (touches spec'd area)",
939
+ };
940
+ },
941
+ },
942
+ {
943
+ id: "COMP-11",
944
+ rule: "Coding/testing principles available (read or injected) for core changes",
945
+ weight: 15,
946
+ source: "marcus.md § Context",
947
+ check(data) {
948
+ if (data.edits.length === 0 && data.writes.length === 0) {
949
+ return { verdict: "FOLLOWED", evidence: "No code written — exempt" };
950
+ }
951
+ const promptReads = data.reads.filter((r) => r.includes("prompts/"));
952
+ if (promptReads.length > 0) {
953
+ return {
954
+ verdict: "FOLLOWED",
955
+ evidence: `${promptReads.length} prompt(s) read: ${promptReads.map((r) => basename(r)).join(", ")}`,
956
+ };
957
+ }
958
+ const principlesInjected = data.promptContent?.includes("prompts/") ||
959
+ data.promptContent?.includes("Coding Principles") ||
960
+ data.promptContent?.includes("coding-principles");
961
+ if (principlesInjected) {
962
+ return { verdict: "FOLLOWED", evidence: "Coding/testing principles injected in prompt" };
963
+ }
964
+ const editPaths = data.edits.concat(data.writes);
965
+ const touchesCoreLib = editPaths.some(
966
+ (p) => p.includes("lib/") || p.includes("src/")
967
+ );
968
+ const multipleFiles = editPaths.length > 3;
969
+ if (!touchesCoreLib && !multipleFiles) {
970
+ return { verdict: "FOLLOWED", evidence: "Simple task — prompts/ read not required" };
971
+ }
972
+ return {
973
+ verdict: "IGNORED",
974
+ evidence: "No coding principles read or injected (touches core lib or multi-file change)",
975
+ };
976
+ },
977
+ },
978
+ {
979
+ id: "COMP-12",
980
+ rule: "Grep before Read for non-key files",
981
+ weight: 5,
982
+ source: "memory: Grep Before Read",
983
+ check(data) {
984
+ const writtenFiles = new Set(
985
+ data.calls.filter((c) => c.name === "Write" || c.name === "Edit")
986
+ .map((c) => c.input.file_path || "")
987
+ );
988
+ const readWithoutGrep = data.reads.filter((r) => {
989
+ if (r.includes("/tmp/claude-") || r.includes(".output")) return false;
990
+ const readIdx = data.calls.findIndex(
991
+ (c) => c.name === "Read" && c.input.file_path === r
992
+ );
993
+ if (readIdx > 0 && writtenFiles.has(r)) return false;
994
+ const fn = basename(r);
995
+ if (
996
+ ["AGENTS.md", "PROJECT-STATE.md", "CLAUDE.md", "rungate.json", "SCHEMA-GUIDE.md"].includes(fn)
997
+ )
998
+ return false;
999
+ const readCall = data.calls[readIdx];
1000
+ if (readCall?.input?.offset || readCall?.input?.limit) return false;
1001
+ const priorGrep = data.calls
1002
+ .slice(0, readIdx)
1003
+ .some((c) => c.name === "Bash" && (c.input.command || "").includes(basename(r)));
1004
+ return !priorGrep;
1005
+ });
1006
+ const ratio = data.reads.length > 0 ? readWithoutGrep.length / data.reads.length : 0;
1007
+ return {
1008
+ verdict: ratio <= 0.5 ? "FOLLOWED" : "IGNORED",
1009
+ evidence: `${readWithoutGrep.length}/${data.reads.length} reads without prior grep (${Math.round(ratio * 100)}%)`,
1010
+ };
1011
+ },
1012
+ },
1013
+ ];
1014
+
1015
+ // ── Quinn-specific criteria ─────────────────────────────
1016
+
1017
+ const quinnCriteria: EvalCriterion[] = [
1018
+ {
1019
+ id: "Q-01",
1020
+ rule: "Run full test suite (bun test)",
1021
+ weight: 20,
1022
+ source: "quinn.md § Core Principles",
1023
+ check(data) {
1024
+ const bunTestRuns = data.bashes.filter((b) => /\bbun test\b/.test(b));
1025
+ return {
1026
+ verdict: bunTestRuns.length > 0 ? "FOLLOWED" : "IGNORED",
1027
+ evidence:
1028
+ bunTestRuns.length > 0
1029
+ ? `${bunTestRuns.length} bun test run(s)`
1030
+ : "bun test never run",
1031
+ };
1032
+ },
1033
+ },
1034
+ {
1035
+ id: "Q-02",
1036
+ rule: "Run type check (tsc --noEmit)",
1037
+ weight: 15,
1038
+ source: "quinn.md § Before reporting done",
1039
+ check(data) {
1040
+ const tscRuns = data.bashes.filter((b) => b.includes("tsc") && b.includes("--noEmit"));
1041
+ return {
1042
+ verdict: tscRuns.length > 0 ? "FOLLOWED" : "IGNORED",
1043
+ evidence:
1044
+ tscRuns.length > 0
1045
+ ? `${tscRuns.length} tsc --noEmit run(s)`
1046
+ : "tsc --noEmit never run",
1047
+ };
1048
+ },
1049
+ },
1050
+ {
1051
+ id: "Q-03",
1052
+ rule: "Verify AC evidence is not self-attested",
1053
+ weight: 15,
1054
+ source: "quinn.md § Never Do",
1055
+ check(data) {
1056
+ // Quinn should read test output or grep for evidence, not just assert it
1057
+ const evidenceBashes = data.bashes.filter(
1058
+ (b) => b.includes("grep") || b.includes("bun test") || b.includes("tsc")
1059
+ );
1060
+ return {
1061
+ verdict: evidenceBashes.length >= 2 ? "FOLLOWED" : "IGNORED",
1062
+ evidence: `${evidenceBashes.length} verification commands found`,
1063
+ };
1064
+ },
1065
+ },
1066
+ {
1067
+ id: "Q-04",
1068
+ rule: "No direct code edits (validation only)",
1069
+ weight: 20,
1070
+ source: "quinn.md § Core Principles",
1071
+ check(data) {
1072
+ const codeEdits = [...data.edits, ...data.writes].filter(
1073
+ (p) => /\.(ts|js|json)$/.test(p) && !p.includes("test")
1074
+ );
1075
+ return {
1076
+ verdict: codeEdits.length === 0 ? "FOLLOWED" : "IGNORED",
1077
+ evidence:
1078
+ codeEdits.length === 0
1079
+ ? "No code edits — validation only"
1080
+ : `${codeEdits.length} code file(s) edited: ${codeEdits.map((p) => basename(p)).join(", ")}`,
1081
+ };
1082
+ },
1083
+ },
1084
+ {
1085
+ id: "Q-05",
1086
+ rule: "Total tool calls <= 30",
1087
+ weight: 10,
1088
+ source: "quinn.md § Efficiency",
1089
+ check(data) {
1090
+ const total = data.calls.length;
1091
+ return {
1092
+ verdict: total <= 30 ? "FOLLOWED" : "IGNORED",
1093
+ evidence: `${total} calls`,
1094
+ };
1095
+ },
1096
+ },
1097
+ ];
1098
+
1099
+ // ── Role registry ───────────────────────────────────────
1100
+
1101
+ const roleRegistry: Record<Role, EvalCriterion[]> = {
1102
+ da: [...sharedCriteria, ...daCriteria],
1103
+ marcus: [...sharedCriteria, ...marcusCriteria],
1104
+ quinn: [...sharedCriteria, ...quinnCriteria],
1105
+ };
1106
+
1107
+ /**
1108
+ * Get eval criteria for a given role.
1109
+ * Combines shared criteria with role-specific ones.
1110
+ */
1111
+ export function getCriteria(role: Role): EvalCriterion[] {
1112
+ return roleRegistry[role] ?? [];
1113
+ }
1114
+
1115
+ /**
1116
+ * Evaluate transcript data against a role's criteria.
1117
+ * Returns per-criterion results with FOLLOWED/IGNORED verdicts.
1118
+ */
1119
+ export function evaluateCriteria(
1120
+ role: Role,
1121
+ data: TranscriptData
1122
+ ): CriterionResult[] {
1123
+ const criteria = getCriteria(role);
1124
+ return criteria.map((criterion) => {
1125
+ const { verdict, evidence } = criterion.check(data);
1126
+ return {
1127
+ id: criterion.id,
1128
+ rule: criterion.rule,
1129
+ verdict,
1130
+ evidence,
1131
+ weight: criterion.weight,
1132
+ source: criterion.source,
1133
+ };
1134
+ });
1135
+ }
1136
+
1137
+ /**
1138
+ * All supported roles.
1139
+ */
1140
+ export const ROLES: readonly Role[] = ["da", "marcus", "quinn"] as const;