rungate 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (282) hide show
  1. package/bin/rungate.ts +70 -0
  2. package/lib/aes-calculator.ts +66 -0
  3. package/lib/agent-audit.ts +139 -0
  4. package/lib/agent-brief-validation.ts +359 -0
  5. package/lib/agents-md-refs.ts +90 -0
  6. package/lib/bash-file-read.ts +197 -0
  7. package/lib/behavioral-cache.ts +78 -0
  8. package/lib/branch-cleanup.ts +234 -0
  9. package/lib/brief-context-parser.ts +35 -0
  10. package/lib/canary.ts +117 -0
  11. package/lib/check-registry.ts +226 -0
  12. package/lib/ci-checkout.ts +400 -0
  13. package/lib/cli-commands.ts +137 -0
  14. package/lib/compliance-backfill.ts +220 -0
  15. package/lib/compliance-report.ts +504 -0
  16. package/lib/compliance.ts +361 -0
  17. package/lib/config-loader.ts +255 -0
  18. package/lib/conformity.ts +1818 -131
  19. package/lib/container-provenance.ts +230 -0
  20. package/lib/create-brief.ts +150 -0
  21. package/lib/create-sc.ts +272 -0
  22. package/lib/deep-merge.ts +69 -0
  23. package/lib/dev-liveness.ts +214 -0
  24. package/lib/directive-extractor.ts +210 -0
  25. package/lib/distribution-standing.ts +113 -0
  26. package/lib/doc-hygiene.ts +238 -0
  27. package/lib/eval-criteria.ts +6 -0
  28. package/lib/evidence-portability.ts +123 -0
  29. package/lib/evidence-prevalidator.ts +390 -0
  30. package/lib/evidence-required.ts +150 -0
  31. package/lib/evidence-satisfiability.ts +288 -0
  32. package/lib/frontmatter.ts +163 -0
  33. package/lib/gap-scanner.ts +348 -0
  34. package/lib/gate-enforcement.ts +366 -0
  35. package/lib/generators/agent-briefs.ts +136 -0
  36. package/lib/generators/agents-md.ts +506 -0
  37. package/lib/generators/code-map.ts +60 -0
  38. package/lib/generators/types.ts +184 -0
  39. package/lib/git-ref-normalizer.ts +107 -0
  40. package/lib/github.ts +539 -0
  41. package/lib/harness-identity.ts +63 -0
  42. package/lib/hill-climb.ts +164 -0
  43. package/lib/instruction-effectiveness.ts +131 -0
  44. package/lib/issue-closure-audit.ts +225 -0
  45. package/lib/issue-create-override.ts +343 -0
  46. package/lib/js-top-level-code.ts +250 -0
  47. package/lib/liveness-contradiction.ts +149 -0
  48. package/lib/organize.ts +505 -0
  49. package/lib/paths.ts +197 -6
  50. package/lib/post-fix-verify.ts +157 -0
  51. package/lib/prior-branch.ts +489 -0
  52. package/lib/project-state-shape.ts +166 -0
  53. package/lib/promote-outputs.ts +156 -0
  54. package/lib/prove-label-gate.ts +35 -0
  55. package/lib/reachability.ts +302 -0
  56. package/lib/release-preflight.ts +186 -0
  57. package/lib/rule-health-pipeline.ts +264 -0
  58. package/lib/rule-health-trend.ts +161 -0
  59. package/lib/rule-provenance.ts +149 -0
  60. package/lib/rule-registry.ts +128 -0
  61. package/lib/rungate-schema.ts +114 -4
  62. package/lib/sc-claim-forms.ts +179 -0
  63. package/lib/sc-cover.ts +150 -0
  64. package/lib/sc-guard.ts +157 -0
  65. package/lib/sc-ids.ts +344 -0
  66. package/lib/scaffold/defaults.ts +165 -0
  67. package/lib/scaffold/managed-workflow.ts +370 -0
  68. package/lib/scaffold/owned-files.ts +265 -0
  69. package/lib/scaffold/steps.ts +2680 -0
  70. package/lib/scanner.ts +701 -0
  71. package/lib/screenshot-upload.ts +73 -0
  72. package/lib/secret-patterns.ts +112 -0
  73. package/lib/security-verdict.ts +616 -0
  74. package/lib/signal-phrases.ts +1 -0
  75. package/lib/spec-change-conformity.ts +80 -0
  76. package/lib/spec-files.ts +62 -0
  77. package/lib/spec-registry.ts +133 -0
  78. package/lib/spec-updater.ts +96 -0
  79. package/lib/stale-cleanup.ts +224 -0
  80. package/lib/stale-issue-scanner.ts +36 -0
  81. package/lib/strip-js-comments.ts +269 -0
  82. package/lib/suite-measurement.ts +228 -0
  83. package/lib/task-completion-checks.ts +177 -0
  84. package/lib/test-suite-lock.ts +1136 -0
  85. package/lib/transcript-checker.ts +1140 -0
  86. package/lib/validators/spec-validators.ts +171 -0
  87. package/lib/verdict-capture.ts +73 -0
  88. package/lib/verified-push.ts +358 -0
  89. package/lib/workflow-security.ts +174 -0
  90. package/lib/worktree-cleanup.ts +250 -0
  91. package/lib/worktree-collect.ts +283 -0
  92. package/lib/worktree-isolation.ts +82 -0
  93. package/package.json +36 -3
  94. package/prompts/ac-adversary.md +14 -2
  95. package/prompts/aditi.md +5 -0
  96. package/prompts/blast-radius.md +5 -0
  97. package/prompts/container-verify.md +5 -0
  98. package/prompts/discovery.md +4 -0
  99. package/prompts/environment.md +5 -0
  100. package/prompts/evidence-hierarchy.md +5 -0
  101. package/prompts/evidence-validator.md +11 -0
  102. package/prompts/marcus.md +5 -0
  103. package/prompts/prevention.md +5 -0
  104. package/prompts/prove-reproducer.md +12 -0
  105. package/prompts/quinn-decision-tree.md +5 -0
  106. package/prompts/quinn-ui-brief.md +5 -0
  107. package/prompts/quinn.md +5 -0
  108. package/prompts/rca.md +5 -0
  109. package/prompts/read-before-write.md +4 -0
  110. package/prompts/regression.md +5 -0
  111. package/prompts/serena.md +5 -0
  112. package/scripts/audit-specs.ts +464 -0
  113. package/scripts/convert-spec.ts +293 -0
  114. package/scripts/create-sc.ts +92 -0
  115. package/scripts/create-spec.ts +166 -0
  116. package/scripts/extract-constraints.ts +4 -3
  117. package/scripts/generate-code-map.ts +1 -1
  118. package/scripts/scaffold-project.ts +138 -1426
  119. package/scripts/sync-spec-tests.ts +211 -126
  120. package/.claude/agents/aditi.md +0 -47
  121. package/.claude/agents/marcus.md +0 -50
  122. package/.claude/agents/quinn.md +0 -58
  123. package/.claude/agents/rook.md +0 -47
  124. package/.claude/agents/serena.md +0 -46
  125. package/.claude/rungate.json +0 -11
  126. package/.fallowrc.json +0 -28
  127. package/.github/copilot-instructions.md +0 -20
  128. package/.rungate/conformity-findings.json +0 -648
  129. package/AGENTS.md +0 -106
  130. package/CODE-MAP.md +0 -50
  131. package/HARNESS.md +0 -88
  132. package/bun.lock +0 -26
  133. package/bunfig.toml +0 -2
  134. package/config/brief-policies.json +0 -28
  135. package/config/ceremony-profiles.json +0 -212
  136. package/config/consumer-cascade-map.json +0 -24
  137. package/config/doc-cascade-map.json +0 -24
  138. package/config/eval.json +0 -84
  139. package/config/profiles/account-plan.yaml +0 -36
  140. package/config/profiles/campaign.yaml +0 -94
  141. package/config/prove-evidence.schema.json +0 -65
  142. package/config/spec-policies.json +0 -36
  143. package/config/workflow-schema.json +0 -281
  144. package/gates/SCHEMA-GUIDE.md +0 -121
  145. package/gates/ac-quality.test.ts +0 -106
  146. package/gates/adversarial.test.ts +0 -274
  147. package/gates/brief-assembler.test.ts +0 -429
  148. package/gates/brief-assembler.ts +0 -334
  149. package/gates/bunfig.toml +0 -2
  150. package/gates/chain.test.ts +0 -91
  151. package/gates/e2e-smoke.test.ts +0 -234
  152. package/gates/error-classifier.test.ts +0 -249
  153. package/gates/error-classifier.ts +0 -89
  154. package/gates/orchestrator.test.ts +0 -950
  155. package/gates/orchestrator.ts +0 -569
  156. package/gates/preload.ts +0 -77
  157. package/gates/prompts/ac-adversary.md +0 -20
  158. package/gates/prompts/evidence-validator.md +0 -22
  159. package/gates/prompts/prove-reproducer.md +0 -53
  160. package/gates/prove.test.ts +0 -164
  161. package/gates/run-gate.ts +0 -936
  162. package/gates/schema-parity.test.ts +0 -197
  163. package/gates/schema.ts +0 -362
  164. package/gates/self-heal.test.ts +0 -511
  165. package/gates/self-heal.ts +0 -121
  166. package/gates/ship-orchestrator.test.ts +0 -379
  167. package/gates/ship-orchestrator.ts +0 -363
  168. package/gates/test-fixtures/broken-no-environments.json +0 -60
  169. package/gates/test-fixtures/valid-standard.json +0 -54
  170. package/gates/test-utils.ts +0 -43
  171. package/gates/witness.ts +0 -220
  172. package/gates/workflow.test.ts +0 -1053
  173. package/hooks/AgentBriefGuard.hook.ts +0 -586
  174. package/hooks/AgentVerdictCapture.hook.ts +0 -134
  175. package/hooks/AutoVerifyGate.hook.ts +0 -76
  176. package/hooks/GateEnforcement.hook.ts +0 -273
  177. package/hooks/IssueCloseGuard.hook.ts +0 -116
  178. package/hooks/MergeGuard.hook.ts +0 -55
  179. package/hooks/StaleTTLCleanup.hook.ts +0 -133
  180. package/hooks/VerifyPhaseLock.hook.ts +0 -81
  181. package/hooks/WorkflowStateGuard.hook.ts +0 -37
  182. package/hooks/lib/agentDetection.ts +0 -70
  183. package/hooks/lib/findWorkflow.ts +0 -101
  184. package/hooks/lib/parseStdin.ts +0 -37
  185. package/hooks/lib/paths.ts +0 -111
  186. package/hooks/lib/utils.ts +0 -81
  187. package/reference/migration-manifest.json +0 -554
  188. package/reference/prompts/agent-aditi.md +0 -38
  189. package/reference/prompts/agent-marcus.md +0 -38
  190. package/reference/prompts/agent-quinn.md +0 -38
  191. package/reference/prompts/agent-rook.md +0 -37
  192. package/reference/prompts/agent-serena.md +0 -37
  193. package/reference/prompts/council-skill.md +0 -170
  194. package/reference/prompts/goal-skill.md +0 -235
  195. package/reference/prompts/prove-skill.md +0 -283
  196. package/reference/prompts/quinn-standard.md +0 -246
  197. package/reference/prompts/ship-convergence.md +0 -65
  198. package/reference/prompts/ship-escalation.md +0 -43
  199. package/reference/prompts/ship-iteration.md +0 -382
  200. package/reference/prompts/ship-pending-patches.md +0 -246
  201. package/reference/prompts/ship-reference.md +0 -250
  202. package/reference/prompts/ship-skill.md +0 -347
  203. package/reference/prompts/ship-templates.md +0 -215
  204. package/reference/scripts/batch-status.sh +0 -68
  205. package/reference/scripts/chain-compat-check.sh +0 -67
  206. package/reference/scripts/context-audit.sh +0 -314
  207. package/reference/scripts/dead-export-check.sh +0 -61
  208. package/reference/scripts/generate-qc-tables.sh +0 -92
  209. package/reference/scripts/harness-init.sh +0 -111
  210. package/reference/scripts/pai-054-baseline-check.sh +0 -40
  211. package/reference/scripts/prove-smoke-test.sh +0 -112
  212. package/reference/scripts/setup-keychain-salt.sh +0 -12
  213. package/reference/scripts/skill-optimizer-friday.sh +0 -88
  214. package/reference/scripts/skill-qc-test.sh +0 -351
  215. package/reference/scripts/skill-synthesize-tonight.sh +0 -66
  216. package/reference/specs/HARNESS-EXTRACTION-SPEC.md +0 -263
  217. package/reference/specs/harness-v3-bash-deletion-plan.md +0 -65
  218. package/reference/specs/harness-v3-migration-plan.md +0 -188
  219. package/scripts/decision-reconcile.sh +0 -74
  220. package/scripts/decision-reconcile.ts +0 -152
  221. package/scripts/git-hooks/pre-commit +0 -76
  222. package/scripts/git-hooks/pre-push +0 -271
  223. package/scripts/harness-skill-check.sh +0 -52
  224. package/scripts/lib/gap-emit.sh +0 -65
  225. package/scripts/lib/jq-update.sh +0 -30
  226. package/scripts/lib/resolve-slug.sh +0 -30
  227. package/scripts/lib/verify-spec-compliance.sh +0 -102
  228. package/scripts/prove-backfill.sh +0 -87
  229. package/scripts/scaffold-rungate-config.ts +0 -91
  230. package/scripts/self-containment-check.sh +0 -109
  231. package/scripts/skill-qc-validator.sh +0 -402
  232. package/scripts/slug-resolver.sh +0 -14
  233. package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +0 -2493
  234. package/specs/BOOTSTRAP-TEST-PLAN.md +0 -571
  235. package/specs/HARNESS-GATES.md +0 -256
  236. package/specs/HARNESS-SKILL-CHAIN.md +0 -286
  237. package/specs/HARNESS-SKILL-CONTRACT.md +0 -93
  238. package/specs/HARNESS-STANDARD.md +0 -488
  239. package/specs/SPEC-TEMPLATE.md +0 -53
  240. package/specs/bootstrap-data-flow.png +0 -0
  241. package/specs/harness-automation-matrix.md +0 -314
  242. package/specs/harness-automation-matrix.rules.json +0 -64
  243. package/test/anti.test.ts +0 -80
  244. package/test/contract-negative.test.ts +0 -86
  245. package/test/contract.test.ts +0 -158
  246. package/test/external-deps.test.ts +0 -48
  247. package/test/fixtures/golden-project/.env.example +0 -3
  248. package/test/fixtures/golden-project/Makefile +0 -7
  249. package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +0 -23
  250. package/test/fixtures/golden-project/docs/setup-guide.md +0 -10
  251. package/test/fixtures/golden-project/package.json +0 -10
  252. package/test/fixtures/golden-project/specs/api-spec.md +0 -20
  253. package/test/fixtures/golden-project/src/auth.ts +0 -8
  254. package/test/fixtures/golden-project/src/db.ts +0 -11
  255. package/test/fixtures/golden-project/src/index.ts +0 -19
  256. package/test/fixtures/golden-project/src/utils/format.ts +0 -3
  257. package/test/fixtures/golden-project/test/auth.test.ts +0 -15
  258. package/test/fixtures/golden-project/tsconfig.json +0 -12
  259. package/test/meta-sc-coverage.test.ts +0 -159
  260. package/test/phase-0.test.ts +0 -846
  261. package/test/phase-1.test.ts +0 -617
  262. package/test/phase-2.test.ts +0 -620
  263. package/test/phase-3.test.ts +0 -99
  264. package/test/scaffold-conformity.test.ts +0 -8
  265. package/test/schema-canary.test.ts +0 -51
  266. package/test/spec-compliance-auto.test.ts +0 -64
  267. package/test/spec-compliance.test.ts +0 -295
  268. package/test/spec-discovery.test.ts +0 -68
  269. package/test/spec-drift.test.ts +0 -5
  270. package/test/structure.test.ts +0 -165
  271. package/test/unit/agent-brief-guard-ship.test.ts +0 -293
  272. package/test/unit/extract-constraints.test.ts +0 -145
  273. package/test/unit/issue-close-guard.test.ts +0 -204
  274. package/test/unit/scaffold-project.test.ts +0 -417
  275. package/test/unit/skill-cooccurrence.test.ts +0 -177
  276. package/test/unit/skill-sequence-logger.test.ts +0 -244
  277. package/test/unit/workflow-state-guard.test.ts +0 -95
  278. package/workflows/batch-ship.js +0 -295
  279. package/workflows/council.js +0 -481
  280. package/workflows/prove.js +0 -560
  281. package/workflows/ship.js +0 -789
  282. package/workflows/verify.js +0 -236
@@ -0,0 +1,164 @@
1
+ /**
2
+ * Hill climb mode — runs test-brief up to MAX_ITERATIONS times,
3
+ * tweaking brief between runs using compliance factors.
4
+ *
5
+ * Stops when target score is reached or max iterations exceeded.
6
+ *
7
+ * SC-404: Hill climb mode: 5 iterations max
8
+ */
9
+
10
+ import type { ComplianceResult } from "./transcript-checker.js";
11
+ import type { Directive } from "./directive-extractor.js";
12
+
13
+ export const MAX_ITERATIONS = 5;
14
+ export const DEFAULT_TARGET_SCORE = 80;
15
+
16
+ export interface HillClimbIteration {
17
+ iteration: number;
18
+ score: number;
19
+ grade: string;
20
+ followed: number;
21
+ ignored: number;
22
+ checkable: number;
23
+ recommendations: string[];
24
+ }
25
+
26
+ export interface HillClimbResult {
27
+ role: string;
28
+ iterations: HillClimbIteration[];
29
+ finalScore: number;
30
+ targetReached: boolean;
31
+ targetScore: number;
32
+ }
33
+
34
+ /**
35
+ * The 7 compliance factors used to generate recommendations
36
+ * for improving brief directive compliance.
37
+ */
38
+ export const COMPLIANCE_FACTORS = [
39
+ "position", // Move ignored directives higher in the brief
40
+ "language", // Strengthen language (MUST, MANDATORY, BEFORE)
41
+ "specificity", // Make targets more specific (exact file paths)
42
+ "deduplication", // Remove duplicate/conflicting directives
43
+ "section", // Move to stronger sections (Context > Rules > Reference)
44
+ "evidence", // Add verification commands for "always" directives
45
+ "consolidation", // Group related directives to reduce cognitive load
46
+ ] as const;
47
+
48
+ export type ComplianceFactor = typeof COMPLIANCE_FACTORS[number];
49
+
50
+ /**
51
+ * Analyze compliance results and generate improvement recommendations.
52
+ */
53
+ export function generateRecommendations(
54
+ results: ComplianceResult[],
55
+ ): { factor: ComplianceFactor; recommendation: string; directive: Directive }[] {
56
+ const recommendations: { factor: ComplianceFactor; recommendation: string; directive: Directive }[] = [];
57
+
58
+ for (const r of results) {
59
+ if (r.status === "IGNORED") {
60
+ // Position: directive is too far down
61
+ if (r.directive.line > 30) {
62
+ recommendations.push({
63
+ factor: "position",
64
+ recommendation: `Move "${r.directive.text.substring(0, 40)}" from L${r.directive.line} to Context section`,
65
+ directive: r.directive,
66
+ });
67
+ }
68
+
69
+ // Language: weak phrasing
70
+ const textLower = r.directive.text.toLowerCase();
71
+ if (!textLower.includes("must") && !textLower.includes("mandatory") && !textLower.includes("always")) {
72
+ recommendations.push({
73
+ factor: "language",
74
+ recommendation: `Strengthen language in "${r.directive.text.substring(0, 40)}" -- add MUST or MANDATORY`,
75
+ directive: r.directive,
76
+ });
77
+ }
78
+
79
+ // Section: not in a strong section
80
+ const sectionLower = r.directive.section.toLowerCase();
81
+ if (!sectionLower.includes("context") && !sectionLower.includes("always") && !sectionLower.includes("core")) {
82
+ recommendations.push({
83
+ factor: "section",
84
+ recommendation: `Move "${r.directive.text.substring(0, 40)}" from "${r.directive.section}" to Context or Always Do`,
85
+ directive: r.directive,
86
+ });
87
+ }
88
+
89
+ // Specificity: vague target
90
+ if (r.directive.type === "read" && r.directive.target && !r.directive.target.includes("/")) {
91
+ recommendations.push({
92
+ factor: "specificity",
93
+ recommendation: `Make target more specific: "${r.directive.target}" -> use full relative path`,
94
+ directive: r.directive,
95
+ });
96
+ }
97
+ }
98
+ }
99
+
100
+ return recommendations;
101
+ }
102
+
103
+ /**
104
+ * Build a hill climb iteration result from compliance data.
105
+ */
106
+ export function buildIteration(
107
+ iteration: number,
108
+ results: ComplianceResult[],
109
+ ): HillClimbIteration {
110
+ const followed = results.filter((r) => r.status === "FOLLOWED").length;
111
+ const ignored = results.filter((r) => r.status === "IGNORED").length;
112
+ const checkable = results.filter((r) => r.status !== "N/A").length;
113
+ const score = checkable > 0 ? Math.round((followed / checkable) * 100) : 0;
114
+ const grade = score >= 90 ? "A" : score >= 75 ? "B" : score >= 60 ? "C" : score >= 40 ? "D" : "F";
115
+
116
+ const recs = generateRecommendations(results);
117
+
118
+ return {
119
+ iteration,
120
+ score,
121
+ grade,
122
+ followed,
123
+ ignored,
124
+ checkable,
125
+ recommendations: recs.map((r) => `[${r.factor}] ${r.recommendation}`),
126
+ };
127
+ }
128
+
129
+ /**
130
+ * Check if target score has been reached.
131
+ */
132
+ export function isTargetReached(score: number, targetScore: number = DEFAULT_TARGET_SCORE): boolean {
133
+ return score >= targetScore;
134
+ }
135
+
136
+ /**
137
+ * Check if more iterations are allowed.
138
+ */
139
+ export function canContinue(iteration: number): boolean {
140
+ return iteration < MAX_ITERATIONS;
141
+ }
142
+
143
+ /**
144
+ * Format hill climb progress report.
145
+ */
146
+ export function formatHillClimbReport(result: HillClimbResult): string {
147
+ const lines: string[] = [];
148
+ lines.push(`\nHILL CLIMB REPORT: ${result.role}`);
149
+ lines.push(`${"=".repeat(50)}`);
150
+ lines.push(`Target: ${result.targetScore}% | Final: ${result.finalScore}% | ${result.targetReached ? "TARGET REACHED" : "TARGET NOT REACHED"}`);
151
+ lines.push(`Iterations: ${result.iterations.length}/${MAX_ITERATIONS}`);
152
+ lines.push("");
153
+
154
+ for (const iter of result.iterations) {
155
+ lines.push(` Iteration ${iter.iteration}: ${iter.grade} (${iter.score}%) -- ${iter.followed}/${iter.checkable} followed`);
156
+ if (iter.recommendations.length > 0) {
157
+ for (const rec of iter.recommendations.slice(0, 3)) {
158
+ lines.push(` ${rec}`);
159
+ }
160
+ }
161
+ }
162
+
163
+ return lines.join("\n");
164
+ }
@@ -0,0 +1,131 @@
1
+ /**
2
+ * Instruction effectiveness tracking — correlates specific instruction words
3
+ * and phrases with compliance outcomes from auditor data.
4
+ *
5
+ * Deep module: 1 export (trackEffectiveness), simple interface hiding
6
+ * tokenization, stop-word filtering, and compliance correlation.
7
+ *
8
+ * AC-7: trackEffectiveness export
9
+ */
10
+
11
+ import type { AgentAuditResult } from "./agent-audit.js";
12
+
13
+ // ── Types ───────────────────────────────────────────────
14
+
15
+ export interface WordComplianceRate {
16
+ word: string;
17
+ occurrences: number;
18
+ followedCount: number;
19
+ ignoredCount: number;
20
+ complianceRate: number;
21
+ }
22
+
23
+ export interface EffectivenessResult {
24
+ wordRates: WordComplianceRate[];
25
+ totalInstructions: number;
26
+ totalRules: number;
27
+ timestamp: string;
28
+ }
29
+
30
+ // ── Stop words ──────────────────────────────────────────
31
+
32
+ const STOP_WORDS = new Set([
33
+ "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
34
+ "have", "has", "had", "do", "does", "did", "will", "would", "could",
35
+ "shall", "can", "may", "might", "to", "of", "in", "for", "on", "with",
36
+ "at", "by", "from", "as", "into", "through", "during", "before", "after",
37
+ "above", "below", "between", "out", "off", "over", "under", "again",
38
+ "further", "then", "once", "here", "there", "when", "where", "why",
39
+ "how", "all", "both", "each", "few", "more", "most", "other", "some",
40
+ "such", "no", "nor", "not", "only", "own", "same", "so", "than", "too",
41
+ "very", "just", "but", "and", "or", "if", "it", "its", "this", "that",
42
+ "these", "those", "i", "me", "my", "we", "our", "you", "your", "he",
43
+ "him", "his", "she", "her", "they", "them", "their", "what", "which",
44
+ "who", "whom",
45
+ ]);
46
+
47
+ // ── Tokenization ────────────────────────────────────────
48
+
49
+ function tokenize(text: string): string[] {
50
+ return text
51
+ .toLowerCase()
52
+ .replace(/[^a-z0-9\s-]/g, "")
53
+ .split(/\s+/)
54
+ .filter((w) => w.length > 1 && !STOP_WORDS.has(w));
55
+ }
56
+
57
+ // ── Public API ──────────────────────────────────────────
58
+
59
+ /**
60
+ * Correlate specific instruction words and phrases with compliance outcomes
61
+ * from auditor data, producing per-word compliance rates.
62
+ *
63
+ * Matches instructions to audit rules by exact text match (case-insensitive).
64
+ * For each word in matched instructions, tracks how often it appears in
65
+ * FOLLOWED vs IGNORED rules.
66
+ *
67
+ * Returns sorted by compliance rate descending.
68
+ */
69
+ export function trackEffectiveness(
70
+ instructions: string[],
71
+ auditRules: AgentAuditResult["rules"],
72
+ ): EffectivenessResult {
73
+ if (instructions.length === 0 || auditRules.length === 0) {
74
+ return {
75
+ wordRates: [],
76
+ totalInstructions: instructions.length,
77
+ totalRules: auditRules.length,
78
+ timestamp: new Date().toISOString(),
79
+ };
80
+ }
81
+
82
+ // Build a map of rule text (lowered) -> verdict
83
+ const ruleVerdicts = new Map<string, "FOLLOWED" | "IGNORED">();
84
+ for (const rule of auditRules) {
85
+ ruleVerdicts.set(rule.rule.toLowerCase(), rule.verdict);
86
+ }
87
+
88
+ // Per-word counters
89
+ const wordFollowed = new Map<string, number>();
90
+ const wordIgnored = new Map<string, number>();
91
+ const wordTotal = new Map<string, number>();
92
+
93
+ for (const instruction of instructions) {
94
+ const verdict = ruleVerdicts.get(instruction.toLowerCase());
95
+ if (!verdict) continue;
96
+
97
+ const tokens = tokenize(instruction);
98
+ for (const token of tokens) {
99
+ wordTotal.set(token, (wordTotal.get(token) || 0) + 1);
100
+ if (verdict === "FOLLOWED") {
101
+ wordFollowed.set(token, (wordFollowed.get(token) || 0) + 1);
102
+ } else {
103
+ wordIgnored.set(token, (wordIgnored.get(token) || 0) + 1);
104
+ }
105
+ }
106
+ }
107
+
108
+ // Build rates
109
+ const wordRates: WordComplianceRate[] = [];
110
+ for (const [word, total] of wordTotal.entries()) {
111
+ const followed = wordFollowed.get(word) || 0;
112
+ const ignored = wordIgnored.get(word) || 0;
113
+ wordRates.push({
114
+ word,
115
+ occurrences: total,
116
+ followedCount: followed,
117
+ ignoredCount: ignored,
118
+ complianceRate: total > 0 ? followed / total : 0,
119
+ });
120
+ }
121
+
122
+ // Sort by compliance rate descending
123
+ wordRates.sort((a, b) => b.complianceRate - a.complianceRate);
124
+
125
+ return {
126
+ wordRates,
127
+ totalInstructions: instructions.length,
128
+ totalRules: auditRules.length,
129
+ timestamp: new Date().toISOString(),
130
+ };
131
+ }
@@ -0,0 +1,225 @@
1
+ /**
2
+ * Which closed issues were closed without going through the pipeline (#144).
3
+ *
4
+ * `IssueCloseGuard` is a `PreToolUse` hook that receives the TEXT of a Bash
5
+ * command and has to decide whether that command will close a protected
6
+ * issue. Five consecutive security-review rounds on 2026-10-06 each found a
7
+ * real differential between what it believes a command does and what the
8
+ * shell plus `gh` actually do. Each was fixed. Each fix was followed by
9
+ * another round.
10
+ *
11
+ * The class does not close, because the class is "every way to close a
12
+ * GitHub issue from a shell" — `gh api -X PATCH repos/o/n/issues/23 -f
13
+ * state=closed` mentions neither `issue` nor `close`; so does a wrapper
14
+ * script, an `xargs`, a Makefile target, or curl against the REST API. None
15
+ * of that is enumerable from a regex over a command string.
16
+ *
17
+ * **Jason's decision, 2026-10-08: Option C.** The hook stays as fast feedback
18
+ * and stops being a security control. The enforcement moves here, because
19
+ * this asks a question that IS enumerable: not "will this command close an
20
+ * issue" but "which issues are closed, and does each one have a record".
21
+ * An outcome cannot be bypassed by phrasing.
22
+ *
23
+ * ## What counts as a record, and why it is not workflow-state.json
24
+ *
25
+ * The issue says "closed with no `workflow-state.json` record", and that file
26
+ * is the right idea in the wrong place: it lives under `~/.rungate/{slug}/run/`
27
+ * on whichever machine ran the ship, is never committed, and is invisible to
28
+ * the scheduled CI job that has to do the checking. A control that can only
29
+ * run on one laptop is the same fail-open as #140.
30
+ *
31
+ * So the record is the pipeline's own OUTCOME, which the API does hold: a
32
+ * shipped issue is closed by a merged PR whose body says `Closes #N`, and
33
+ * GitHub stamps that close event with the merge `commit_id`. A close typed by
34
+ * a human — the web UI, `gh issue close`, `gh api -X PATCH` — carries no
35
+ * commit. That is a binary discriminator, visible to any token, and it does
36
+ * not care how the close was phrased.
37
+ *
38
+ * ## What this deliberately does NOT catch
39
+ *
40
+ * **A close attributable to the wrong commit.** On 2026-10-08 a commit body
41
+ * reading "not yet fixed: #178" auto-closed #178. That close has a
42
+ * `commit_id`, so this audit passes it. Attribution is not authorization, and
43
+ * pretending otherwise would be the "detector wider than what it detects"
44
+ * shape pointed inward. That defect is #178's own; this one is about closes
45
+ * with no attribution at all.
46
+ *
47
+ * Stated rather than hidden, because an audit whose scope nobody wrote down
48
+ * gets reviewed as though it claimed everything.
49
+ */
50
+
51
+ /** One closed issue, reduced to what the decision reads. */
52
+ export interface ClosedIssue {
53
+ number: number;
54
+ title: string;
55
+ /**
56
+ * What the close is attributable to — `commit <oid>` or `pull request #N` —
57
+ * or `null` for a close GitHub could attribute to nothing, which is what
58
+ * the web UI and every hand-typed close produce alike.
59
+ */
60
+ closedBy: string | null;
61
+ /** GitHub's own `state_reason`: `completed`, `not_planned`, or absent. */
62
+ stateReason: string | null;
63
+ labels: string[];
64
+ /** ISO-8601 close timestamp, used against `AUDIT_EPOCH`. */
65
+ closedAt: string | null;
66
+ }
67
+
68
+ /**
69
+ * The date this control went live. Issues closed before it are not findings.
70
+ *
71
+ * **This is a ratchet, and the number it banks was measured.** Run read-only
72
+ * against `hornjason/pai-harness` on 2026-10-13 with no epoch, the audit
73
+ * reported **19 of the last 100 closed issues**. Spot-checking #344, #343,
74
+ * #152, #274 and #141 found no merge commit, no closing PR and no
75
+ * `## Completion Report` comment on any of them — so they are true positives,
76
+ * not a broken detector. Nothing in this repo has ever written a durable
77
+ * record of a hand-typed close, which is precisely #144's point.
78
+ *
79
+ * Flagging all nineteen on day one would be the behaviour AC-4 warns about:
80
+ * a check that arrives with a backlog of findings nobody asked for gets
81
+ * switched off, and then it is worth nothing. So the control starts from
82
+ * today and ratchets forward. The nineteen are recorded in
83
+ * `specs/HARNESS-STANDARD.md` rather than labelled, which is this repo's
84
+ * convention for a ratchet: bank the number, say what it means.
85
+ *
86
+ * Lowering this date is how someone deliberately audits history. Raising it
87
+ * is how someone hides a finding, so it is a constant in version control and
88
+ * not a flag with a default.
89
+ */
90
+ export const AUDIT_EPOCH = "2026-10-13T00:00:00Z";
91
+
92
+ /**
93
+ * Flatten one GraphQL `issues.nodes[]` entry into the shape above.
94
+ *
95
+ * Lives here and not in the script because the script runs `await main()` at
96
+ * import time, so a test importing it from there would execute the audit.
97
+ *
98
+ * `ClosedEvent.closer` is why the reading is GraphQL and not REST: it
99
+ * resolves to the Commit or the PullRequest responsible, and to `null` when
100
+ * GitHub has nothing to attribute the close to. REST's `state_reason` cannot
101
+ * distinguish those, and the timeline endpoint is one request per issue.
102
+ */
103
+ export function toClosedIssue(node: any): ClosedIssue {
104
+ const closer = node?.timelineItems?.nodes?.[0]?.closer ?? null;
105
+ let closedBy: string | null = null;
106
+ if (closer?.__typename === "Commit") closedBy = `commit ${closer.oid}`;
107
+ else if (closer?.__typename === "PullRequest") closedBy = `pull request #${closer.number}`;
108
+ return {
109
+ number: node.number,
110
+ title: node.title ?? "",
111
+ // GraphQL shouts its enums. The decision compares against `not_planned`,
112
+ // and a case mismatch here would silently disable the whole exemption —
113
+ // every deliberate close would become a finding, which is how an audit
114
+ // gets switched off.
115
+ stateReason: node.stateReason ? String(node.stateReason).toLowerCase() : null,
116
+ labels: (node.labels?.nodes ?? []).map((l: any) => l.name),
117
+ closedAt: node.closedAt ?? null,
118
+ closedBy,
119
+ };
120
+ }
121
+
122
+ /** A finding: one issue, and why it was flagged. */
123
+ export interface ClosureFinding {
124
+ issue: ClosedIssue;
125
+ reason: string;
126
+ }
127
+
128
+ /**
129
+ * GitHub's marker for "closed as not planned" — duplicate, wontfix, obsolete.
130
+ *
131
+ * Honoured because those closes are legitimate and deliberate, and an audit
132
+ * that flags them cries wolf until someone switches it off. That is the
133
+ * failure AC-4 names, and it has happened to checks in this repo before.
134
+ */
135
+ export const NOT_PLANNED = "not_planned";
136
+
137
+ /**
138
+ * The label that exempts an issue, and the label the audit applies.
139
+ *
140
+ * Two different labels on purpose. One is a human saying "I closed this
141
+ * deliberately and I know it did not ship"; the other is the audit saying
142
+ * "nobody has said that about this one". Collapsing them into a single label
143
+ * would let the audit silence itself on its second run — it would apply the
144
+ * mark, then read its own mark as an exemption, and the finding would
145
+ * disappear while the issue stayed closed.
146
+ */
147
+ export const EXEMPT_LABEL = "closed-by-hand";
148
+ export const FINDING_LABEL = "closed-without-ship";
149
+
150
+ /**
151
+ * The single place that decides. One exported function, so there is one
152
+ * binding to mutate; a second comparison of the same facts elsewhere would
153
+ * survive any mutation of this one and the guard would go green for an
154
+ * unrelated reason.
155
+ *
156
+ * Takes the whole closed-issue list rather than one issue, because "which
157
+ * issues need attention" is the question the job answers and an empty result
158
+ * is a meaningful answer.
159
+ */
160
+ export function closedWithoutRecord(
161
+ issues: ClosedIssue[],
162
+ epoch: string = AUDIT_EPOCH,
163
+ ): ClosureFinding[] {
164
+ const before = Date.parse(epoch);
165
+ const out: ClosureFinding[] = [];
166
+ for (const issue of issues) {
167
+ // An unparseable or absent close date is NOT treated as old. Skipping it
168
+ // would make a malformed reading the cheapest way past the audit, which
169
+ // is the fail-open shape this whole control replaces.
170
+ const closedAt = issue.closedAt ? Date.parse(issue.closedAt) : NaN;
171
+ if (!Number.isNaN(closedAt) && closedAt < before) continue;
172
+ if (issue.closedBy !== null) continue;
173
+ if (issue.stateReason === NOT_PLANNED) continue;
174
+ if (issue.labels.includes(EXEMPT_LABEL)) continue;
175
+ if (issue.labels.includes(FINDING_LABEL)) continue; // already reported
176
+ out.push({
177
+ issue,
178
+ reason:
179
+ `closed with nothing to attribute it to, no ${NOT_PLANNED} reason and no ` +
180
+ `${EXEMPT_LABEL} label — nothing records that it went through the ship pipeline`,
181
+ });
182
+ }
183
+ return out;
184
+ }
185
+
186
+ /**
187
+ * Whether a finding should be reopened as well as labelled.
188
+ *
189
+ * **The default is to label and report, not to reopen.** AC-3 asked for a
190
+ * choice and a justification, and the three reasons are:
191
+ *
192
+ * 1. The audit is new and its false-positive rate is unmeasured. A control
193
+ * that overrules people before it has earned trust gets switched off,
194
+ * which is precisely the failure AC-4 warns about.
195
+ * 2. A label is queryable, so the control's own hit rate is measurable over
196
+ * time — `is:closed label:closed-without-ship` is a number you can watch.
197
+ * Reopening changes the state that makes the finding findable.
198
+ * 3. Reopening does not restore the gate. It restores the issue. The thing
199
+ * actually wanted is a human looking at it.
200
+ *
201
+ * `--reopen` exists so that can be revisited from evidence rather than from a
202
+ * rewrite, and it is scoped to the issues the hook itself treats as protected
203
+ * rather than to everything. A flag that reopened all findings would make the
204
+ * cautious default pointless the first time anyone used it.
205
+ */
206
+ export const PROTECTED_LABELS = ["p1-ship-next", "p2-this-week"];
207
+
208
+ export function shouldReopen(finding: ClosureFinding, reopenEnabled: boolean): boolean {
209
+ if (!reopenEnabled) return false;
210
+ return finding.issue.labels.some(l => PROTECTED_LABELS.includes(l));
211
+ }
212
+
213
+ /** The job summary, as text. Empty findings say so rather than printing nothing. */
214
+ export function report(findings: ClosureFinding[], scanned: number): string {
215
+ if (findings.length === 0) {
216
+ return `Issue closure audit: ${scanned} closed issues scanned, none closed without a record.`;
217
+ }
218
+ const lines = findings.map(
219
+ f => `- #${f.issue.number} ${f.issue.title} — ${f.reason}`,
220
+ );
221
+ return (
222
+ `Issue closure audit: ${findings.length} of ${scanned} closed issues have no ship record.\n\n` +
223
+ lines.join("\n")
224
+ );
225
+ }