rungate 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +70 -0
- package/lib/aes-calculator.ts +66 -0
- package/lib/agent-audit.ts +139 -0
- package/lib/agent-brief-validation.ts +359 -0
- package/lib/agents-md-refs.ts +90 -0
- package/lib/bash-file-read.ts +197 -0
- package/lib/behavioral-cache.ts +78 -0
- package/lib/branch-cleanup.ts +234 -0
- package/lib/brief-context-parser.ts +35 -0
- package/lib/brief-precedence.ts +114 -0
- package/lib/canary.ts +117 -0
- package/lib/check-registry.ts +226 -0
- package/lib/ci-checkout.ts +426 -0
- package/lib/cli-commands.ts +137 -0
- package/lib/compliance-backfill.ts +232 -0
- package/lib/compliance-report.ts +605 -0
- package/lib/compliance.ts +361 -0
- package/lib/config-loader.ts +255 -0
- package/lib/config-readers.ts +86 -0
- package/lib/conformity.ts +2161 -137
- package/lib/container-provenance.ts +230 -0
- package/lib/create-brief.ts +150 -0
- package/lib/create-sc.ts +267 -0
- package/lib/decomposition-waves.ts +197 -0
- package/lib/deep-merge.ts +69 -0
- package/lib/dev-liveness.ts +214 -0
- package/lib/directive-extractor.ts +210 -0
- package/lib/distribution-standing.ts +113 -0
- package/lib/doc-hygiene.ts +238 -0
- package/lib/eval-criteria.ts +6 -0
- package/lib/evidence-portability.ts +123 -0
- package/lib/evidence-prevalidator.ts +390 -0
- package/lib/evidence-required.ts +150 -0
- package/lib/evidence-satisfiability.ts +288 -0
- package/lib/frontmatter.ts +163 -0
- package/lib/gap-scanner.ts +348 -0
- package/lib/gate-check-audit.ts +190 -0
- package/lib/gate-enforcement.ts +366 -0
- package/lib/generators/agent-briefs.ts +136 -0
- package/lib/generators/agents-md.ts +511 -0
- package/lib/generators/code-map.ts +60 -0
- package/lib/generators/types.ts +184 -0
- package/lib/git-ref-normalizer.ts +107 -0
- package/lib/github.ts +539 -0
- package/lib/harness-identity.ts +63 -0
- package/lib/hill-climb.ts +164 -0
- package/lib/instruction-effectiveness.ts +131 -0
- package/lib/issue-closure-audit.ts +225 -0
- package/lib/issue-create-override.ts +343 -0
- package/lib/issue-extraction.ts +368 -0
- package/lib/js-top-level-code.ts +250 -0
- package/lib/liveness-contradiction.ts +149 -0
- package/lib/organize.ts +505 -0
- package/lib/paths.ts +197 -6
- package/lib/phase2-decomposition.ts +270 -0
- package/lib/post-fix-verify.ts +157 -0
- package/lib/prior-branch.ts +512 -0
- package/lib/project-state-shape.ts +166 -0
- package/lib/promote-outputs.ts +156 -0
- package/lib/prove-label-gate.ts +128 -0
- package/lib/reachability.ts +302 -0
- package/lib/release-preflight.ts +288 -0
- package/lib/rule-health-pipeline.ts +264 -0
- package/lib/rule-health-trend.ts +161 -0
- package/lib/rule-provenance.ts +149 -0
- package/lib/rule-registry.ts +128 -0
- package/lib/rungate-schema.ts +127 -5
- package/lib/sc-claim-forms.ts +179 -0
- package/lib/sc-cover.ts +150 -0
- package/lib/sc-guard.ts +157 -0
- package/lib/sc-ids.ts +344 -0
- package/lib/scaffold/defaults.ts +165 -0
- package/lib/scaffold/managed-workflow.ts +370 -0
- package/lib/scaffold/owned-files.ts +265 -0
- package/lib/scaffold/pre-push-hook.ts +165 -0
- package/lib/scaffold/steps.ts +2621 -0
- package/lib/scanner.ts +701 -0
- package/lib/scope-denylist.ts +258 -0
- package/lib/screenshot-upload.ts +73 -0
- package/lib/secret-patterns.ts +112 -0
- package/lib/security-verdict.ts +616 -0
- package/lib/signal-phrases.ts +1 -0
- package/lib/spec-change-conformity.ts +80 -0
- package/lib/spec-files.ts +62 -0
- package/lib/spec-registry.ts +133 -0
- package/lib/spec-updater.ts +96 -0
- package/lib/stale-cleanup.ts +224 -0
- package/lib/stale-issue-scanner.ts +36 -0
- package/lib/strip-js-comments.ts +269 -0
- package/lib/suite-measurement.ts +228 -0
- package/lib/task-completion-checks.ts +224 -0
- package/lib/test-suite-lock.ts +1136 -0
- package/lib/transcript-checker.ts +1140 -0
- package/lib/typescript-project.ts +233 -0
- package/lib/validators/spec-validators.ts +171 -0
- package/lib/verdict-capture.ts +73 -0
- package/lib/verified-push.ts +535 -0
- package/lib/workflow-security.ts +174 -0
- package/lib/worktree-cleanup.ts +432 -0
- package/lib/worktree-collect.ts +283 -0
- package/lib/worktree-isolation.ts +82 -0
- package/lib/worktree-repo.ts +300 -0
- package/package.json +27 -3
- package/prompts/ac-adversary.md +14 -2
- package/prompts/aditi.md +5 -0
- package/prompts/blast-radius.md +5 -0
- package/prompts/container-verify.md +5 -0
- package/prompts/discovery.md +4 -0
- package/prompts/environment.md +5 -0
- package/prompts/evidence-hierarchy.md +5 -0
- package/prompts/evidence-validator.md +11 -0
- package/prompts/marcus.md +5 -0
- package/prompts/prevention.md +5 -0
- package/prompts/prove-reproducer.md +12 -0
- package/prompts/quinn-decision-tree.md +5 -0
- package/prompts/quinn-ui-brief.md +5 -0
- package/prompts/quinn.md +5 -0
- package/prompts/rca.md +5 -0
- package/prompts/read-before-write.md +4 -0
- package/prompts/regression.md +5 -0
- package/prompts/serena.md +5 -0
- package/scripts/audit-specs.ts +464 -0
- package/scripts/convert-spec.ts +293 -0
- package/scripts/create-sc.ts +92 -0
- package/scripts/create-spec.ts +166 -0
- package/scripts/extract-constraints.ts +4 -3
- package/scripts/generate-code-map.ts +1 -1
- package/scripts/scaffold-project.ts +138 -1426
- package/scripts/sync-spec-tests.ts +211 -126
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hill climb mode — runs test-brief up to MAX_ITERATIONS times,
|
|
3
|
+
* tweaking brief between runs using compliance factors.
|
|
4
|
+
*
|
|
5
|
+
* Stops when target score is reached or max iterations exceeded.
|
|
6
|
+
*
|
|
7
|
+
* SC-404: Hill climb mode: 5 iterations max
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ComplianceResult } from "./transcript-checker.js";
|
|
11
|
+
import type { Directive } from "./directive-extractor.js";
|
|
12
|
+
|
|
13
|
+
export const MAX_ITERATIONS = 5;
|
|
14
|
+
export const DEFAULT_TARGET_SCORE = 80;
|
|
15
|
+
|
|
16
|
+
export interface HillClimbIteration {
|
|
17
|
+
iteration: number;
|
|
18
|
+
score: number;
|
|
19
|
+
grade: string;
|
|
20
|
+
followed: number;
|
|
21
|
+
ignored: number;
|
|
22
|
+
checkable: number;
|
|
23
|
+
recommendations: string[];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface HillClimbResult {
|
|
27
|
+
role: string;
|
|
28
|
+
iterations: HillClimbIteration[];
|
|
29
|
+
finalScore: number;
|
|
30
|
+
targetReached: boolean;
|
|
31
|
+
targetScore: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The 7 compliance factors used to generate recommendations
|
|
36
|
+
* for improving brief directive compliance.
|
|
37
|
+
*/
|
|
38
|
+
export const COMPLIANCE_FACTORS = [
|
|
39
|
+
"position", // Move ignored directives higher in the brief
|
|
40
|
+
"language", // Strengthen language (MUST, MANDATORY, BEFORE)
|
|
41
|
+
"specificity", // Make targets more specific (exact file paths)
|
|
42
|
+
"deduplication", // Remove duplicate/conflicting directives
|
|
43
|
+
"section", // Move to stronger sections (Context > Rules > Reference)
|
|
44
|
+
"evidence", // Add verification commands for "always" directives
|
|
45
|
+
"consolidation", // Group related directives to reduce cognitive load
|
|
46
|
+
] as const;
|
|
47
|
+
|
|
48
|
+
export type ComplianceFactor = typeof COMPLIANCE_FACTORS[number];
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Analyze compliance results and generate improvement recommendations.
|
|
52
|
+
*/
|
|
53
|
+
export function generateRecommendations(
|
|
54
|
+
results: ComplianceResult[],
|
|
55
|
+
): { factor: ComplianceFactor; recommendation: string; directive: Directive }[] {
|
|
56
|
+
const recommendations: { factor: ComplianceFactor; recommendation: string; directive: Directive }[] = [];
|
|
57
|
+
|
|
58
|
+
for (const r of results) {
|
|
59
|
+
if (r.status === "IGNORED") {
|
|
60
|
+
// Position: directive is too far down
|
|
61
|
+
if (r.directive.line > 30) {
|
|
62
|
+
recommendations.push({
|
|
63
|
+
factor: "position",
|
|
64
|
+
recommendation: `Move "${r.directive.text.substring(0, 40)}" from L${r.directive.line} to Context section`,
|
|
65
|
+
directive: r.directive,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Language: weak phrasing
|
|
70
|
+
const textLower = r.directive.text.toLowerCase();
|
|
71
|
+
if (!textLower.includes("must") && !textLower.includes("mandatory") && !textLower.includes("always")) {
|
|
72
|
+
recommendations.push({
|
|
73
|
+
factor: "language",
|
|
74
|
+
recommendation: `Strengthen language in "${r.directive.text.substring(0, 40)}" -- add MUST or MANDATORY`,
|
|
75
|
+
directive: r.directive,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Section: not in a strong section
|
|
80
|
+
const sectionLower = r.directive.section.toLowerCase();
|
|
81
|
+
if (!sectionLower.includes("context") && !sectionLower.includes("always") && !sectionLower.includes("core")) {
|
|
82
|
+
recommendations.push({
|
|
83
|
+
factor: "section",
|
|
84
|
+
recommendation: `Move "${r.directive.text.substring(0, 40)}" from "${r.directive.section}" to Context or Always Do`,
|
|
85
|
+
directive: r.directive,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Specificity: vague target
|
|
90
|
+
if (r.directive.type === "read" && r.directive.target && !r.directive.target.includes("/")) {
|
|
91
|
+
recommendations.push({
|
|
92
|
+
factor: "specificity",
|
|
93
|
+
recommendation: `Make target more specific: "${r.directive.target}" -> use full relative path`,
|
|
94
|
+
directive: r.directive,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return recommendations;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Build a hill climb iteration result from compliance data.
|
|
105
|
+
*/
|
|
106
|
+
export function buildIteration(
|
|
107
|
+
iteration: number,
|
|
108
|
+
results: ComplianceResult[],
|
|
109
|
+
): HillClimbIteration {
|
|
110
|
+
const followed = results.filter((r) => r.status === "FOLLOWED").length;
|
|
111
|
+
const ignored = results.filter((r) => r.status === "IGNORED").length;
|
|
112
|
+
const checkable = results.filter((r) => r.status !== "N/A").length;
|
|
113
|
+
const score = checkable > 0 ? Math.round((followed / checkable) * 100) : 0;
|
|
114
|
+
const grade = score >= 90 ? "A" : score >= 75 ? "B" : score >= 60 ? "C" : score >= 40 ? "D" : "F";
|
|
115
|
+
|
|
116
|
+
const recs = generateRecommendations(results);
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
iteration,
|
|
120
|
+
score,
|
|
121
|
+
grade,
|
|
122
|
+
followed,
|
|
123
|
+
ignored,
|
|
124
|
+
checkable,
|
|
125
|
+
recommendations: recs.map((r) => `[${r.factor}] ${r.recommendation}`),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Check if target score has been reached.
|
|
131
|
+
*/
|
|
132
|
+
export function isTargetReached(score: number, targetScore: number = DEFAULT_TARGET_SCORE): boolean {
|
|
133
|
+
return score >= targetScore;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Check if more iterations are allowed.
|
|
138
|
+
*/
|
|
139
|
+
export function canContinue(iteration: number): boolean {
|
|
140
|
+
return iteration < MAX_ITERATIONS;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Format hill climb progress report.
|
|
145
|
+
*/
|
|
146
|
+
export function formatHillClimbReport(result: HillClimbResult): string {
|
|
147
|
+
const lines: string[] = [];
|
|
148
|
+
lines.push(`\nHILL CLIMB REPORT: ${result.role}`);
|
|
149
|
+
lines.push(`${"=".repeat(50)}`);
|
|
150
|
+
lines.push(`Target: ${result.targetScore}% | Final: ${result.finalScore}% | ${result.targetReached ? "TARGET REACHED" : "TARGET NOT REACHED"}`);
|
|
151
|
+
lines.push(`Iterations: ${result.iterations.length}/${MAX_ITERATIONS}`);
|
|
152
|
+
lines.push("");
|
|
153
|
+
|
|
154
|
+
for (const iter of result.iterations) {
|
|
155
|
+
lines.push(` Iteration ${iter.iteration}: ${iter.grade} (${iter.score}%) -- ${iter.followed}/${iter.checkable} followed`);
|
|
156
|
+
if (iter.recommendations.length > 0) {
|
|
157
|
+
for (const rec of iter.recommendations.slice(0, 3)) {
|
|
158
|
+
lines.push(` ${rec}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return lines.join("\n");
|
|
164
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Instruction effectiveness tracking — correlates specific instruction words
|
|
3
|
+
* and phrases with compliance outcomes from auditor data.
|
|
4
|
+
*
|
|
5
|
+
* Deep module: 1 export (trackEffectiveness), simple interface hiding
|
|
6
|
+
* tokenization, stop-word filtering, and compliance correlation.
|
|
7
|
+
*
|
|
8
|
+
* AC-7: trackEffectiveness export
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { AgentAuditResult } from "./agent-audit.js";
|
|
12
|
+
|
|
13
|
+
// ── Types ───────────────────────────────────────────────
|
|
14
|
+
|
|
15
|
+
export interface WordComplianceRate {
|
|
16
|
+
word: string;
|
|
17
|
+
occurrences: number;
|
|
18
|
+
followedCount: number;
|
|
19
|
+
ignoredCount: number;
|
|
20
|
+
complianceRate: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface EffectivenessResult {
|
|
24
|
+
wordRates: WordComplianceRate[];
|
|
25
|
+
totalInstructions: number;
|
|
26
|
+
totalRules: number;
|
|
27
|
+
timestamp: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// ── Stop words ──────────────────────────────────────────
|
|
31
|
+
|
|
32
|
+
const STOP_WORDS = new Set([
|
|
33
|
+
"a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
|
|
34
|
+
"have", "has", "had", "do", "does", "did", "will", "would", "could",
|
|
35
|
+
"shall", "can", "may", "might", "to", "of", "in", "for", "on", "with",
|
|
36
|
+
"at", "by", "from", "as", "into", "through", "during", "before", "after",
|
|
37
|
+
"above", "below", "between", "out", "off", "over", "under", "again",
|
|
38
|
+
"further", "then", "once", "here", "there", "when", "where", "why",
|
|
39
|
+
"how", "all", "both", "each", "few", "more", "most", "other", "some",
|
|
40
|
+
"such", "no", "nor", "not", "only", "own", "same", "so", "than", "too",
|
|
41
|
+
"very", "just", "but", "and", "or", "if", "it", "its", "this", "that",
|
|
42
|
+
"these", "those", "i", "me", "my", "we", "our", "you", "your", "he",
|
|
43
|
+
"him", "his", "she", "her", "they", "them", "their", "what", "which",
|
|
44
|
+
"who", "whom",
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
// ── Tokenization ────────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
function tokenize(text: string): string[] {
|
|
50
|
+
return text
|
|
51
|
+
.toLowerCase()
|
|
52
|
+
.replace(/[^a-z0-9\s-]/g, "")
|
|
53
|
+
.split(/\s+/)
|
|
54
|
+
.filter((w) => w.length > 1 && !STOP_WORDS.has(w));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Public API ──────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Correlate specific instruction words and phrases with compliance outcomes
|
|
61
|
+
* from auditor data, producing per-word compliance rates.
|
|
62
|
+
*
|
|
63
|
+
* Matches instructions to audit rules by exact text match (case-insensitive).
|
|
64
|
+
* For each word in matched instructions, tracks how often it appears in
|
|
65
|
+
* FOLLOWED vs IGNORED rules.
|
|
66
|
+
*
|
|
67
|
+
* Returns sorted by compliance rate descending.
|
|
68
|
+
*/
|
|
69
|
+
export function trackEffectiveness(
|
|
70
|
+
instructions: string[],
|
|
71
|
+
auditRules: AgentAuditResult["rules"],
|
|
72
|
+
): EffectivenessResult {
|
|
73
|
+
if (instructions.length === 0 || auditRules.length === 0) {
|
|
74
|
+
return {
|
|
75
|
+
wordRates: [],
|
|
76
|
+
totalInstructions: instructions.length,
|
|
77
|
+
totalRules: auditRules.length,
|
|
78
|
+
timestamp: new Date().toISOString(),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Build a map of rule text (lowered) -> verdict
|
|
83
|
+
const ruleVerdicts = new Map<string, "FOLLOWED" | "IGNORED">();
|
|
84
|
+
for (const rule of auditRules) {
|
|
85
|
+
ruleVerdicts.set(rule.rule.toLowerCase(), rule.verdict);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Per-word counters
|
|
89
|
+
const wordFollowed = new Map<string, number>();
|
|
90
|
+
const wordIgnored = new Map<string, number>();
|
|
91
|
+
const wordTotal = new Map<string, number>();
|
|
92
|
+
|
|
93
|
+
for (const instruction of instructions) {
|
|
94
|
+
const verdict = ruleVerdicts.get(instruction.toLowerCase());
|
|
95
|
+
if (!verdict) continue;
|
|
96
|
+
|
|
97
|
+
const tokens = tokenize(instruction);
|
|
98
|
+
for (const token of tokens) {
|
|
99
|
+
wordTotal.set(token, (wordTotal.get(token) || 0) + 1);
|
|
100
|
+
if (verdict === "FOLLOWED") {
|
|
101
|
+
wordFollowed.set(token, (wordFollowed.get(token) || 0) + 1);
|
|
102
|
+
} else {
|
|
103
|
+
wordIgnored.set(token, (wordIgnored.get(token) || 0) + 1);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Build rates
|
|
109
|
+
const wordRates: WordComplianceRate[] = [];
|
|
110
|
+
for (const [word, total] of wordTotal.entries()) {
|
|
111
|
+
const followed = wordFollowed.get(word) || 0;
|
|
112
|
+
const ignored = wordIgnored.get(word) || 0;
|
|
113
|
+
wordRates.push({
|
|
114
|
+
word,
|
|
115
|
+
occurrences: total,
|
|
116
|
+
followedCount: followed,
|
|
117
|
+
ignoredCount: ignored,
|
|
118
|
+
complianceRate: total > 0 ? followed / total : 0,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Sort by compliance rate descending
|
|
123
|
+
wordRates.sort((a, b) => b.complianceRate - a.complianceRate);
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
wordRates,
|
|
127
|
+
totalInstructions: instructions.length,
|
|
128
|
+
totalRules: auditRules.length,
|
|
129
|
+
timestamp: new Date().toISOString(),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which closed issues were closed without going through the pipeline (#144).
|
|
3
|
+
*
|
|
4
|
+
* `IssueCloseGuard` is a `PreToolUse` hook that receives the TEXT of a Bash
|
|
5
|
+
* command and has to decide whether that command will close a protected
|
|
6
|
+
* issue. Five consecutive security-review rounds on 2026-10-06 each found a
|
|
7
|
+
* real differential between what it believes a command does and what the
|
|
8
|
+
* shell plus `gh` actually do. Each was fixed. Each fix was followed by
|
|
9
|
+
* another round.
|
|
10
|
+
*
|
|
11
|
+
* The class does not close, because the class is "every way to close a
|
|
12
|
+
* GitHub issue from a shell" — `gh api -X PATCH repos/o/n/issues/23 -f
|
|
13
|
+
* state=closed` mentions neither `issue` nor `close`; so does a wrapper
|
|
14
|
+
* script, an `xargs`, a Makefile target, or curl against the REST API. None
|
|
15
|
+
* of that is enumerable from a regex over a command string.
|
|
16
|
+
*
|
|
17
|
+
* **Jason's decision, 2026-10-08: Option C.** The hook stays as fast feedback
|
|
18
|
+
* and stops being a security control. The enforcement moves here, because
|
|
19
|
+
* this asks a question that IS enumerable: not "will this command close an
|
|
20
|
+
* issue" but "which issues are closed, and does each one have a record".
|
|
21
|
+
* An outcome cannot be bypassed by phrasing.
|
|
22
|
+
*
|
|
23
|
+
* ## What counts as a record, and why it is not workflow-state.json
|
|
24
|
+
*
|
|
25
|
+
* The issue says "closed with no `workflow-state.json` record", and that file
|
|
26
|
+
* is the right idea in the wrong place: it lives under `~/.rungate/{slug}/run/`
|
|
27
|
+
* on whichever machine ran the ship, is never committed, and is invisible to
|
|
28
|
+
* the scheduled CI job that has to do the checking. A control that can only
|
|
29
|
+
* run on one laptop is the same fail-open as #140.
|
|
30
|
+
*
|
|
31
|
+
* So the record is the pipeline's own OUTCOME, which the API does hold: a
|
|
32
|
+
* shipped issue is closed by a merged PR whose body says `Closes #N`, and
|
|
33
|
+
* GitHub stamps that close event with the merge `commit_id`. A close typed by
|
|
34
|
+
* a human — the web UI, `gh issue close`, `gh api -X PATCH` — carries no
|
|
35
|
+
* commit. That is a binary discriminator, visible to any token, and it does
|
|
36
|
+
* not care how the close was phrased.
|
|
37
|
+
*
|
|
38
|
+
* ## What this deliberately does NOT catch
|
|
39
|
+
*
|
|
40
|
+
* **A close attributable to the wrong commit.** On 2026-10-08 a commit body
|
|
41
|
+
* reading "not yet fixed: #178" auto-closed #178. That close has a
|
|
42
|
+
* `commit_id`, so this audit passes it. Attribution is not authorization, and
|
|
43
|
+
* pretending otherwise would be the "detector wider than what it detects"
|
|
44
|
+
* shape pointed inward. That defect is #178's own; this one is about closes
|
|
45
|
+
* with no attribution at all.
|
|
46
|
+
*
|
|
47
|
+
* Stated rather than hidden, because an audit whose scope nobody wrote down
|
|
48
|
+
* gets reviewed as though it claimed everything.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
/** One closed issue, reduced to what the decision reads. */
|
|
52
|
+
export interface ClosedIssue {
|
|
53
|
+
number: number;
|
|
54
|
+
title: string;
|
|
55
|
+
/**
|
|
56
|
+
* What the close is attributable to — `commit <oid>` or `pull request #N` —
|
|
57
|
+
* or `null` for a close GitHub could attribute to nothing, which is what
|
|
58
|
+
* the web UI and every hand-typed close produce alike.
|
|
59
|
+
*/
|
|
60
|
+
closedBy: string | null;
|
|
61
|
+
/** GitHub's own `state_reason`: `completed`, `not_planned`, or absent. */
|
|
62
|
+
stateReason: string | null;
|
|
63
|
+
labels: string[];
|
|
64
|
+
/** ISO-8601 close timestamp, used against `AUDIT_EPOCH`. */
|
|
65
|
+
closedAt: string | null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The date this control went live. Issues closed before it are not findings.
|
|
70
|
+
*
|
|
71
|
+
* **This is a ratchet, and the number it banks was measured.** Run read-only
|
|
72
|
+
* against `hornjason/pai-harness` on 2026-10-13 with no epoch, the audit
|
|
73
|
+
* reported **19 of the last 100 closed issues**. Spot-checking #344, #343,
|
|
74
|
+
* #152, #274 and #141 found no merge commit, no closing PR and no
|
|
75
|
+
* `## Completion Report` comment on any of them — so they are true positives,
|
|
76
|
+
* not a broken detector. Nothing in this repo has ever written a durable
|
|
77
|
+
* record of a hand-typed close, which is precisely #144's point.
|
|
78
|
+
*
|
|
79
|
+
* Flagging all nineteen on day one would be the behaviour AC-4 warns about:
|
|
80
|
+
* a check that arrives with a backlog of findings nobody asked for gets
|
|
81
|
+
* switched off, and then it is worth nothing. So the control starts from
|
|
82
|
+
* today and ratchets forward. The nineteen are recorded in
|
|
83
|
+
* `specs/HARNESS-STANDARD.md` rather than labelled, which is this repo's
|
|
84
|
+
* convention for a ratchet: bank the number, say what it means.
|
|
85
|
+
*
|
|
86
|
+
* Lowering this date is how someone deliberately audits history. Raising it
|
|
87
|
+
* is how someone hides a finding, so it is a constant in version control and
|
|
88
|
+
* not a flag with a default.
|
|
89
|
+
*/
|
|
90
|
+
export const AUDIT_EPOCH = "2026-10-13T00:00:00Z";
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Flatten one GraphQL `issues.nodes[]` entry into the shape above.
|
|
94
|
+
*
|
|
95
|
+
* Lives here and not in the script because the script runs `await main()` at
|
|
96
|
+
* import time, so a test importing it from there would execute the audit.
|
|
97
|
+
*
|
|
98
|
+
* `ClosedEvent.closer` is why the reading is GraphQL and not REST: it
|
|
99
|
+
* resolves to the Commit or the PullRequest responsible, and to `null` when
|
|
100
|
+
* GitHub has nothing to attribute the close to. REST's `state_reason` cannot
|
|
101
|
+
* distinguish those, and the timeline endpoint is one request per issue.
|
|
102
|
+
*/
|
|
103
|
+
export function toClosedIssue(node: any): ClosedIssue {
|
|
104
|
+
const closer = node?.timelineItems?.nodes?.[0]?.closer ?? null;
|
|
105
|
+
let closedBy: string | null = null;
|
|
106
|
+
if (closer?.__typename === "Commit") closedBy = `commit ${closer.oid}`;
|
|
107
|
+
else if (closer?.__typename === "PullRequest") closedBy = `pull request #${closer.number}`;
|
|
108
|
+
return {
|
|
109
|
+
number: node.number,
|
|
110
|
+
title: node.title ?? "",
|
|
111
|
+
// GraphQL shouts its enums. The decision compares against `not_planned`,
|
|
112
|
+
// and a case mismatch here would silently disable the whole exemption —
|
|
113
|
+
// every deliberate close would become a finding, which is how an audit
|
|
114
|
+
// gets switched off.
|
|
115
|
+
stateReason: node.stateReason ? String(node.stateReason).toLowerCase() : null,
|
|
116
|
+
labels: (node.labels?.nodes ?? []).map((l: any) => l.name),
|
|
117
|
+
closedAt: node.closedAt ?? null,
|
|
118
|
+
closedBy,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** A finding: one issue, and why it was flagged. */
|
|
123
|
+
export interface ClosureFinding {
|
|
124
|
+
issue: ClosedIssue;
|
|
125
|
+
reason: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* GitHub's marker for "closed as not planned" — duplicate, wontfix, obsolete.
|
|
130
|
+
*
|
|
131
|
+
* Honoured because those closes are legitimate and deliberate, and an audit
|
|
132
|
+
* that flags them cries wolf until someone switches it off. That is the
|
|
133
|
+
* failure AC-4 names, and it has happened to checks in this repo before.
|
|
134
|
+
*/
|
|
135
|
+
export const NOT_PLANNED = "not_planned";
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The label that exempts an issue, and the label the audit applies.
|
|
139
|
+
*
|
|
140
|
+
* Two different labels on purpose. One is a human saying "I closed this
|
|
141
|
+
* deliberately and I know it did not ship"; the other is the audit saying
|
|
142
|
+
* "nobody has said that about this one". Collapsing them into a single label
|
|
143
|
+
* would let the audit silence itself on its second run — it would apply the
|
|
144
|
+
* mark, then read its own mark as an exemption, and the finding would
|
|
145
|
+
* disappear while the issue stayed closed.
|
|
146
|
+
*/
|
|
147
|
+
export const EXEMPT_LABEL = "closed-by-hand";
|
|
148
|
+
export const FINDING_LABEL = "closed-without-ship";
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The single place that decides. One exported function, so there is one
|
|
152
|
+
* binding to mutate; a second comparison of the same facts elsewhere would
|
|
153
|
+
* survive any mutation of this one and the guard would go green for an
|
|
154
|
+
* unrelated reason.
|
|
155
|
+
*
|
|
156
|
+
* Takes the whole closed-issue list rather than one issue, because "which
|
|
157
|
+
* issues need attention" is the question the job answers and an empty result
|
|
158
|
+
* is a meaningful answer.
|
|
159
|
+
*/
|
|
160
|
+
export function closedWithoutRecord(
|
|
161
|
+
issues: ClosedIssue[],
|
|
162
|
+
epoch: string = AUDIT_EPOCH,
|
|
163
|
+
): ClosureFinding[] {
|
|
164
|
+
const before = Date.parse(epoch);
|
|
165
|
+
const out: ClosureFinding[] = [];
|
|
166
|
+
for (const issue of issues) {
|
|
167
|
+
// An unparseable or absent close date is NOT treated as old. Skipping it
|
|
168
|
+
// would make a malformed reading the cheapest way past the audit, which
|
|
169
|
+
// is the fail-open shape this whole control replaces.
|
|
170
|
+
const closedAt = issue.closedAt ? Date.parse(issue.closedAt) : NaN;
|
|
171
|
+
if (!Number.isNaN(closedAt) && closedAt < before) continue;
|
|
172
|
+
if (issue.closedBy !== null) continue;
|
|
173
|
+
if (issue.stateReason === NOT_PLANNED) continue;
|
|
174
|
+
if (issue.labels.includes(EXEMPT_LABEL)) continue;
|
|
175
|
+
if (issue.labels.includes(FINDING_LABEL)) continue; // already reported
|
|
176
|
+
out.push({
|
|
177
|
+
issue,
|
|
178
|
+
reason:
|
|
179
|
+
`closed with nothing to attribute it to, no ${NOT_PLANNED} reason and no ` +
|
|
180
|
+
`${EXEMPT_LABEL} label — nothing records that it went through the ship pipeline`,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
return out;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Whether a finding should be reopened as well as labelled.
|
|
188
|
+
*
|
|
189
|
+
* **The default is to label and report, not to reopen.** AC-3 asked for a
|
|
190
|
+
* choice and a justification, and the three reasons are:
|
|
191
|
+
*
|
|
192
|
+
* 1. The audit is new and its false-positive rate is unmeasured. A control
|
|
193
|
+
* that overrules people before it has earned trust gets switched off,
|
|
194
|
+
* which is precisely the failure AC-4 warns about.
|
|
195
|
+
* 2. A label is queryable, so the control's own hit rate is measurable over
|
|
196
|
+
* time — `is:closed label:closed-without-ship` is a number you can watch.
|
|
197
|
+
* Reopening changes the state that makes the finding findable.
|
|
198
|
+
* 3. Reopening does not restore the gate. It restores the issue. The thing
|
|
199
|
+
* actually wanted is a human looking at it.
|
|
200
|
+
*
|
|
201
|
+
* `--reopen` exists so that can be revisited from evidence rather than from a
|
|
202
|
+
* rewrite, and it is scoped to the issues the hook itself treats as protected
|
|
203
|
+
* rather than to everything. A flag that reopened all findings would make the
|
|
204
|
+
* cautious default pointless the first time anyone used it.
|
|
205
|
+
*/
|
|
206
|
+
export const PROTECTED_LABELS = ["p1-ship-next", "p2-this-week"];
|
|
207
|
+
|
|
208
|
+
export function shouldReopen(finding: ClosureFinding, reopenEnabled: boolean): boolean {
|
|
209
|
+
if (!reopenEnabled) return false;
|
|
210
|
+
return finding.issue.labels.some(l => PROTECTED_LABELS.includes(l));
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** The job summary, as text. Empty findings say so rather than printing nothing. */
|
|
214
|
+
export function report(findings: ClosureFinding[], scanned: number): string {
|
|
215
|
+
if (findings.length === 0) {
|
|
216
|
+
return `Issue closure audit: ${scanned} closed issues scanned, none closed without a record.`;
|
|
217
|
+
}
|
|
218
|
+
const lines = findings.map(
|
|
219
|
+
f => `- #${f.issue.number} ${f.issue.title} — ${f.reason}`,
|
|
220
|
+
);
|
|
221
|
+
return (
|
|
222
|
+
`Issue closure audit: ${findings.length} of ${scanned} closed issues have no ship record.\n\n` +
|
|
223
|
+
lines.join("\n")
|
|
224
|
+
);
|
|
225
|
+
}
|