rungate 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,259 @@
1
+ #!/usr/bin/env bun
2
+ import { readFileSync, readdirSync } from "fs";
3
+ import { join, basename, resolve } from "path";
4
+ import {
5
+ type ToolCall,
6
+ type TranscriptData,
7
+ type CriterionResult,
8
+ type Role,
9
+ type Verdict,
10
+ evaluateCriteria,
11
+ ROLES,
12
+ } from "../lib/eval-criteria.js";
13
+
14
+ export { type ToolCall, type CriterionResult, type Role, type Verdict };
15
+
16
+ export interface AgentAudit {
17
+ agentId: string;
18
+ label: string;
19
+ phase: string;
20
+ role: Role;
21
+ toolCalls: ToolCall[];
22
+ reads: string[];
23
+ duplicateReads: Record<string, number>;
24
+ bashes: string[];
25
+ edits: string[];
26
+ writes: string[];
27
+ totalCalls: number;
28
+ firstThreeReads: string[];
29
+ rules: CriterionResult[];
30
+ score: number;
31
+ grade: string;
32
+ }
33
+
34
+ export function parseTranscript(filePath: string): ToolCall[] {
35
+ const content = readFileSync(filePath, "utf-8");
36
+ const calls: ToolCall[] = [];
37
+ let order = 0;
38
+
39
+ for (const line of content.split("\n").filter(Boolean)) {
40
+ try {
41
+ const entry = JSON.parse(line);
42
+ if (entry.type === "assistant") {
43
+ const msg = entry.message?.content;
44
+ if (Array.isArray(msg)) {
45
+ for (const block of msg) {
46
+ if (block?.type === "tool_use") {
47
+ calls.push({ name: block.name, input: block.input || {}, order: order++ });
48
+ }
49
+ }
50
+ }
51
+ }
52
+ } catch {}
53
+ }
54
+ return calls;
55
+ }
56
+
57
+ function extractMeta(filePath: string): { label: string; phase: string; role: Role } {
58
+ const metaPath = filePath.replace(".jsonl", ".meta.json");
59
+ try {
60
+ const meta = JSON.parse(readFileSync(metaPath, "utf-8"));
61
+ const role = inferRole(meta.agentType || meta.description || "");
62
+ return {
63
+ label: meta.description || "unknown",
64
+ phase: meta.workflowPhase || "unknown",
65
+ role,
66
+ };
67
+ } catch {
68
+ return { label: "unknown", phase: "unknown", role: "marcus" };
69
+ }
70
+ }
71
+
72
+ function inferRole(hint: string): Role {
73
+ const lower = hint.toLowerCase();
74
+ if (lower.includes("da") || lower.includes("orchestrat")) return "da";
75
+ if (lower.includes("quinn") || lower.includes("valid") || lower.includes("qa")) return "quinn";
76
+ return "marcus";
77
+ }
78
+
79
+ function buildTranscriptData(
80
+ calls: ToolCall[],
81
+ reads: string[],
82
+ bashes: string[],
83
+ edits: string[],
84
+ writes: string[],
85
+ duplicateReads: Record<string, number>,
86
+ firstThreeReads: string[]
87
+ ): TranscriptData {
88
+ return { calls, reads, bashes, edits, writes, duplicateReads, firstThreeReads, promptContent: '' };
89
+ }
90
+
91
+ function applyUtilityExemptions(results: CriterionResult[], totalCalls: number): CriterionResult[] {
92
+ if (totalCalls > 3) return results;
93
+ return results.map((r) => {
94
+ if (r.id === "COMP-1" || r.id === "COMP-8") {
95
+ return { ...r, verdict: "FOLLOWED" as Verdict, evidence: "Exempt (utility agent, <= 3 calls)" };
96
+ }
97
+ return r;
98
+ });
99
+ }
100
+
101
+ /**
102
+ * Grade a transcript using role-specific evaluation criteria.
103
+ * Central grading entry point used by both audit-transcript and da-compliance.
104
+ */
105
+ export function gradeByRole(
106
+ role: Role,
107
+ calls: ToolCall[],
108
+ reads: string[],
109
+ bashes: string[],
110
+ edits: string[],
111
+ writes: string[],
112
+ duplicateReads: Record<string, number>,
113
+ firstThreeReads: string[]
114
+ ): CriterionResult[] {
115
+ const data = buildTranscriptData(calls, reads, bashes, edits, writes, duplicateReads, firstThreeReads);
116
+ const results = evaluateCriteria(role, data);
117
+ return applyUtilityExemptions(results, calls.length);
118
+ }
119
+
120
+ export function auditAgent(filePath: string, roleOverride?: Role): AgentAudit {
121
+ const agentId = basename(filePath).replace("agent-", "").replace(".jsonl", "");
122
+ const { label, phase, role: inferredRole } = extractMeta(filePath);
123
+ const role = roleOverride ?? inferredRole;
124
+ const toolCalls = parseTranscript(filePath);
125
+
126
+ const reads: string[] = [];
127
+ const bashes: string[] = [];
128
+ const edits: string[] = [];
129
+ const writes: string[] = [];
130
+
131
+ for (const tc of toolCalls) {
132
+ switch (tc.name) {
133
+ case "Read":
134
+ reads.push(tc.input.file_path || "?");
135
+ break;
136
+ case "Bash":
137
+ bashes.push(tc.input.command || "?");
138
+ break;
139
+ case "Edit":
140
+ edits.push(tc.input.file_path || "?");
141
+ break;
142
+ case "Write":
143
+ writes.push(tc.input.file_path || "?");
144
+ break;
145
+ }
146
+ }
147
+
148
+ const readCounts: Record<string, number> = {};
149
+ for (const r of reads) {
150
+ readCounts[r] = (readCounts[r] || 0) + 1;
151
+ }
152
+ const duplicateReads: Record<string, number> = {};
153
+ for (const [path, count] of Object.entries(readCounts)) {
154
+ if (count > 1) duplicateReads[path] = count;
155
+ }
156
+
157
+ const firstThreeReads = reads.slice(0, 3).map((r) => basename(r));
158
+ const rules = gradeByRole(role, toolCalls, reads, bashes, edits, writes, duplicateReads, firstThreeReads);
159
+ const maxScore = rules.reduce((s, r) => s + r.weight, 0);
160
+ const score = rules.reduce((s, r) => s + (r.verdict === "FOLLOWED" ? r.weight : 0), 0);
161
+ const pct = maxScore > 0 ? Math.round((score / maxScore) * 100) : 0;
162
+ const grade = pct >= 90 ? "A" : pct >= 75 ? "B" : pct >= 60 ? "C" : pct >= 40 ? "D" : "F";
163
+
164
+ return {
165
+ agentId,
166
+ label,
167
+ phase,
168
+ role,
169
+ toolCalls,
170
+ reads,
171
+ duplicateReads,
172
+ bashes,
173
+ edits,
174
+ writes,
175
+ totalCalls: toolCalls.length,
176
+ firstThreeReads,
177
+ rules,
178
+ score: pct,
179
+ grade,
180
+ };
181
+ }
182
+
183
+ export function formatReport(audits: AgentAudit[]): string {
184
+ const lines: string[] = [];
185
+ lines.push("# Workflow Transcript Audit Report");
186
+ lines.push(`\nAudited: ${audits.length} agents`);
187
+ lines.push(`Date: ${new Date().toISOString().split("T")[0]}`);
188
+
189
+ const avgScore = Math.round(audits.reduce((s, a) => s + a.score, 0) / audits.length);
190
+ lines.push(`\n## Summary: ${avgScore}% average (${audits.map((a) => a.grade).join(", ")})`);
191
+
192
+ for (const audit of audits) {
193
+ lines.push(`\n### ${audit.label} [${audit.role}] (${audit.phase}) — ${audit.grade} (${audit.score}%)`);
194
+ lines.push(`- Tool calls: ${audit.totalCalls} (${audit.reads.length} reads, ${audit.bashes.length} bash, ${audit.edits.length} edits)`);
195
+
196
+ if (Object.keys(audit.duplicateReads).length > 0) {
197
+ lines.push(`- **Duplicate reads:** ${Object.entries(audit.duplicateReads).map(([p, c]) => `${basename(p)} (${c}x)`).join(", ")}`);
198
+ }
199
+
200
+ lines.push(`- First reads: ${audit.firstThreeReads.join(" -> ")}`);
201
+ lines.push("");
202
+ lines.push("| ID | Rule | Verdict | Evidence |");
203
+ lines.push("|----|------|---------|----------|");
204
+ for (const r of audit.rules) {
205
+ lines.push(`| ${r.id} | ${r.rule} | ${r.verdict} | ${r.evidence} |`);
206
+ }
207
+ }
208
+
209
+ lines.push("\n## Recommendations");
210
+ const allFails = audits.flatMap((a) =>
211
+ a.rules.filter((r) => r.verdict === "IGNORED").map((r) => ({ ...r, agent: a.label }))
212
+ );
213
+ const failCounts: Record<string, number> = {};
214
+ for (const f of allFails) {
215
+ failCounts[f.rule] = (failCounts[f.rule] || 0) + 1;
216
+ }
217
+ for (const [rule, count] of Object.entries(failCounts).sort((a, b) => b[1] - a[1])) {
218
+ lines.push(`- **${rule}** — failed by ${count} agent(s)`);
219
+ }
220
+
221
+ return lines.join("\n");
222
+ }
223
+
224
+ // ── CLI ──────────────────────────────────────────────────
225
+ if (import.meta.main) {
226
+ const args = process.argv.slice(2);
227
+ const roleFlag = args.find((a) => a.startsWith("--role="));
228
+ const roleOverride = roleFlag ? (roleFlag.split("=")[1] as Role) : undefined;
229
+ const dir = args.find((a) => !a.startsWith("--"));
230
+
231
+ if (!dir) {
232
+ console.error("Usage: bun scripts/audit-transcript.ts [--role=da|marcus|quinn] <transcript-dir>");
233
+ console.error(" transcript-dir: path to workflow transcript directory containing agent-*.jsonl files");
234
+ console.error(" --role=ROLE: override role detection for all agents");
235
+ process.exit(1);
236
+ }
237
+
238
+ if (roleOverride && !ROLES.includes(roleOverride)) {
239
+ console.error(`Invalid role: ${roleOverride}. Must be one of: ${ROLES.join(", ")}`);
240
+ process.exit(1);
241
+ }
242
+
243
+ const resolvedDir = resolve(dir);
244
+ const files = readdirSync(resolvedDir).filter((f) => f.startsWith("agent-") && f.endsWith(".jsonl"));
245
+
246
+ if (files.length === 0) {
247
+ console.error(`No agent-*.jsonl files found in ${resolvedDir}`);
248
+ process.exit(1);
249
+ }
250
+
251
+ const audits = files.map((f) => auditAgent(join(resolvedDir, f), roleOverride));
252
+ const report = formatReport(audits);
253
+
254
+ console.log(report);
255
+
256
+ const outPath = join(resolvedDir, "audit-report.md");
257
+ Bun.write(outPath, report);
258
+ console.error(`\nReport written to: ${outPath}`);
259
+ }
@@ -17,6 +17,11 @@ import {
17
17
  formatSCLine,
18
18
  } from "../lib/create-sc";
19
19
  import { readScRegistry } from "./sync-sc-status";
20
+ import { scIdsSpelledInSource } from "../lib/sc-ids";
21
+ import { join } from "path";
22
+
23
+ /** Same resolution sync-sc-status uses, so both halves read one tree. */
24
+ const ROOT = join(import.meta.dir, "..");
20
25
 
21
26
  // ── Argument parsing ──────────────────────────────────────────
22
27
 
@@ -73,7 +78,16 @@ try {
73
78
  // The whole namespace, not just this file: SC ids are global, and
74
79
  // allocating from one spec's content is what produced ten collisions
75
80
  // (#392). A spec outside specs/ still gets the local answer.
76
- const scNumber = nextSCNumber(content, readScRegistry());
81
+ //
82
+ // BOTH halves of it (#399). The registry is specs only, and test fixtures
83
+ // spell SC ids in the same namespace without appearing in any registry,
84
+ // so an allocation that consults only the registry can mint a criterion
85
+ // out of a number a fixture already depends on.
86
+ const scNumber = nextSCNumber(
87
+ content,
88
+ readScRegistry(),
89
+ scIdsSpelledInSource(ROOT).keys(),
90
+ );
77
91
  const scLine = formatSCLine(scNumber, statement);
78
92
 
79
93
  // Append SC line to the end of the file
@@ -0,0 +1,200 @@
1
+ #!/usr/bin/env bun
2
+ /**
3
+ * Detect SC drift: SCs referencing nonexistent files or missing keywords.
4
+ *
5
+ * Reports two categories:
6
+ * - DRIFT: SC references a file that doesn't exist
7
+ * - STALE: SC expects a keyword in an existing file, but keyword is missing
8
+ *
9
+ * Usage:
10
+ * bun scripts/detect-sc-drift.ts # Run against project root
11
+ * import { detectDrift } from "./detect-sc-drift" # Programmatic use
12
+ */
13
+
14
+ import { existsSync, readFileSync } from "fs";
15
+ import { collectSpecFiles } from "../lib/spec-files";
16
+ import { join } from "path";
17
+
18
+ export interface DriftResult {
19
+ drifted: Map<string, DriftDetail[]>; // SCs referencing nonexistent files
20
+ stale: Map<string, StaleDetail[]>; // SCs with missing keywords
21
+ checked: number; // Total SCs checked
22
+ }
23
+
24
+ export interface DriftDetail {
25
+ scId: string;
26
+ statement: string;
27
+ path: string; // The file path that doesn't exist
28
+ }
29
+
30
+ export interface StaleDetail {
31
+ scId: string;
32
+ statement: string;
33
+ path: string; // The file that exists
34
+ keyword: string; // The keyword that's missing
35
+ }
36
+
37
+ /**
38
+ * Extract file paths from SC statement text.
39
+ * Matches patterns like: lib/foo.ts, scripts/bar.ts, test/baz.test.ts, etc.
40
+ */
41
+ function extractFilePaths(statement: string): string[] {
42
+ const paths: string[] = [];
43
+ // Match file paths: (lib|scripts|test|gates|workflows|hooks)/...(.ts|.js|.md|.json|.hook.ts)
44
+ const pathPattern = /((?:lib|scripts|test|gates|workflows|hooks)\/[^\s,\]]+\.(?:ts|js|md|json))/g;
45
+ let match;
46
+ while ((match = pathPattern.exec(statement)) !== null) {
47
+ paths.push(match[1]);
48
+ }
49
+ return paths;
50
+ }
51
+
52
+ /**
53
+ * The keywords a `contains [a, b, c]` clause requires.
54
+ *
55
+ * It used to return the bracket contents as ONE string, so a multi-keyword SC
56
+ * was checked for the literal text `"resolveSyncPaths, harnessRoot"` — comma
57
+ * and space included — which no source file ever contains. Every such SC was
58
+ * therefore reported stale no matter what the file held, and
59
+ * `scripts/sync-sc-status.ts` refuses to flip a stale SC. A criterion the
60
+ * conformity engine had just PASSED could not be marked done, and the reason
61
+ * printed was "keyword missing from file", which is not what was wrong.
62
+ *
63
+ * A comma inside a keyword would split wrongly, so only split where the parts
64
+ * look like separate terms: the registry's own syntax is comma-separated and
65
+ * no matcher keyword in this repo contains one.
66
+ */
67
+ export function extractKeywords(statement: string): string[] {
68
+ // `not contains [...]` is an absence clause, and this detector only answers
69
+ // "a file exists but lacks a term it should have". Reading it as a positive
70
+ // requirement inverted the SC: SC-539 says ship.js must NOT contain
71
+ // `git merge ${worktreeBranch}`, and the detector reported it stale FOR
72
+ // being correct.
73
+ const match = statement.match(/(?<!not )contains \[([^\]]+)\]/);
74
+ if (!match) return [];
75
+ return match[1]
76
+ .split(",")
77
+ .map(k => k.trim())
78
+ .filter(Boolean);
79
+ }
80
+
81
+ /**
82
+ * Detect drift and staleness in SCs across all spec files.
83
+ *
84
+ * @param specsDir - Path to specs directory
85
+ * @param projectRoot - Path to project root for resolving file paths
86
+ * @returns DriftResult with drifted and stale SCs, grouped by spec file
87
+ */
88
+ export function detectDrift(specsDir: string, projectRoot: string): DriftResult {
89
+ const drifted = new Map<string, DriftDetail[]>();
90
+ const stale = new Map<string, StaleDetail[]>();
91
+ let checked = 0;
92
+
93
+ if (!existsSync(specsDir)) {
94
+ return { drifted, stale, checked };
95
+ }
96
+
97
+ // Every spec, including the ones in subdirectories (#158, missed by #225).
98
+ //
99
+ // This was `readdirSync(specsDir).filter(f => f.endsWith(".md"))` — a tenth
100
+ // flat collector that #225's sweep did not reach. The consequence is worse
101
+ // than a missed file, because of what sits on the other side of it:
102
+ // `scripts/sync-sc-status.ts` scans recursively (`new Bun.Glob("**/*.md")`,
103
+ // :54) and then calls `detectDrift` (:366) as its gate. So the flipper could
104
+ // see all 254 SCs under `specs/bootstrap-data-flow/` while the drift
105
+ // detector it trusts could see none of them: a subdirectory SC could never
106
+ // be reported stale, and therefore auto-flipped to [x] with no drift gate at
107
+ // all. The two halves disagreed about what a spec is, and the half with the
108
+ // narrower view was the one holding the brakes.
109
+ const specFiles = collectSpecFiles(specsDir);
110
+
111
+ for (const specFile of specFiles) {
112
+ const specPath = join(specsDir, specFile);
113
+ const content = readFileSync(specPath, "utf-8");
114
+
115
+ // Find all SC lines (both checked and unchecked)
116
+ const scPattern = /^- \[[ x]\] (SC-\d+):\s*(.+)$/gm;
117
+ let match;
118
+
119
+ while ((match = scPattern.exec(content)) !== null) {
120
+ const scId = match[1];
121
+ const statement = match[2].trim();
122
+ checked++;
123
+
124
+ // Check for file path references
125
+ const paths = extractFilePaths(statement);
126
+ for (const path of paths) {
127
+ const fullPath = join(projectRoot, path);
128
+ if (!existsSync(fullPath)) {
129
+ if (!drifted.has(specFile)) {
130
+ drifted.set(specFile, []);
131
+ }
132
+ drifted.get(specFile)!.push({ scId, statement, path });
133
+ } else {
134
+ // File exists — check each required keyword separately.
135
+ const keywords = extractKeywords(statement);
136
+ if (keywords.length) {
137
+ const fileContent = readFileSync(fullPath, "utf-8");
138
+ const missing = keywords.filter(k => !fileContent.includes(k));
139
+ if (missing.length) {
140
+ if (!stale.has(specFile)) {
141
+ stale.set(specFile, []);
142
+ }
143
+ // Report every missing term, not just the first: a reader who
144
+ // fixes one and re-runs should not discover the next one by
145
+ // iteration.
146
+ stale.get(specFile)!.push({ scId, statement, path, keyword: missing.join(", ") });
147
+ }
148
+ }
149
+ }
150
+ }
151
+ }
152
+ }
153
+
154
+ return { drifted, stale, checked };
155
+ }
156
+
157
+ // --- Main (only runs when executed directly) ---
158
+ const isDirectExecution = import.meta.main;
159
+
160
+ if (isDirectExecution) {
161
+ const ROOT = join(import.meta.dir, "..");
162
+ const SPECS_DIR = join(ROOT, "specs");
163
+
164
+ const result = detectDrift(SPECS_DIR, ROOT);
165
+
166
+ console.log(`\n=== SC Drift Detection ===`);
167
+ console.log(`Checked ${result.checked} SCs across specs\n`);
168
+
169
+ let totalDrifted = 0;
170
+ let totalStale = 0;
171
+
172
+ if (result.drifted.size > 0) {
173
+ console.log("DRIFTED (file doesn't exist):");
174
+ for (const [specFile, details] of result.drifted) {
175
+ for (const detail of details) {
176
+ console.log(` ${detail.scId} in ${specFile}: references ${detail.path} but file does not exist`);
177
+ totalDrifted++;
178
+ }
179
+ }
180
+ console.log();
181
+ }
182
+
183
+ if (result.stale.size > 0) {
184
+ console.log("STALE (keyword missing):");
185
+ for (const [specFile, details] of result.stale) {
186
+ for (const detail of details) {
187
+ console.log(` ${detail.scId} in ${specFile}: expects ${detail.path} to contain [${detail.keyword}] but it doesn't`);
188
+ totalStale++;
189
+ }
190
+ }
191
+ console.log();
192
+ }
193
+
194
+ if (totalDrifted === 0 && totalStale === 0) {
195
+ console.log("✓ No drift or staleness detected\n");
196
+ } else {
197
+ console.log(`Summary: ${totalDrifted} drifted, ${totalStale} stale\n`);
198
+ process.exit(1); // Non-zero exit for CI integration
199
+ }
200
+ }
@@ -3,6 +3,7 @@ import { existsSync, readFileSync, writeFileSync, readdirSync } from "fs";
3
3
  import { join, relative } from "path";
4
4
  import { createHash } from "crypto";
5
5
  import { SIGNAL_PHRASE_PATTERNS } from "../lib/signal-phrases";
6
+ import { planDocScan, type DocScanPlan } from "../lib/doc-scan-targets";
6
7
 
7
8
  // ── Types ────────────────────────────────────────────────────────
8
9
 
@@ -28,6 +29,15 @@ export interface ExtractResult {
28
29
  candidatesRemoved: number;
29
30
  };
30
31
  staleness: StalenessEntry[];
32
+ /**
33
+ * What was scanned and what could not be (#440).
34
+ *
35
+ * Carried on the result rather than printed, because the thing that went
36
+ * wrong for four months was a scan silently covering four flat directories
37
+ * while the project declared eight. A candidate count tells you nothing
38
+ * about what was never read.
39
+ */
40
+ scan: DocScanPlan;
31
41
  }
32
42
 
33
43
  // ── Helpers ──────────────────────────────────────────────────────
@@ -42,41 +52,9 @@ function surroundingContext(lines: string[], lineIdx: number): string {
42
52
  return lines.slice(start, end).join("\n");
43
53
  }
44
54
 
45
- function collectScanTargets(projectPath: string): string[] {
46
- const targets: string[] = [];
47
-
48
- // Scan all root-level .md files (not just hardcoded names)
49
- for (const f of readdirSync(projectPath)) {
50
- if (!f.endsWith(".md")) continue;
51
- const p = join(projectPath, f);
52
- targets.push(p);
53
- }
54
-
55
- const docsDir = join(projectPath, "docs");
56
- if (existsSync(docsDir)) {
57
- for (const f of readdirSync(docsDir)) {
58
- if (f === "archive" || !f.endsWith(".md")) continue;
59
- targets.push(join(docsDir, f));
60
- }
61
- const adrDir = join(docsDir, "adr");
62
- if (existsSync(adrDir)) {
63
- for (const f of readdirSync(adrDir)) {
64
- if (!f.endsWith(".md")) continue;
65
- targets.push(join(adrDir, f));
66
- }
67
- }
68
- }
69
-
70
- const specsDir = join(projectPath, "specs");
71
- if (existsSync(specsDir)) {
72
- for (const f of readdirSync(specsDir)) {
73
- if (!f.endsWith(".md")) continue;
74
- targets.push(join(specsDir, f));
75
- }
76
- }
77
-
78
- return targets;
79
- }
55
+ // `collectScanTargets` was four flat `readdirSync` loops and no recursion.
56
+ // `lib/doc-scan-targets.ts` owns this now, and derives the directories from
57
+ // the project's own routing table instead of hardcoding them. See #440.
80
58
 
81
59
  function extractSignalPhrases(content: string, filePath: string, projectPath: string): Candidate[] {
82
60
  const lines = content.split("\n");
@@ -195,7 +173,8 @@ export async function extractConstraints(
195
173
  projectPath: string,
196
174
  options?: { apply?: boolean }
197
175
  ): Promise<ExtractResult> {
198
- const scanFiles = collectScanTargets(projectPath);
176
+ const scan = planDocScan(projectPath);
177
+ const scanFiles = scan.files;
199
178
  const existingConstraints = loadExistingConstraints(projectPath);
200
179
  const rejectedHashes = loadRejectedHashes(projectPath);
201
180
 
@@ -236,6 +215,7 @@ export async function extractConstraints(
236
215
  candidatesRemoved: existingRemoved + rejectedRemoved,
237
216
  },
238
217
  staleness,
218
+ scan,
239
219
  };
240
220
  }
241
221