scip-query 0.1.0

package/src/queries/redundant-reexports.ts

@@ -0,0 +1,170 @@
+import type { ScipDatabase } from '../db.js';
+import type { RedundantReexport } from '../types.js';
+import { shortenSymbol } from '../symbol-parser.js';
+
+/**
+ * Find barrel re-exports that no consumer actually imports through.
+ *
+ * If `queries/index.ts` re-exports `byKind` from `by-kind.ts`, but every
+ * consumer of `byKind` imports it directly from `by-kind.ts` (not through
+ * `index.ts`), the re-export in the barrel is dead weight.
+ *
+ * Algorithm:
+ * 1. Find all barrel files (index.ts / index.js)
+ * 2. For each barrel, find symbols it re-exports (defined elsewhere, referenced in barrel with role=0)
+ * 3. For each re-exported symbol, count consumers through the barrel vs direct from the source
+ * 4. If zero consumers go through the barrel, the re-export is redundant
+ */
+export function redundantReexports(
+  db: ScipDatabase,
+  opts: { scope?: string; limit?: number } = {},
+): RedundantReexport[] {
+  const { scope, limit } = opts;
+
+  const scopeFilter = scope ? `AND barrel_d.relative_path LIKE '%${scope}%'` : '';
+
+  // Step 1 + 2: Find all barrel files and symbols they re-export.
+  // A re-export is a symbol that:
+  //   - is mentioned in a barrel file with role=0 (reference/import)
+  //   - has its definition (defn_enclosing_ranges) in a DIFFERENT file
+  const reexportRows = db.all<{
+    barrel_doc_id: number;
+    barrel_path: string;
+    symbol_id: number;
+    symbol: string;
+    original_doc_id: number;
+    original_path: string;
+  }>(
+    `SELECT DISTINCT
+      barrel_d.id AS barrel_doc_id,
+      barrel_d.relative_path AS barrel_path,
+      gs.id AS symbol_id,
+      gs.symbol AS symbol,
+      orig_d.id AS original_doc_id,
+      orig_d.relative_path AS original_path
+    FROM mentions m
+    JOIN chunks c ON m.chunk_id = c.id
+    JOIN documents barrel_d ON c.document_id = barrel_d.id
+    JOIN global_symbols gs ON m.symbol_id = gs.id
+    JOIN defn_enclosing_ranges der ON gs.id = der.symbol_id
+    JOIN documents orig_d ON der.document_id = orig_d.id
+    WHERE m.role = 0
+      AND (barrel_d.relative_path LIKE '%/index.ts'
+        OR barrel_d.relative_path LIKE '%/index.js'
+        OR barrel_d.relative_path = 'index.ts'
+        OR barrel_d.relative_path = 'index.js')
+      AND orig_d.id != barrel_d.id
+      ${db.pathExclusionsFor('barrel_d', 'orig_d')}
+      ${db.symbolNoiseFor('gs')}
+      -- Only function-level symbols (ending with ().), not module-level
+      AND gs.symbol LIKE '%().'
+      ${scopeFilter}
+    ORDER BY barrel_d.relative_path, gs.symbol`,
+  );
+
+  const results: RedundantReexport[] = [];
+
+  for (const row of reexportRows) {
+    if (db.isIgnored(row.barrel_path) || db.isIgnored(row.original_path)) continue;
+
+    // Step 3: Count consumers that reference this symbol through the barrel
+    // A "barrel consumer" is a file (other than the barrel itself and the original file)
+    // that mentions this symbol AND also mentions something from the barrel document.
+    // More precisely: count distinct files that reference this symbol AND whose
+    // chunk is in a document that also has a role=0 mention pointing to the barrel file's symbols.
+    //
+    // Simpler approach: count distinct documents that reference this symbol (role=0)
+    // grouped by whether the reference chunk is in a file that imports from the barrel
+    // or from the original.
+    //
+    // Actually, the most reliable approach with SCIP data: count how many distinct
+    // consumer documents reference this symbol_id with role=0, excluding the barrel
+    // and the original file themselves. Then check if those consumers also reference
+    // ANY symbol through a mention in the barrel doc vs the original doc.
+    //
+    // Simplest correct approach: In SCIP, when file A does `import { foo } from './bar/index'`,
+    // the mention of `foo` in file A points to the same global symbol regardless of import path.
+    // SCIP doesn't track import provenance. BUT the barrel file itself contains mentions
+    // (role=0 references) of the re-exported symbols. So we can check:
+    // - barrelConsumers: files that mention both this symbol AND any symbol whose definition
+    //   is in the barrel (i.e., they import the barrel)
+    // - directConsumers: files that mention this symbol but don't import the barrel
+    //
+    // Even simpler: check if the barrel document is in the deps of the consumer.
+    // A consumer "goes through the barrel" if it has ANY role=0 mention pointing to a
+    // chunk in the barrel file. Otherwise it goes direct.
+
+    const consumerCounts = db.get<{
+      barrel_consumers: number;
+      direct_consumers: number;
+    }>(
+      `SELECT
+        SUM(CASE WHEN uses_barrel = 1 THEN 1 ELSE 0 END) AS barrel_consumers,
+        SUM(CASE WHEN uses_barrel = 0 THEN 1 ELSE 0 END) AS direct_consumers
+      FROM (
+        SELECT
+          consumer_d.id AS consumer_doc_id,
+          MAX(CASE WHEN EXISTS (
+            SELECT 1
+            FROM mentions barrel_m
+            JOIN chunks barrel_c ON barrel_m.chunk_id = barrel_c.id
+            WHERE barrel_c.document_id = consumer_d.id
+              AND barrel_m.role = 0
+              AND barrel_m.symbol_id IN (
+                SELECT m2.symbol_id
+                FROM mentions m2
+                JOIN chunks c2 ON m2.chunk_id = c2.id
+                WHERE c2.document_id = ?
+                  AND m2.role = 0
+              )
+          ) THEN 1 ELSE 0 END) AS uses_barrel
+        FROM mentions ref_m
+        JOIN chunks ref_c ON ref_m.chunk_id = ref_c.id
+        JOIN documents consumer_d ON ref_c.document_id = consumer_d.id
+        WHERE ref_m.symbol_id = ?
+          AND ref_m.role = 0
+          AND consumer_d.id != ?
+          AND consumer_d.id != ?
+          ${db.pathExclusionsFor('consumer_d')}
+        GROUP BY consumer_d.id
+      )`,
+      row.barrel_doc_id,  // for the inner subquery checking barrel mentions
+      row.symbol_id,      // the re-exported symbol
+      row.barrel_doc_id,  // exclude the barrel itself
+      row.original_doc_id, // exclude the original file
+    );
+
+    const barrelConsumers = consumerCounts?.barrel_consumers ?? 0;
+    const directConsumers = consumerCounts?.direct_consumers ?? 0;
+
+    // In TypeScript, `import * as X from './barrel'` resolves all references
+    // directly to the source file — the barrel is transparent to SCIP.
+    // This means barrelConsumers is always 0 for namespace imports.
+    //
+    // We can only confidently report symbols with 0 consumers EVERYWHERE
+    // (both barrel and direct). These are truly dead re-exports.
+    //
+    // Symbols with directConsumers > 0 but barrelConsumers === 0 might be
+    // consumed through a namespace import — we can't tell, so we skip them.
+    if (barrelConsumers === 0 && directConsumers === 0) {
+      results.push({
+        barrelFile: row.barrel_path,
+        symbol: row.symbol,
+        shortName: shortenSymbol(row.symbol),
+        originalFile: row.original_path,
+        barrelConsumers,
+        directConsumers,
+      });
+    }
+  }
+
+  // Sort: symbols with the most direct consumers first (biggest cleanup wins),
+  // then by barrel file path for stable output
+  results.sort((a, b) =>
+    b.directConsumers - a.directConsumers
+    || a.barrelFile.localeCompare(b.barrelFile)
+    || a.shortName.localeCompare(b.shortName),
+  );
+
+  return limit ? results.slice(0, limit) : results;
+}

package/src/queries/refs.ts

@@ -0,0 +1,27 @@
+import type { ScipDatabase } from '../db.js';
+import type { RefResult } from '../types.js';
+
+export function refs(db: ScipDatabase, symbolPattern: string): RefResult[] {
+  const rows = db.all<{
+    relative_path: string;
+    start_line: number;
+  }>(
+    `SELECT DISTINCT d.relative_path, c.start_line
+    FROM mentions m
+    JOIN chunks c ON m.chunk_id = c.id
+    JOIN documents d ON c.document_id = d.id
+    JOIN global_symbols gs ON m.symbol_id = gs.id
+    WHERE gs.symbol LIKE ?
+      AND ${db.localSymbolPredicate}
+      AND m.role = 0
+    ORDER BY d.relative_path, c.start_line`,
+    `%${symbolPattern}%`,
+  );
+
+  return rows
+    .filter((r) => !db.isIgnored(r.relative_path))
+    .map((r) => ({
+      relativePath: r.relative_path,
+      line: r.start_line,
+    }));
+}

package/src/queries/similar-chains.ts

@@ -0,0 +1,314 @@
+import type { ScipDatabase } from '../db.js';
+import { buildFileDepGraph } from '../query-support.js';
+import type { SimilarChainResult } from '../types.js';
+
+/**
+ * Find end-to-end dependency flows that are structurally similar
+ * but diverge at a few points — indicating duplicated pipelines
+ * that could be consolidated.
+ *
+ * Uses infrastructure-filtered chain comparison:
+ *
+ * 1. Build all dependency chains via DFS
+ * 2. Compute node frequency across all chains
+ * 3. Filter out infrastructure nodes (appearing in >50% of chains)
+ * 4. Compare the filtered chains — what's left is the unique pipeline
+ * 5. Edit distance on filtered chains finds real structural duplicates
+ *
+ * Two chains that both pass through db.ts → types.ts is meaningless.
+ * Two chains that both pass through userValidation → userRepo → emailService
+ * after filtering is a strong consolidation signal.
+ */
+export function similarChains(
+  db: ScipDatabase,
+  opts: {
+    minSimilarity?: number;
+    limit?: number;
+    scope?: string;
+    minChainLength?: number;
+    maxChainLength?: number;
+  } = {},
+): SimilarChainResult[] {
+  const {
+    minSimilarity = 0.5,
+    limit = 15,
+    scope,
+    minChainLength = 3,
+    maxChainLength = 8,
+  } = opts;
+
+  const graph = buildFileDepGraph(db, scope);
+  const rawChains = generateChains(graph, minChainLength, maxChainLength);
+
+  if (rawChains.length === 0) return [];
+
+  // Compute node frequency to identify infrastructure.
+  // Count both general frequency (appears anywhere in chain) and
+  // tail frequency (appears as one of the last 2 nodes). Shared
+  // tails like "→ db.ts → types.ts" are infrastructure even if
+  // they don't appear in >50% of chains overall.
+  const nodeFreq = new Map<string, number>();
+  const tailFreq = new Map<string, number>();
+  for (const chain of rawChains) {
+    const seen = new Set<string>();
+    for (const node of chain) {
+      if (!seen.has(node)) {
+        nodeFreq.set(node, (nodeFreq.get(node) ?? 0) + 1);
+        seen.add(node);
+      }
+    }
+    // Track tail nodes (last 2)
+    for (let t = Math.max(0, chain.length - 2); t < chain.length; t++) {
+      tailFreq.set(chain[t]!, (tailFreq.get(chain[t]!) ?? 0) + 1);
+    }
+  }
+
+  // Infrastructure: nodes in >40% of chains OR tail nodes in >30% of chains
+  const infraThreshold = rawChains.length * 0.4;
+  const tailThreshold = rawChains.length * 0.3;
+  const infraNodes = new Set<string>();
+  for (const [node, freq] of nodeFreq) {
+    if (freq > infraThreshold) infraNodes.add(node);
+  }
+  for (const [node, freq] of tailFreq) {
+    if (freq > tailThreshold) infraNodes.add(node);
+  }
+
+  // Also treat structural role files as infrastructure — entry points,
+  // barrels, and orchestrators are not meaningful pipeline nodes.
+  const structuralNames = ['index.ts', 'index.js', 'cli.ts', 'main.ts', 'health.ts', 'health.js'];
+  for (const node of nodeFreq.keys()) {
+    const basename = node.split('/').pop() ?? '';
+    if (structuralNames.includes(basename)) infraNodes.add(node);
+  }
+
+  // Filter chains: remove infrastructure nodes, keep the "unique pipeline"
+  const filteredChains: { original: string[]; filtered: string[] }[] = [];
+  for (const chain of rawChains) {
+    const filtered = chain.filter((n) => !infraNodes.has(n));
+    // Only keep chains that have at least 3 non-infrastructure nodes.
+    // Chains with 1-2 unique nodes are just "query → infra" which is
+    // the expected pattern, not a consolidation opportunity.
+    if (filtered.length >= 3) {
+      filteredChains.push({ original: chain, filtered });
+    }
+  }
+
+  if (filteredChains.length < 2) return [];
+
+  // Pairwise comparison on filtered chains
+  const results: SimilarChainResult[] = [];
+
+  for (let i = 0; i < filteredChains.length; i++) {
+    for (let j = i + 1; j < filteredChains.length; j++) {
+      const a = filteredChains[i]!;
+      const b = filteredChains[j]!;
+
+      // Quick reject: filtered chains must share at least one non-infra node
+      const setA = new Set(a.filtered);
+      let hasShared = false;
+      for (const node of b.filtered) {
+        if (setA.has(node)) { hasShared = true; break; }
+      }
+      if (!hasShared) continue;
+
+      // Edit distance on the FILTERED chains (infrastructure stripped)
+      const { distance, ops } = editDistance(a.filtered, b.filtered);
+      const maxLen = Math.max(a.filtered.length, b.filtered.length);
+      if (maxLen === 0) continue;
+
+      const similarity = 1 - distance / maxLen;
+      if (similarity < minSimilarity) continue;
+      if (distance === 0) continue; // identical filtered chains = not interesting
+
+      // Divergence points from the filtered comparison
+      const divergencePoints = ops
+        .filter((op) => op.type === 'substitute')
+        .map((op) => ({
+          index: op.indexA,
+          nodeA: a.filtered[op.indexA]!,
+          nodeB: b.filtered[op.indexB]!,
+        }));
+
+      if (divergencePoints.length === 0) continue;
+
+      // Require at least 2 matching (non-divergent) filtered nodes.
+      // A chain pair with 1 match and 1 divergence is just "two things
+      // that share one dependency" — not a duplicated pipeline.
+      const matchCount = ops.filter((op) => op.type === 'match').length;
+      if (matchCount < 2) continue;
+
+      // Report using original chains for context, but similarity is from filtered
+      const commonPrefix = getCommonPrefix(a.original, b.original);
+      const commonSuffix = getCommonSuffix(a.original, b.original);
+
+      results.push({
+        chainA: a.original,
+        chainB: b.original,
+        similarity,
+        editDistance: distance,
+        divergencePoints,
+        commonPrefix,
+        commonSuffix,
+      });
+    }
+
+    if (results.length > limit * 10) break;
+  }
+
+  // Sort by similarity desc, then fewest divergence points
+  results.sort((a, b) => {
+    if (Math.abs(b.similarity - a.similarity) > 0.01) return b.similarity - a.similarity;
+    return a.divergencePoints.length - b.divergencePoints.length;
+  });
+
+  // Deduplicate sub-chains
+  const deduped: SimilarChainResult[] = [];
+  for (const r of results) {
+    const isDuplicate = deduped.some(
+      (existing) =>
+        isSubChain(r.chainA, existing.chainA) && isSubChain(r.chainB, existing.chainB),
+    );
+    if (!isDuplicate) deduped.push(r);
+    if (deduped.length >= limit) break;
+  }
+
+  return deduped;
+}
+
+// ── Chain generation ───────────────────────────────────────
+
+function generateChains(
+  graph: Map<string, Set<string>>,
+  minLen: number,
+  maxLen: number,
+): string[][] {
+  const chains: string[][] = [];
+  const maxChains = 500;
+
+  for (const startNode of graph.keys()) {
+    if (chains.length >= maxChains) break;
+    dfsChains(graph, startNode, [startNode], new Set([startNode]), minLen, maxLen, chains, maxChains);
+  }
+
+  return chains;
+}
+
+function dfsChains(
+  graph: Map<string, Set<string>>,
+  node: string,
+  path: string[],
+  visited: Set<string>,
+  minLen: number,
+  maxLen: number,
+  results: string[][],
+  maxResults: number,
+): void {
+  if (results.length >= maxResults) return;
+  if (path.length >= maxLen) {
+    if (path.length >= minLen) results.push([...path]);
+    return;
+  }
+
+  const neighbors = graph.get(node);
+  if (!neighbors || neighbors.size === 0) {
+    if (path.length >= minLen) results.push([...path]);
+    return;
+  }
+
+  let extended = false;
+  for (const next of neighbors) {
+    if (visited.has(next)) continue;
+    visited.add(next);
+    path.push(next);
+    dfsChains(graph, next, path, visited, minLen, maxLen, results, maxResults);
+    path.pop();
+    visited.delete(next);
+    extended = true;
+    if (results.length >= maxResults) return;
+  }
+
+  if (!extended && path.length >= minLen) {
+    results.push([...path]);
+  }
+}
+
+// ── Edit distance ──────────────────────────────────────────
+
+interface EditOp {
+  type: 'match' | 'substitute' | 'insert' | 'delete';
+  indexA: number;
+  indexB: number;
+}
+
+function editDistance(a: string[], b: string[]): { distance: number; ops: EditOp[] } {
+  const m = a.length;
+  const n = b.length;
+
+  const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
+  for (let i = 0; i <= m; i++) dp[i]![0] = i;
+  for (let j = 0; j <= n; j++) dp[0]![j] = j;
+
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (a[i - 1] === b[j - 1]) {
+        dp[i]![j] = dp[i - 1]![j - 1]!;
+      } else {
+        dp[i]![j] = 1 + Math.min(
+          dp[i - 1]![j]!,
+          dp[i]![j - 1]!,
+          dp[i - 1]![j - 1]!,
+        );
+      }
+    }
+  }
+
+  const ops: EditOp[] = [];
+  let i = m, j = n;
+  while (i > 0 || j > 0) {
+    if (i > 0 && j > 0 && a[i - 1] === b[j - 1]) {
+      ops.unshift({ type: 'match', indexA: i - 1, indexB: j - 1 });
+      i--; j--;
+    } else if (i > 0 && j > 0 && dp[i]![j] === dp[i - 1]![j - 1]! + 1) {
+      ops.unshift({ type: 'substitute', indexA: i - 1, indexB: j - 1 });
+      i--; j--;
+    } else if (j > 0 && dp[i]![j] === dp[i]![j - 1]! + 1) {
+      ops.unshift({ type: 'insert', indexA: i, indexB: j - 1 });
+      j--;
+    } else {
+      ops.unshift({ type: 'delete', indexA: i - 1, indexB: j });
+      i--;
+    }
+  }
+
+  return { distance: dp[m]![n]!, ops };
+}
+
+// ── Utility ────────────────────────────────────────────────
+
+function getCommonPrefix(a: string[], b: string[]): string[] {
+  const prefix: string[] = [];
+  for (let i = 0; i < Math.min(a.length, b.length); i++) {
+    if (a[i] === b[i]) prefix.push(a[i]!);
+    else break;
+  }
+  return prefix;
+}
+
+function getCommonSuffix(a: string[], b: string[]): string[] {
+  const suffix: string[] = [];
+  let ai = a.length - 1;
+  let bi = b.length - 1;
+  while (ai >= 0 && bi >= 0 && a[ai] === b[bi]) {
+    suffix.unshift(a[ai]!);
+    ai--; bi--;
+  }
+  return suffix;
+}
+
+function isSubChain(sub: string[], full: string[]): boolean {
+  if (sub.length > full.length) return false;
+  const fullStr = full.join('→');
+  const subStr = sub.join('→');
+  return fullStr.includes(subStr);
+}

package/src/queries/similar-files.ts

@@ -0,0 +1,140 @@
+import type { ScipDatabase } from '../db.js';
+import { buildFileDepGraph } from '../query-support.js';
+import type { SimilarFileResult } from '../types.js';
+
+/**
+ * Find files with similar dependency profiles.
+ *
+ * Two files that depend on (import from) the same set of other files
+ * are structurally doing similar work. High Jaccard similarity between
+ * their dependency sets = likely copy-paste variants or consolidation candidates.
+ */
+export function similarFiles(
+  db: ScipDatabase,
+  opts: {
+    minSimilarity?: number;
+    limit?: number;
+    scope?: string;
+    minDeps?: number;
+    filePattern?: string;
+  } = {},
+): SimilarFileResult[] {
+  const { minSimilarity = 0.5, limit = 20, scope, minDeps = 3, filePattern } = opts;
+
+  // Build dependency profile for each file
+  const profiles = buildFileProfiles(db, { scope, minDeps });
+
+  const results: SimilarFileResult[] = [];
+
+  if (filePattern) {
+    // Compare one file against all others
+    const target = profiles.find((p) => p.file.includes(filePattern));
+    if (!target) return [];
+
+    for (const candidate of profiles) {
+      if (candidate.file === target.file) continue;
+      const result = compareProfiles(target, candidate, minSimilarity);
+      if (result) results.push(result);
+    }
+  } else {
+    // Pairwise comparison across all files
+    for (let i = 0; i < profiles.length; i++) {
+      for (let j = i + 1; j < profiles.length; j++) {
+        const result = compareProfiles(profiles[i]!, profiles[j]!, minSimilarity);
+        if (result) results.push(result);
+      }
+      if (results.length > limit * 5) break;
+    }
+  }
+
+  results.sort((a, b) => b.similarity - a.similarity);
+  return results.slice(0, limit);
+}
+
+// ── Internal ───────────────────────────────────────────────
+
+interface FileProfile {
+  file: string;
+  deps: Set<string>;
+}
+
+function buildFileProfiles(
+  db: ScipDatabase,
+  opts: { scope?: string; minDeps: number },
+): FileProfile[] {
+  const { scope, minDeps } = opts;
+  const depMap = buildFileDepGraph(db, scope);
+  const universalDeps = findUniversalDependencies(depMap);
+
+  // Filter to files with enough deps
+  const profiles: FileProfile[] = [];
+  for (const [file, deps] of depMap) {
+    if (deps.size >= minDeps) {
+      profiles.push({
+        file,
+        deps: new Set([...deps].filter((dep) => !universalDeps.has(dep))),
+      });
+    }
+  }
+
+  return profiles;
+}
+
+function findUniversalDependencies(
+  depMap: Map<string, Set<string>>,
+): Set<string> {
+  const universalDeps = new Set<string>();
+  const fileCount = depMap.size;
+  if (fileCount === 0) return universalDeps;
+
+  const depCounts = new Map<string, number>();
+  for (const deps of depMap.values()) {
+    for (const dep of deps) {
+      depCounts.set(dep, (depCounts.get(dep) ?? 0) + 1);
+    }
+  }
+
+  for (const [dep, count] of depCounts) {
+    if (count / fileCount > 0.5) {
+      universalDeps.add(dep);
+    }
+  }
+
+  return universalDeps;
+}
+
+function compareProfiles(
+  a: FileProfile,
+  b: FileProfile,
+  minSimilarity: number,
+): SimilarFileResult | null {
+  const shared = new Set<string>();
+  for (const dep of a.deps) {
+    if (b.deps.has(dep)) shared.add(dep);
+  }
+
+  if (shared.size === 0) return null;
+
+  const unionSize = new Set([...a.deps, ...b.deps]).size;
+  const similarity = shared.size / unionSize;
+
+  if (similarity < minSimilarity) return null;
+
+  const uniqueA: string[] = [];
+  for (const dep of a.deps) {
+    if (!b.deps.has(dep)) uniqueA.push(dep);
+  }
+  const uniqueB: string[] = [];
+  for (const dep of b.deps) {
+    if (!a.deps.has(dep)) uniqueB.push(dep);
+  }
+
+  return {
+    fileA: a.file,
+    fileB: b.file,
+    similarity,
+    sharedDeps: [...shared],
+    uniqueToA: uniqueA,
+    uniqueToB: uniqueB,
+  };
+}