prjct-cli 1.16.0 → 1.17.0

package/core/domain/git-cochange.ts

@@ -0,0 +1,250 @@
+/**
+ * Git Co-Change Analyzer
+ *
+ * Analyzes git history to find files that frequently change together.
+ * If middleware.ts appears in 8 of 10 commits that touch auth.ts,
+ * they're a cluster — when one is relevant, include the other.
+ *
+ * Uses Jaccard similarity: |A ∩ B| / |A ∪ B| for each file pair.
+ *
+ * Zero API calls — pure math on git log data.
+ *
+ * @module domain/git-cochange
+ * @version 1.0.0
+ */
+
+import { exec as execCallback } from 'node:child_process'
+import { promisify } from 'node:util'
+import prjctDb from '../storage/database'
+
+const exec = promisify(execCallback)
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/** Co-change matrix: file → { related_file: similarity_score } */
+export type CoChangeMatrix = Record<string, Record<string, number>>
+
+export interface CoChangeIndex {
+  /** The co-change similarity matrix */
+  matrix: CoChangeMatrix
+  /** Number of commits analyzed */
+  commitsAnalyzed: number
+  /** Total unique files seen */
+  filesAnalyzed: number
+  /** Build timestamp */
+  builtAt: string
+}
+
+export interface CoChangeScore {
+  path: string
+  score: number
+}
+
+// =============================================================================
+// Constants
+// =============================================================================
+
+/** Minimum Jaccard similarity to include in the matrix */
+const MIN_SIMILARITY = 0.1
+
+/** Minimum times a file must appear in commits to be included */
+const MIN_FILE_OCCURRENCES = 2
+
+/** Maximum number of files in a single commit to consider (skip merges/bulk) */
+const MAX_FILES_PER_COMMIT = 30
+
+// =============================================================================
+// Git Log Parsing
+// =============================================================================
+
+/**
+ * Parse git log to extract commit → files mapping.
+ *
+ * @param projectPath - Project root path
+ * @param maxCommits - Maximum number of commits to analyze (default: 100)
+ * @returns Array of file sets, one per commit
+ */
+async function parseGitLog(projectPath: string, maxCommits = 100): Promise<Set<string>[]> {
+  try {
+    const { stdout } = await exec(
+      `git log --name-only --pretty=format:'---COMMIT---' -${maxCommits}`,
+      { cwd: projectPath, maxBuffer: 10 * 1024 * 1024 }
+    )
+
+    const commits: Set<string>[] = []
+    let currentFiles: Set<string> | null = null
+
+    for (const line of stdout.split('\n')) {
+      const trimmed = line.trim()
+      if (trimmed === '---COMMIT---') {
+        if (currentFiles && currentFiles.size > 0 && currentFiles.size <= MAX_FILES_PER_COMMIT) {
+          commits.push(currentFiles)
+        }
+        currentFiles = new Set()
+      } else if (trimmed && currentFiles) {
+        // Only include source files (skip binaries, lockfiles, etc.)
+        if (isSourceFile(trimmed)) {
+          currentFiles.add(trimmed)
+        }
+      }
+    }
+
+    // Don't forget the last commit
+    if (currentFiles && currentFiles.size > 0 && currentFiles.size <= MAX_FILES_PER_COMMIT) {
+      commits.push(currentFiles)
+    }
+
+    return commits
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Check if a path looks like a source file worth tracking.
+ */
+function isSourceFile(filePath: string): boolean {
+  const sourceExtensions = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|java|cs|rb|php|vue|svelte)$/i
+  return sourceExtensions.test(filePath) && !filePath.includes('node_modules/')
+}
+
+// =============================================================================
+// Co-Change Matrix
+// =============================================================================
+
+/**
+ * Build a co-change matrix from git history.
+ *
+ * For each pair of files that appear in the same commit,
+ * calculate Jaccard similarity = |commits_both| / |commits_either|.
+ *
+ * Performance target: <2 seconds for 100 commits.
+ */
+export async function buildMatrix(projectPath: string, maxCommits = 100): Promise<CoChangeIndex> {
+  const commitSets = await parseGitLog(projectPath, maxCommits)
+
+  // Count how many commits each file appears in
+  const fileCommitCount = new Map<string, number>()
+  // Count how many commits each pair appears in together
+  const pairCount = new Map<string, number>()
+
+  for (const files of commitSets) {
+    const fileArray = Array.from(files)
+
+    for (const file of fileArray) {
+      fileCommitCount.set(file, (fileCommitCount.get(file) || 0) + 1)
+    }
+
+    // Count co-occurrences for each pair
+    for (let i = 0; i < fileArray.length; i++) {
+      for (let j = i + 1; j < fileArray.length; j++) {
+        const key = pairKey(fileArray[i], fileArray[j])
+        pairCount.set(key, (pairCount.get(key) || 0) + 1)
+      }
+    }
+  }
+
+  // Build Jaccard similarity matrix
+  const matrix: CoChangeMatrix = {}
+
+  for (const [key, count] of pairCount) {
+    const [fileA, fileB] = key.split('\0')
+    const countA = fileCommitCount.get(fileA) || 0
+    const countB = fileCommitCount.get(fileB) || 0
+
+    // Skip rare files
+    if (countA < MIN_FILE_OCCURRENCES || countB < MIN_FILE_OCCURRENCES) continue
+
+    // Jaccard similarity
+    const unionCount = countA + countB - count
+    const similarity = unionCount > 0 ? count / unionCount : 0
+
+    if (similarity < MIN_SIMILARITY) continue
+
+    // Store bidirectionally
+    if (!matrix[fileA]) matrix[fileA] = {}
+    if (!matrix[fileB]) matrix[fileB] = {}
+    matrix[fileA][fileB] = similarity
+    matrix[fileB][fileA] = similarity
+  }
+
+  return {
+    matrix,
+    commitsAnalyzed: commitSets.length,
+    filesAnalyzed: fileCommitCount.size,
+    builtAt: new Date().toISOString(),
+  }
+}
+
+/** Create a canonical pair key (sorted to avoid duplicates) */
+function pairKey(a: string, b: string): string {
+  return a < b ? `${a}\0${b}` : `${b}\0${a}`
+}
+
+// =============================================================================
+// Scoring
+// =============================================================================
+
+/**
+ * Given a set of seed files, find co-changed files and their scores.
+ *
+ * @param seedFiles - Files already identified as relevant
+ * @param index - The co-change index
+ * @returns Scored files NOT in the seed set
+ */
+export function scoreFromSeeds(seedFiles: string[], index: CoChangeIndex): CoChangeScore[] {
+  const seedSet = new Set(seedFiles)
+  const scores = new Map<string, number>()
+
+  for (const seed of seedFiles) {
+    const related = index.matrix[seed]
+    if (!related) continue
+
+    for (const [file, similarity] of Object.entries(related)) {
+      if (seedSet.has(file)) continue
+
+      // Take the max similarity across all seed connections
+      const existing = scores.get(file) || 0
+      if (similarity > existing) {
+        scores.set(file, similarity)
+      }
+    }
+  }
+
+  return Array.from(scores.entries())
+    .map(([p, s]) => ({ path: p, score: s }))
+    .sort((a, b) => b.score - a.score)
+}
+
+// =============================================================================
+// SQLite Persistence
+// =============================================================================
+
+const INDEX_KEY = 'cochange-index'
+
+export function saveMatrix(projectId: string, index: CoChangeIndex): void {
+  prjctDb.setDoc(projectId, INDEX_KEY, index)
+}
+
+export function loadMatrix(projectId: string): CoChangeIndex | null {
+  return prjctDb.getDoc<CoChangeIndex>(projectId, INDEX_KEY)
+}
+
+// =============================================================================
+// High-level API
+// =============================================================================
+
+/**
+ * Build and persist the co-change matrix for a project.
+ */
+export async function indexCoChanges(
+  projectPath: string,
+  projectId: string,
+  maxCommits = 100
+): Promise<CoChangeIndex> {
+  const index = await buildMatrix(projectPath, maxCommits)
+  saveMatrix(projectId, index)
+  return index
+}

package/core/domain/import-graph.ts

@@ -0,0 +1,315 @@
+/**
+ * Import Graph Builder
+ *
+ * Builds a directed dependency graph from TypeScript/JavaScript imports.
+ * Uses the existing imports-tool parser to extract import relationships.
+ *
+ * When BM25 identifies a file as relevant, this follows the import chain
+ * N levels deep (default: 2) to include closely related files.
+ *
+ * Score = 1 / (depth + 1) for each reachable file.
+ * Direct imports get 0.5, 2nd-level imports get 0.33.
+ *
+ * @module domain/import-graph
+ * @version 1.0.0
+ */
+
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import prjctDb from '../storage/database'
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/** Adjacency list: file → list of files it imports (resolved paths) */
+export type ImportAdjacency = Record<string, string[]>
+
+/** Reverse adjacency: file → list of files that import it */
+export type ReverseAdjacency = Record<string, string[]>
+
+export interface ImportGraph {
+  /** Forward edges: file imports these files */
+  forward: ImportAdjacency
+  /** Reverse edges: file is imported by these files */
+  reverse: ReverseAdjacency
+  /** Total number of files in the graph */
+  fileCount: number
+  /** Total number of edges */
+  edgeCount: number
+  /** Build timestamp */
+  builtAt: string
+}
+
+export interface ImportScore {
+  path: string
+  score: number
+  depth: number
+}
+
+// =============================================================================
+// Constants
+// =============================================================================
+
+const INDEXABLE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'])
+
+const SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  'out',
+  '.next',
+  'coverage',
+  '.cache',
+  '.turbo',
+  '.vercel',
+])
+
+/** Extensions to try when resolving imports */
+const RESOLVE_EXTENSIONS = ['', '.ts', '.tsx', '.js', '.jsx', '/index.ts', '/index.js']
+
+// =============================================================================
+// Import Extraction (lightweight — no dep on imports-tool for build speed)
+// =============================================================================
+
+const IMPORT_REGEX = /(?:import|from)\s+['"]([^'"]+)['"]/g
+
+/**
+ * Extract internal import paths from file content.
+ * Only resolves relative imports (starting with . or @/).
+ */
+function extractImportSources(content: string): string[] {
+  const sources: string[] = []
+  let match: RegExpExecArray | null
+  const regex = new RegExp(IMPORT_REGEX.source, 'g')
+  while ((match = regex.exec(content)) !== null) {
+    const source = match[1]
+    if (source.startsWith('.') || source.startsWith('@/')) {
+      sources.push(source)
+    }
+  }
+  return sources
+}
+
+/**
+ * Try to resolve an import source to an actual file path.
+ */
+async function resolveImport(
+  source: string,
+  fromFile: string,
+  projectPath: string
+): Promise<string | null> {
+  let basePath: string
+
+  if (source.startsWith('@/')) {
+    basePath = path.join(projectPath, 'src', source.slice(2))
+  } else {
+    const fromDir = path.dirname(path.join(projectPath, fromFile))
+    basePath = path.resolve(fromDir, source)
+  }
+
+  for (const ext of RESOLVE_EXTENSIONS) {
+    const fullPath = basePath + ext
+    try {
+      const stat = await fs.stat(fullPath)
+      if (stat.isFile()) {
+        return path.relative(projectPath, fullPath)
+      }
+    } catch {
+      // Extension not found, try next
+    }
+  }
+  return null
+}
+
+// =============================================================================
+// Graph Building
+// =============================================================================
+
+/**
+ * Recursively list all indexable files.
+ */
+async function listFiles(dir: string, projectPath: string): Promise<string[]> {
+  const files: string[] = []
+  const entries = await fs.readdir(dir, { withFileTypes: true })
+
+  for (const entry of entries) {
+    if (SKIP_DIRS.has(entry.name)) continue
+
+    const fullPath = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      files.push(...(await listFiles(fullPath, projectPath)))
+    } else if (entry.isFile()) {
+      const ext = path.extname(entry.name).toLowerCase()
+      if (INDEXABLE_EXTENSIONS.has(ext)) {
+        files.push(path.relative(projectPath, fullPath))
+      }
+    }
+  }
+  return files
+}
+
+/**
+ * Build the import graph for a project.
+ *
+ * Performance target: <3 seconds for 500-file project.
+ */
+export async function buildGraph(projectPath: string): Promise<ImportGraph> {
+  const files = await listFiles(projectPath, projectPath)
+  const forward: ImportAdjacency = {}
+  const reverse: ReverseAdjacency = {}
+  let edgeCount = 0
+
+  // Process files in parallel batches
+  const BATCH_SIZE = 50
+  for (let i = 0; i < files.length; i += BATCH_SIZE) {
+    const batch = files.slice(i, i + BATCH_SIZE)
+    const results = await Promise.all(
+      batch.map(async (filePath) => {
+        try {
+          const content = await fs.readFile(path.join(projectPath, filePath), 'utf-8')
+          const sources = extractImportSources(content)
+          const resolved: string[] = []
+
+          for (const source of sources) {
+            const target = await resolveImport(source, filePath, projectPath)
+            if (target && target !== filePath) {
+              resolved.push(target)
+            }
+          }
+
+          return { filePath, imports: resolved }
+        } catch {
+          return { filePath, imports: [] as string[] }
+        }
+      })
+    )
+
+    for (const { filePath, imports } of results) {
+      if (imports.length === 0) continue
+
+      forward[filePath] = imports
+      edgeCount += imports.length
+
+      for (const target of imports) {
+        if (!reverse[target]) reverse[target] = []
+        reverse[target].push(filePath)
+      }
+    }
+  }
+
+  return {
+    forward,
+    reverse,
+    fileCount: files.length,
+    edgeCount,
+    builtAt: new Date().toISOString(),
+  }
+}
+
+// =============================================================================
+// Graph Scoring
+// =============================================================================
+
+/**
+ * Given a set of seed files (e.g., from BM25), follow import chains
+ * and score connected files by proximity.
+ *
+ * Score = 1 / (depth + 1):
+ * - Seed file itself: not scored (already scored by BM25)
+ * - Direct import/importer: 0.5 (depth=1)
+ * - 2nd-level: 0.33 (depth=2)
+ *
+ * Follows both forward (imports) and reverse (imported-by) edges.
+ *
+ * @param seedFiles - Files already identified as relevant
+ * @param graph - The import graph
+ * @param maxDepth - Maximum depth to follow (default: 2)
+ * @returns Scored files NOT in the seed set
+ */
+export function scoreFromSeeds(
+  seedFiles: string[],
+  graph: ImportGraph,
+  maxDepth = 2
+): ImportScore[] {
+  const seedSet = new Set(seedFiles)
+  const visited = new Map<string, { score: number; depth: number }>()
+
+  // BFS from each seed
+  const queue: Array<{ file: string; depth: number }> = []
+
+  for (const seed of seedFiles) {
+    // Add direct neighbors at depth 1
+    const forwardEdges = graph.forward[seed] || []
+    const reverseEdges = graph.reverse[seed] || []
+
+    for (const neighbor of [...forwardEdges, ...reverseEdges]) {
+      if (!seedSet.has(neighbor)) {
+        queue.push({ file: neighbor, depth: 1 })
+      }
+    }
+  }
+
+  // Process queue
+  while (queue.length > 0) {
+    const { file, depth } = queue.shift()!
+    if (depth > maxDepth) continue
+
+    const score = 1 / (depth + 1)
+    const existing = visited.get(file)
+
+    if (existing) {
+      // Keep the better (higher) score
+      if (score > existing.score) {
+        visited.set(file, { score, depth })
+      }
+      continue
+    }
+
+    visited.set(file, { score, depth })
+
+    // Continue BFS for next level
+    if (depth < maxDepth) {
+      const forwardEdges = graph.forward[file] || []
+      const reverseEdges = graph.reverse[file] || []
+
+      for (const neighbor of [...forwardEdges, ...reverseEdges]) {
+        if (!seedSet.has(neighbor) && !visited.has(neighbor)) {
+          queue.push({ file: neighbor, depth: depth + 1 })
+        }
+      }
+    }
+  }
+
+  return Array.from(visited.entries())
+    .map(([p, { score, depth }]) => ({ path: p, score, depth }))
+    .sort((a, b) => b.score - a.score)
+}
+
+// =============================================================================
+// SQLite Persistence
+// =============================================================================
+
+const INDEX_KEY = 'import-graph'
+
+export function saveGraph(projectId: string, graph: ImportGraph): void {
+  prjctDb.setDoc(projectId, INDEX_KEY, graph)
+}
+
+export function loadGraph(projectId: string): ImportGraph | null {
+  return prjctDb.getDoc<ImportGraph>(projectId, INDEX_KEY)
+}
+
+// =============================================================================
+// High-level API
+// =============================================================================
+
+/**
+ * Build and persist the import graph for a project.
+ */
+export async function indexImports(projectPath: string, projectId: string): Promise<ImportGraph> {
+  const graph = await buildGraph(projectPath)
+  saveGraph(projectId, graph)
+  return graph
+}

package/core/services/sync-service.ts

@@ -27,6 +27,9 @@ import {
   type ProjectContext,
   resolveToolIds,
 } from '../ai-tools'
+import { indexProject } from '../domain/bm25'
+import { indexCoChanges } from '../domain/git-cochange'
+import { indexImports } from '../domain/import-graph'
 import { getErrorMessage } from '../errors'
 import commandInstaller from '../infrastructure/command-installer'
 import configManager from '../infrastructure/config-manager'
@@ -138,6 +141,20 @@ class SyncService {
         this.detectStack(),
       ])
 
+      // 3b. Build file-ranking indexes IN PARALLEL (BM25, import graph, co-change)
+      // These are independent and run after directory setup
+      try {
+        await Promise.all([
+          indexProject(this.projectPath, this.projectId!),
+          indexImports(this.projectPath, this.projectId!),
+          indexCoChanges(this.projectPath, this.projectId!),
+        ])
+      } catch (error) {
+        log.debug('File ranking index build failed (non-critical)', {
+          error: getErrorMessage(error),
+        })
+      }
+
       // 4. Generate all files (depends on gathered data)
       const agents = await this.generateAgents(stack, stats)
       const skills = this.configureSkills(agents)