prjct-cli 1.16.0 → 1.17.0

package/CHANGELOG.md

@@ -1,12 +1,52 @@
 # Changelog
 
+## [1.17.0] - 2026-02-09
+
+### Features
+
+- implement BM25 + import graph + git co-change for zero-cost file selection (PRJ-304) (#159)
+
+
+## [1.17.0] - 2026-02-08
+
+### Features
+- **BM25 + import graph + git co-change file selection** (PRJ-304): Zero-cost file selection using three mathematical signals combined into a weighted ranker. Replaces keyword matching in smart-context with precision that matches LLM-based classification — at zero API cost.
+
+### Implementation Details
+- `core/domain/bm25.ts` — BM25 indexer: tokenizes files (exports, functions, imports, comments, path segments), builds inverted index, scores queries using Okapi BM25 (k1=1.2, b=0.75). Stores in SQLite kv_store.
+- `core/domain/import-graph.ts` — Import graph builder: parses TS/JS imports to build directed adjacency list, follows chains 2 levels deep, scores by proximity (1/(depth+1)).
+- `core/domain/git-cochange.ts` — Git co-change analyzer: parses last 100 commits, builds Jaccard similarity matrix for file pairs that change together.
+- `core/domain/file-ranker.ts` — Combined ranker: `BM25 × 0.5 + imports × 0.3 + cochange × 0.2`, normalizes each signal to [0,1], returns top 15 files.
+- `core/agentic/smart-context.ts` — Uses ranker when indexes exist, graceful fallback to regex-based domain filtering.
+- `core/services/sync-service.ts` — Builds all 3 indexes in parallel during `prjct sync`.
+
+### Learnings
+- `tokenizeQuery` must split camelCase BEFORE lowercasing — otherwise "getUserById" becomes "getuserbyid" and doesn't split
+- Jaccard similarity > cosine for co-change because data is binary (file present or not in commit)
+- Batch file reads (50 at a time) needed for indexing performance on large projects
+- Stop words list must include code keywords (import, export, const) to reduce noise in scoring
+
+### Test Plan
+
+#### For QA
+1. Run `prjct sync` — verify BM25, import graph, and co-change indexes build without errors
+2. Query "Fix auth middleware" — verify auth-related files rank higher than unrelated files
+3. Query "Build responsive dashboard" — verify frontend files rank higher than backend files
+4. Verify index rebuild time <5 seconds on 300+ file project
+5. Verify query time <50ms
+6. Verify zero API calls during file selection
+
+#### For Users
+**What changed:** File selection during task context is now powered by BM25 text search, import graph proximity, and git co-change analysis instead of keyword matching.
+**How to use:** Run `p. sync` to build indexes — file selection is automatic and more accurate.
+**Breaking changes:** None. Falls back to previous filtering if indexes don't exist.
+
 ## [1.16.0] - 2026-02-09
 
 ### Features
 
 - remove JSON storage redundancy, SQLite-only backend (PRJ-303) (#158)
 
-
 ## [1.16.0] - 2026-02-08
 
 ### Features

package/core/__tests__/domain/bm25.test.ts

@@ -0,0 +1,225 @@
+/**
+ * Tests for BM25 Text Search Index
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { buildIndex, score, tokenizeFile, tokenizeQuery } from '../../domain/bm25'
+
+// =============================================================================
+// Tokenization Tests
+// =============================================================================
+
+describe('BM25', () => {
+  describe('tokenizeFile', () => {
+    it('should extract path segments', () => {
+      const tokens = tokenizeFile('', 'core/domain/bm25.ts')
+      expect(tokens).toContain('core')
+      expect(tokens).toContain('domain')
+      expect(tokens).toContain('bm25')
+    })
+
+    it('should extract function names and split camelCase', () => {
+      const content = 'export function getUserById(id: string) { return id }'
+      const tokens = tokenizeFile(content, 'user-service.ts')
+      expect(tokens).toContain('get')
+      expect(tokens).toContain('user')
+      // 'by' and 'id' are filtered (stop word / too short)
+    })
+
+    it('should extract class names', () => {
+      const content = 'export class AuthMiddleware { handle() {} }'
+      const tokens = tokenizeFile(content, 'middleware.ts')
+      expect(tokens).toContain('auth')
+      expect(tokens).toContain('middleware')
+    })
+
+    it('should extract interface names', () => {
+      const content = 'export interface JwtPayload { sub: string }'
+      const tokens = tokenizeFile(content, 'types.ts')
+      expect(tokens).toContain('jwt')
+      expect(tokens).toContain('payload')
+    })
+
+    it('should extract import sources', () => {
+      const content = `import { Router } from './router'\nimport express from 'express'`
+      const tokens = tokenizeFile(content, 'app.ts')
+      expect(tokens).toContain('router')
+      expect(tokens).toContain('express')
+    })
+
+    it('should extract words from comments', () => {
+      const content = '// Handle authentication for JWT tokens'
+      const tokens = tokenizeFile(content, 'auth.ts')
+      expect(tokens).toContain('handle')
+      expect(tokens).toContain('authentication')
+      expect(tokens).toContain('jwt')
+      expect(tokens).toContain('tokens')
+    })
+
+    it('should extract words from JSDoc comments', () => {
+      const content = '/** Validates user session and refreshes token */'
+      const tokens = tokenizeFile(content, 'session.ts')
+      expect(tokens).toContain('validates')
+      expect(tokens).toContain('session')
+      expect(tokens).toContain('refreshes')
+      expect(tokens).toContain('token')
+    })
+
+    it('should filter out stop words', () => {
+      const content = 'export function getTheData() {}'
+      const tokens = tokenizeFile(content, 'data.ts')
+      expect(tokens).not.toContain('the')
+      expect(tokens).not.toContain('export')
+      expect(tokens).not.toContain('function')
+    })
+
+    it('should handle empty content', () => {
+      const tokens = tokenizeFile('', 'empty.ts')
+      // Should still have path segments
+      expect(tokens).toContain('empty')
+    })
+  })
+
+  describe('tokenizeQuery', () => {
+    it('should tokenize a task description', () => {
+      const tokens = tokenizeQuery('Fix the auth middleware for JWT validation')
+      expect(tokens).toContain('fix')
+      expect(tokens).toContain('auth')
+      expect(tokens).toContain('middleware')
+      expect(tokens).toContain('jwt')
+      expect(tokens).toContain('validation')
+    })
+
+    it('should split camelCase in queries', () => {
+      const tokens = tokenizeQuery('update getUserById function')
+      expect(tokens).toContain('update')
+      expect(tokens).toContain('get')
+      expect(tokens).toContain('user')
+    })
+
+    it('should remove stop words from queries', () => {
+      const tokens = tokenizeQuery('Fix the bug in the login')
+      expect(tokens).not.toContain('the')
+      expect(tokens).not.toContain('in')
+      expect(tokens).toContain('fix')
+      expect(tokens).toContain('bug')
+      expect(tokens).toContain('login')
+    })
+  })
+
+  // =============================================================================
+  // Index Building & Scoring Tests
+  // =============================================================================
+
+  describe('buildIndex + score', () => {
+    let testDir: string
+
+    beforeEach(async () => {
+      testDir = path.join(os.tmpdir(), `prjct-bm25-test-${Date.now()}`)
+      await fs.mkdir(testDir, { recursive: true })
+    })
+
+    afterEach(async () => {
+      try {
+        await fs.rm(testDir, { recursive: true, force: true })
+      } catch {
+        // Ignore cleanup errors
+      }
+    })
+
+    it('should build an index from project files', async () => {
+      // Create test files
+      await fs.writeFile(
+        path.join(testDir, 'auth.ts'),
+        `export class AuthService {\n  validateJwt(token: string) {}\n  refreshSession() {}\n}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'middleware.ts'),
+        `import { AuthService } from './auth'\nexport function authMiddleware(req: any) {}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'button.tsx'),
+        `export function Button({ label }: { label: string }) {\n  return <button>{label}</button>\n}`
+      )
+
+      const index = await buildIndex(testDir)
+
+      expect(index.totalDocs).toBe(3)
+      expect(index.avgDocLength).toBeGreaterThan(0)
+      expect(Object.keys(index.documents)).toContain('auth.ts')
+      expect(Object.keys(index.documents)).toContain('middleware.ts')
+      expect(Object.keys(index.documents)).toContain('button.tsx')
+    })
+
+    it('should rank auth files higher for auth query', async () => {
+      await fs.writeFile(
+        path.join(testDir, 'auth.ts'),
+        `export class AuthService {\n  // Authenticate user with JWT\n  validateJwt(token: string) {}\n  refreshSession() {}\n}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'middleware.ts'),
+        `import { AuthService } from './auth'\n// Auth middleware for JWT validation\nexport function authMiddleware(req: any) {}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'button.tsx'),
+        `// Render a UI button component\nexport function Button({ label }: { label: string }) {\n  return <button>{label}</button>\n}`
+      )
+
+      const index = await buildIndex(testDir)
+      const results = score('Fix the auth middleware for JWT validation', index)
+
+      // Auth and middleware should rank higher than button
+      expect(results.length).toBeGreaterThan(0)
+      const authIndex = results.findIndex((r) => r.path === 'auth.ts')
+      const middlewareIndex = results.findIndex((r) => r.path === 'middleware.ts')
+      const buttonIndex = results.findIndex((r) => r.path === 'button.tsx')
+
+      expect(authIndex).toBeLessThan(buttonIndex === -1 ? Infinity : buttonIndex)
+      expect(middlewareIndex).toBeLessThan(buttonIndex === -1 ? Infinity : buttonIndex)
+    })
+
+    it('should rank frontend files higher for UI query', async () => {
+      await fs.writeFile(
+        path.join(testDir, 'dashboard.tsx'),
+        `// Responsive dashboard with charts and data grid\nexport function Dashboard() {\n  return <div className="dashboard">Charts here</div>\n}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'api-handler.ts'),
+        `// Handle API requests for user data\nexport function handleRequest(req: any) { return {} }`
+      )
+
+      const index = await buildIndex(testDir)
+      const results = score('Build responsive dashboard', index)
+
+      expect(results.length).toBeGreaterThan(0)
+      expect(results[0].path).toBe('dashboard.tsx')
+    })
+
+    it('should skip node_modules', async () => {
+      await fs.mkdir(path.join(testDir, 'node_modules', 'pkg'), { recursive: true })
+      await fs.writeFile(path.join(testDir, 'node_modules', 'pkg', 'index.ts'), 'export default {}')
+      await fs.writeFile(path.join(testDir, 'app.ts'), 'export function main() {}')
+
+      const index = await buildIndex(testDir)
+      expect(index.totalDocs).toBe(1)
+      expect(Object.keys(index.documents)).toContain('app.ts')
+    })
+
+    it('should handle empty query', async () => {
+      await fs.writeFile(path.join(testDir, 'app.ts'), 'export function main() {}')
+
+      const index = await buildIndex(testDir)
+      const results = score('', index)
+      expect(results).toEqual([])
+    })
+
+    it('should handle empty project', async () => {
+      const index = await buildIndex(testDir)
+      expect(index.totalDocs).toBe(0)
+      expect(score('anything', index)).toEqual([])
+    })
+  })
+})

package/core/__tests__/domain/file-ranker.test.ts

@@ -0,0 +1,169 @@
+/**
+ * Tests for Combined File Ranker
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import { exec as execCallback } from 'node:child_process'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { indexProject } from '../../domain/bm25'
+import { hasIndexes, rankFiles } from '../../domain/file-ranker'
+import { indexCoChanges } from '../../domain/git-cochange'
+import { indexImports } from '../../domain/import-graph'
+import pathManager from '../../infrastructure/path-manager'
+import prjctDb from '../../storage/database'
+
+const exec = promisify(execCallback)
+
+describe('FileRanker', () => {
+  let testDir: string
+  let testProjectId: string
+  const originalGetGlobalProjectPath = pathManager.getGlobalProjectPath.bind(pathManager)
+
+  beforeEach(async () => {
+    testDir = path.join(os.tmpdir(), `prjct-ranker-test-${Date.now()}`)
+    testProjectId = `test-ranker-${Date.now()}`
+    await fs.mkdir(testDir, { recursive: true })
+
+    // Mock path manager to use temp dir
+    pathManager.getGlobalProjectPath = () => testDir
+
+    // Initialize git repo
+    await exec('git init', { cwd: testDir })
+    await exec('git config user.email "test@test.com"', { cwd: testDir })
+    await exec('git config user.name "Test"', { cwd: testDir })
+  })
+
+  afterEach(async () => {
+    pathManager.getGlobalProjectPath = originalGetGlobalProjectPath
+    prjctDb.close()
+    try {
+      await fs.rm(testDir, { recursive: true, force: true })
+    } catch {
+      // Ignore
+    }
+  })
+
+  describe('hasIndexes', () => {
+    it('should return false when no indexes exist', () => {
+      const result = hasIndexes(testProjectId)
+      expect(result.bm25).toBe(false)
+      expect(result.imports).toBe(false)
+      expect(result.cochange).toBe(false)
+    })
+
+    it('should return true after building indexes', async () => {
+      // Create a test file
+      await fs.writeFile(path.join(testDir, 'app.ts'), 'export function main() {}')
+      await exec('git add -A && git commit -m "init"', { cwd: testDir })
+
+      await indexProject(testDir, testProjectId)
+      await indexImports(testDir, testProjectId)
+      await indexCoChanges(testDir, testProjectId)
+
+      const result = hasIndexes(testProjectId)
+      expect(result.bm25).toBe(true)
+      expect(result.imports).toBe(true)
+      expect(result.cochange).toBe(true)
+    })
+  })
+
+  describe('rankFiles', () => {
+    it('should return empty array when no indexes exist', () => {
+      const result = rankFiles(testProjectId, 'anything')
+      expect(result).toEqual([])
+    })
+
+    it('should rank relevant files higher', async () => {
+      // Create auth-related files
+      await fs.writeFile(
+        path.join(testDir, 'auth.ts'),
+        `// Authentication service for JWT handling\nexport class AuthService {\n  validateJwt(token: string) { return true }\n}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'middleware.ts'),
+        `import { AuthService } from './auth'\n// Auth middleware\nexport function authMiddleware() {}`
+      )
+      await fs.writeFile(
+        path.join(testDir, 'session.ts'),
+        `import { AuthService } from './auth'\n// Session management\nexport function refreshSession() {}`
+      )
+      // Create unrelated file
+      await fs.writeFile(
+        path.join(testDir, 'button.tsx'),
+        `// UI button component\nexport function Button() { return null }`
+      )
+
+      // Create git history with co-changes
+      await exec('git add -A && git commit -m "init"', { cwd: testDir })
+      await fs.writeFile(path.join(testDir, 'auth.ts'), `export class AuthService { v2() {} }`)
+      await fs.writeFile(
+        path.join(testDir, 'middleware.ts'),
+        `export function authMiddleware() { v2 }`
+      )
+      await exec('git add -A && git commit -m "update auth"', { cwd: testDir })
+
+      // Build all indexes
+      await indexProject(testDir, testProjectId)
+      await indexImports(testDir, testProjectId)
+      await indexCoChanges(testDir, testProjectId)
+
+      const results = rankFiles(testProjectId, 'Fix auth middleware for JWT validation')
+
+      expect(results.length).toBeGreaterThan(0)
+
+      // Auth and middleware should be in results
+      const authResult = results.find((r) => r.path === 'auth.ts')
+      const middlewareResult = results.find((r) => r.path === 'middleware.ts')
+
+      expect(authResult).toBeDefined()
+      expect(middlewareResult).toBeDefined()
+
+      // Auth should rank higher than button
+      const buttonResult = results.find((r) => r.path === 'button.tsx')
+      if (authResult && buttonResult) {
+        expect(authResult.finalScore).toBeGreaterThan(buttonResult.finalScore)
+      }
+    })
+
+    it('should include signal breakdown', async () => {
+      await fs.writeFile(
+        path.join(testDir, 'auth.ts'),
+        `// Authentication\nexport class AuthService {}`
+      )
+      await exec('git add -A && git commit -m "init"', { cwd: testDir })
+
+      await indexProject(testDir, testProjectId)
+      await indexImports(testDir, testProjectId)
+      await indexCoChanges(testDir, testProjectId)
+
+      const results = rankFiles(testProjectId, 'authentication')
+
+      if (results.length > 0) {
+        const first = results[0]
+        expect(first.signals).toBeDefined()
+        expect(typeof first.signals.bm25).toBe('number')
+        expect(typeof first.signals.imports).toBe('number')
+        expect(typeof first.signals.cochange).toBe('number')
+      }
+    })
+
+    it('should respect topN config', async () => {
+      // Create many files
+      for (let i = 0; i < 20; i++) {
+        await fs.writeFile(
+          path.join(testDir, `service-${i}.ts`),
+          `// Service module ${i}\nexport function service${i}() {}`
+        )
+      }
+      await exec('git add -A && git commit -m "init"', { cwd: testDir })
+
+      await indexProject(testDir, testProjectId)
+
+      const results = rankFiles(testProjectId, 'service module', { topN: 5 })
+      expect(results.length).toBeLessThanOrEqual(5)
+    })
+  })
+})

package/core/__tests__/domain/git-cochange.test.ts

@@ -0,0 +1,121 @@
+/**
+ * Tests for Git Co-Change Analyzer
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import { exec as execCallback } from 'node:child_process'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { buildMatrix, scoreFromSeeds } from '../../domain/git-cochange'
+
+const exec = promisify(execCallback)
+
+describe('GitCoChange', () => {
+  let testDir: string
+
+  beforeEach(async () => {
+    testDir = path.join(os.tmpdir(), `prjct-cochange-test-${Date.now()}`)
+    await fs.mkdir(testDir, { recursive: true })
+
+    // Initialize a git repo
+    await exec('git init', { cwd: testDir })
+    await exec('git config user.email "test@test.com"', { cwd: testDir })
+    await exec('git config user.name "Test"', { cwd: testDir })
+  })
+
+  afterEach(async () => {
+    try {
+      await fs.rm(testDir, { recursive: true, force: true })
+    } catch {
+      // Ignore cleanup errors
+    }
+  })
+
+  describe('buildMatrix', () => {
+    it('should detect co-changed files', async () => {
+      // Create files that change together
+      for (let i = 0; i < 5; i++) {
+        await fs.writeFile(path.join(testDir, 'auth.ts'), `export const v${i} = ${i}`)
+        await fs.writeFile(path.join(testDir, 'middleware.ts'), `export const v${i} = ${i}`)
+        await exec('git add -A', { cwd: testDir })
+        await exec(`git commit -m "commit ${i}"`, { cwd: testDir })
+      }
+
+      // Create a file that changes independently
+      await fs.writeFile(path.join(testDir, 'unrelated.ts'), 'export const x = 1')
+      await exec('git add -A', { cwd: testDir })
+      await exec('git commit -m "unrelated"', { cwd: testDir })
+
+      const index = await buildMatrix(testDir, 100)
+
+      expect(index.commitsAnalyzed).toBeGreaterThan(0)
+      expect(index.matrix['auth.ts']).toBeDefined()
+      expect(index.matrix['auth.ts']['middleware.ts']).toBeGreaterThan(0)
+
+      // Auth and middleware should have high co-change
+      const similarity = index.matrix['auth.ts']['middleware.ts']
+      expect(similarity).toBeGreaterThan(0.5)
+    })
+
+    it('should be symmetric', async () => {
+      for (let i = 0; i < 3; i++) {
+        await fs.writeFile(path.join(testDir, 'a.ts'), `const v${i} = ${i}`)
+        await fs.writeFile(path.join(testDir, 'b.ts'), `const v${i} = ${i}`)
+        await exec('git add -A', { cwd: testDir })
+        await exec(`git commit -m "commit ${i}"`, { cwd: testDir })
+      }
+
+      const index = await buildMatrix(testDir, 100)
+
+      if (index.matrix['a.ts'] && index.matrix['b.ts']) {
+        expect(index.matrix['a.ts']['b.ts']).toBe(index.matrix['b.ts']['a.ts'])
+      }
+    })
+
+    it('should handle no git history', async () => {
+      const emptyDir = path.join(os.tmpdir(), `prjct-cochange-empty-${Date.now()}`)
+      await fs.mkdir(emptyDir, { recursive: true })
+
+      const index = await buildMatrix(emptyDir, 100)
+      expect(index.commitsAnalyzed).toBe(0)
+      expect(Object.keys(index.matrix)).toHaveLength(0)
+
+      await fs.rm(emptyDir, { recursive: true, force: true })
+    })
+  })
+
+  describe('scoreFromSeeds', () => {
+    it('should score co-changed files', async () => {
+      // Create co-change history
+      for (let i = 0; i < 5; i++) {
+        await fs.writeFile(path.join(testDir, 'auth.ts'), `v${i}`)
+        await fs.writeFile(path.join(testDir, 'session.ts'), `v${i}`)
+        await exec('git add -A', { cwd: testDir })
+        await exec(`git commit -m "commit ${i}"`, { cwd: testDir })
+      }
+
+      const index = await buildMatrix(testDir, 100)
+      const scores = scoreFromSeeds(['auth.ts'], index)
+
+      const sessionScore = scores.find((s) => s.path === 'session.ts')
+      expect(sessionScore).toBeDefined()
+      expect(sessionScore!.score).toBeGreaterThan(0)
+    })
+
+    it('should not include seed files in results', async () => {
+      for (let i = 0; i < 3; i++) {
+        await fs.writeFile(path.join(testDir, 'a.ts'), `v${i}`)
+        await fs.writeFile(path.join(testDir, 'b.ts'), `v${i}`)
+        await exec('git add -A', { cwd: testDir })
+        await exec(`git commit -m "commit ${i}"`, { cwd: testDir })
+      }
+
+      const index = await buildMatrix(testDir, 100)
+      const scores = scoreFromSeeds(['a.ts'], index)
+
+      expect(scores.find((s) => s.path === 'a.ts')).toBeUndefined()
+    })
+  })
+})