prjct-cli 1.22.0 → 1.23.0

package/core/services/file-categorizer.ts

@@ -1,448 +0,0 @@
-/**
- * FileCategorizer - LLM-based file categorization for Smart Context Selection
- *
- * Two-phase process:
- * 1. DISCOVER DOMAINS: LLM analyzes project structure to identify functional domains
- * 2. CATEGORIZE FILES: LLM assigns files to discovered domains
- *
- * Features:
- * - Dynamic domain discovery (no hardcoded domains)
- * - Batch processing (20 files per LLM call)
- * - Heuristic fallback when LLM unavailable
- * - Caching for efficiency
- */
-
-import path from 'node:path'
-import {
-  type CategoriesCache,
-  type DiscoveredDomains,
-  type DomainDefinition,
-  type FileCategory,
-  INDEX_VERSION,
-  indexStorage,
-  type ScoredFile,
-} from '../storage/index-storage'
-import { getTimestamp } from '../utils/date-helper'
-
-// ============================================================================
-// TYPES
-// ============================================================================
-
-export interface CategorizationResult {
-  domains: DomainDefinition[]
-  categories: FileCategory[]
-  metrics: {
-    totalFiles: number
-    categorizedFiles: number
-    domainsDiscovered: number
-    llmCalls: number
-    usedHeuristics: boolean
-  }
-}
-
-export interface CategorizationOptions {
-  batchSize?: number // Files per LLM call (default: 20)
-  maxDomains?: number // Max domains to discover (default: 15)
-  useLLM?: boolean // Use LLM or heuristics only (default: true)
-  projectId?: string // For caching
-}
-
-// ============================================================================
-// HEURISTIC PATTERNS
-// ============================================================================
-
-/**
- * Fallback heuristic patterns for when LLM is unavailable
- * Maps directory/filename patterns to domain names
- */
-const HEURISTIC_PATTERNS: { pattern: RegExp; domain: string }[] = [
-  // Payment-related
-  { pattern: /\b(payment|stripe|billing|checkout|invoice)/i, domain: 'payments' },
-
-  // User/Auth
-  { pattern: /\b(auth|login|signup|user|session|password|oauth)/i, domain: 'auth' },
-
-  // API
-  { pattern: /\b(api|endpoint|route|controller)/i, domain: 'api' },
-
-  // Database
-  { pattern: /\b(model|schema|migration|database|db|prisma|drizzle)/i, domain: 'database' },
-
-  // Frontend
-  { pattern: /\b(component|page|view|layout|ui|button|form|modal)/i, domain: 'frontend' },
-
-  // Testing
-  { pattern: /\b(test|spec|__tests__|e2e|cypress)/i, domain: 'testing' },
-
-  // Configuration
-  { pattern: /\b(config|setting|env)/i, domain: 'config' },
-
-  // Utilities
-  { pattern: /\b(util|helper|lib|common|shared)/i, domain: 'utilities' },
-
-  // Services/Business Logic
-  { pattern: /\b(service|handler|processor|worker)/i, domain: 'services' },
-
-  // Types/Interfaces
-  { pattern: /\b(type|interface|dto)/i, domain: 'types' },
-]
-
-// ============================================================================
-// FILE CATEGORIZER CLASS
-// ============================================================================
-
-export class FileCategorizer {
-  private batchSize: number
-  private maxDomains: number
-
-  constructor(options: CategorizationOptions = {}) {
-    this.batchSize = options.batchSize || 20
-    this.maxDomains = options.maxDomains || 15
-  }
-
-  // ==========================================================================
-  // MAIN METHODS
-  // ==========================================================================
-
-  /**
-   * Full analysis: discover domains + categorize files
-   */
-  async analyzeProject(
-    projectPath: string,
-    files: ScoredFile[],
-    options: CategorizationOptions = {}
-  ): Promise<CategorizationResult> {
-    const useLLM = options.useLLM !== false
-
-    // Phase 1: Discover domains
-    const domains = useLLM
-      ? await this.discoverDomainsWithLLM(projectPath, files)
-      : this.discoverDomainsHeuristic(files)
-
-    // Phase 2: Categorize files
-    const categories = useLLM
-      ? await this.categorizeFilesWithLLM(files, domains)
-      : this.categorizeFilesHeuristic(files, domains)
-
-    // Update domain file counts
-    for (const domain of domains) {
-      domain.fileCount = categories.filter((c) => c.primaryDomain === domain.name).length
-    }
-
-    // Save to cache if projectId provided
-    if (options.projectId) {
-      await this.saveToCache(options.projectId, domains, categories)
-    }
-
-    return {
-      domains,
-      categories,
-      metrics: {
-        totalFiles: files.length,
-        categorizedFiles: categories.length,
-        domainsDiscovered: domains.length,
-        llmCalls: useLLM ? Math.ceil(files.length / this.batchSize) + 1 : 0,
-        usedHeuristics: !useLLM,
-      },
-    }
-  }
-
-  /**
-   * Discover domains from project structure (LLM)
-   */
-  async discoverDomainsWithLLM(
-    _projectPath: string,
-    files: ScoredFile[]
-  ): Promise<DomainDefinition[]> {
-    // For now, fall back to heuristics
-    // TODO: Implement LLM call when LLM service is available
-    // The prompt would analyze directory structure and file names
-    // to identify functional domains unique to this project
-    return this.discoverDomainsHeuristic(files)
-  }
-
-  /**
-   * Categorize files using discovered domains (LLM)
-   */
-  async categorizeFilesWithLLM(
-    files: ScoredFile[],
-    domains: DomainDefinition[]
-  ): Promise<FileCategory[]> {
-    // For now, fall back to heuristics
-    // TODO: Implement LLM batch processing when LLM service is available
-    // The prompt would ask LLM to categorize batches of files using
-    // the discovered domain definitions
-    return this.categorizeFilesHeuristic(files, domains)
-  }
-
-  // ==========================================================================
-  // HEURISTIC METHODS (Fallback)
-  // ==========================================================================
-
-  /**
-   * Discover domains using heuristics (fallback)
-   */
-  discoverDomainsHeuristic(files: ScoredFile[]): DomainDefinition[] {
-    const domainCounts = new Map<string, number>()
-    const domainPatterns = new Map<string, Set<string>>()
-
-    // Count files matching each heuristic pattern
-    for (const file of files) {
-      const filePath = file.path.toLowerCase()
-      for (const { pattern, domain } of HEURISTIC_PATTERNS) {
-        if (pattern.test(filePath)) {
-          domainCounts.set(domain, (domainCounts.get(domain) || 0) + 1)
-          if (!domainPatterns.has(domain)) {
-            domainPatterns.set(domain, new Set())
-          }
-          // Extract the matched directory as a pattern
-          const dir = path.dirname(file.path)
-          domainPatterns.get(domain)!.add(`**/${path.basename(dir)}/**`)
-        }
-      }
-    }
-
-    // Also detect domains from common directory names
-    const dirDomains = this.extractDirectoryDomains(files)
-    for (const [domain, count] of dirDomains) {
-      domainCounts.set(domain, (domainCounts.get(domain) || 0) + count)
-    }
-
-    // Build domain definitions for domains with at least 2 files
-    const domains: DomainDefinition[] = []
-    for (const [name, count] of domainCounts) {
-      if (count >= 2) {
-        const heuristic = HEURISTIC_PATTERNS.find((h) => h.domain === name)
-        domains.push({
-          name,
-          description: `Files related to ${name}`,
-          keywords: heuristic ? [name] : [name],
-          filePatterns: Array.from(domainPatterns.get(name) || []),
-          fileCount: count,
-        })
-      }
-    }
-
-    // Sort by file count (most files first) and limit
-    return domains.sort((a, b) => b.fileCount - a.fileCount).slice(0, this.maxDomains)
-  }
-
-  /**
-   * Extract domains from directory structure
-   */
-  private extractDirectoryDomains(files: ScoredFile[]): Map<string, number> {
-    const dirCounts = new Map<string, number>()
-
-    for (const file of files) {
-      const parts = file.path.split('/')
-      // Look at immediate parent directories
-      for (const part of parts.slice(0, -1)) {
-        const normalizedDir = part.toLowerCase()
-        // Skip generic directories
-        if (['src', 'lib', 'app', 'core', 'dist', 'build'].includes(normalizedDir)) {
-          continue
-        }
-        // Count meaningful directory names
-        if (normalizedDir.length > 2 && !normalizedDir.startsWith('.')) {
-          dirCounts.set(normalizedDir, (dirCounts.get(normalizedDir) || 0) + 1)
-        }
-      }
-    }
-
-    return dirCounts
-  }
-
-  /**
-   * Categorize files using heuristics (fallback)
-   */
-  categorizeFilesHeuristic(files: ScoredFile[], domains: DomainDefinition[]): FileCategory[] {
-    const categories: FileCategory[] = []
-    const now = getTimestamp()
-
-    for (const file of files) {
-      const matchedDomains: { domain: string; score: number }[] = []
-      const filePath = file.path.toLowerCase()
-
-      // Check against each domain's keywords and patterns
-      for (const domain of domains) {
-        let score = 0
-
-        // Check keywords
-        for (const keyword of domain.keywords) {
-          if (filePath.includes(keyword.toLowerCase())) {
-            score += 1
-          }
-        }
-
-        // Check heuristic patterns
-        for (const { pattern, domain: patternDomain } of HEURISTIC_PATTERNS) {
-          if (patternDomain === domain.name && pattern.test(filePath)) {
-            score += 2
-          }
-        }
-
-        // Check directory patterns
-        for (const pattern of domain.filePatterns) {
-          const patternBase = pattern.replace(/\*\*/g, '').replace(/\//g, '')
-          if (filePath.includes(patternBase.toLowerCase())) {
-            score += 1
-          }
-        }
-
-        if (score > 0) {
-          matchedDomains.push({ domain: domain.name, score })
-        }
-      }
-
-      // Sort by score and take top matches
-      matchedDomains.sort((a, b) => b.score - a.score)
-
-      const fileCategories =
-        matchedDomains.length > 0 ? matchedDomains.slice(0, 3).map((m) => m.domain) : ['general']
-
-      const primaryDomain = fileCategories[0]
-      const confidence = matchedDomains.length > 0 ? Math.min(1, matchedDomains[0].score / 5) : 0.1
-
-      categories.push({
-        path: file.path,
-        categories: fileCategories,
-        primaryDomain,
-        confidence,
-        categorizedAt: now,
-        method: 'heuristic',
-      })
-    }
-
-    return categories
-  }
-
-  // ==========================================================================
-  // CACHING
-  // ==========================================================================
-
-  /**
-   * Save domains and categories to cache
-   */
-  async saveToCache(
-    projectId: string,
-    domains: DomainDefinition[],
-    categories: FileCategory[]
-  ): Promise<void> {
-    const now = getTimestamp()
-
-    // Save domains
-    const domainsData: DiscoveredDomains = {
-      version: INDEX_VERSION,
-      projectId,
-      domains,
-      discoveredAt: now,
-    }
-    await indexStorage.writeDomains(projectId, domainsData)
-
-    // Build domain index (domain -> file paths)
-    const domainIndex: Record<string, string[]> = {}
-    for (const cat of categories) {
-      for (const domain of cat.categories) {
-        if (!domainIndex[domain]) {
-          domainIndex[domain] = []
-        }
-        domainIndex[domain].push(cat.path)
-      }
-    }
-
-    // Save categories
-    const cacheData: CategoriesCache = {
-      version: INDEX_VERSION,
-      lastUpdate: now,
-      fileCategories: categories,
-      domainIndex,
-    }
-    await indexStorage.writeCategories(projectId, cacheData)
-  }
-
-  /**
-   * Load cached categorization
-   */
-  async loadFromCache(projectId: string): Promise<CategorizationResult | null> {
-    const [domainsData, cacheData] = await Promise.all([
-      indexStorage.readDomains(projectId),
-      indexStorage.readCategories(projectId),
-    ])
-
-    if (!domainsData || !cacheData) {
-      return null
-    }
-
-    return {
-      domains: domainsData.domains,
-      categories: cacheData.fileCategories,
-      metrics: {
-        totalFiles: cacheData.fileCategories.length,
-        categorizedFiles: cacheData.fileCategories.length,
-        domainsDiscovered: domainsData.domains.length,
-        llmCalls: 0,
-        usedHeuristics: cacheData.fileCategories[0]?.method === 'heuristic',
-      },
-    }
-  }
-
-  /**
-   * Update categories for specific files (incremental)
-   */
-  async updateFilesCategories(
-    projectId: string,
-    files: ScoredFile[],
-    options: CategorizationOptions = {}
-  ): Promise<FileCategory[]> {
-    // Load existing domains
-    const domainsData = await indexStorage.readDomains(projectId)
-    if (!domainsData) {
-      // No domains yet, need full analysis
-      const result = await this.analyzeProject('', files, { ...options, projectId })
-      return result.categories
-    }
-
-    // Categorize just the new/changed files
-    const newCategories =
-      options.useLLM !== false
-        ? await this.categorizeFilesWithLLM(files, domainsData.domains)
-        : this.categorizeFilesHeuristic(files, domainsData.domains)
-
-    // Load existing cache and merge
-    const existingCache = await indexStorage.readCategories(projectId)
-    if (existingCache) {
-      // Remove old entries for updated files
-      const updatedPaths = new Set(files.map((f) => f.path))
-      const existingCategories = existingCache.fileCategories.filter(
-        (c) => !updatedPaths.has(c.path)
-      )
-
-      // Merge and save
-      const allCategories = [...existingCategories, ...newCategories]
-
-      // Rebuild domain index
-      const domainIndex: Record<string, string[]> = {}
-      for (const cat of allCategories) {
-        for (const domain of cat.categories) {
-          if (!domainIndex[domain]) {
-            domainIndex[domain] = []
-          }
-          domainIndex[domain].push(cat.path)
-        }
-      }
-
-      const cacheData: CategoriesCache = {
-        version: INDEX_VERSION,
-        lastUpdate: getTimestamp(),
-        fileCategories: allCategories,
-        domainIndex,
-      }
-      await indexStorage.writeCategories(projectId, cacheData)
-    }
-
-    return newCategories
-  }
-}
-
-export const fileCategorizer = new FileCategorizer()
-export default FileCategorizer

package/core/services/file-scorer.ts

@@ -1,270 +0,0 @@
-/**
- * FileScorer - Calculates relevance scores for files
- *
- * Scoring factors:
- * - recency: Modified recently? (0-20)
- * - centrality: Imported by many files? (0-25)
- * - configRelevance: Is config file? (0-20)
- * - nameRelevance: Name indicates importance? (0-15)
- * - sizeOptimal: Useful size (not too large)? (0-10)
- * - gitActivity: Recent commits? (0-10)
- *
- * Total score: 0-100
- * Files with score > 30 are considered relevant
- */
-
-import path from 'node:path'
-
-// ============================================================================
-// TYPES
-// ============================================================================
-
-export interface FileScore {
-  path: string
-  score: number
-  factors: {
-    recency: number // 0-20
-    centrality: number // 0-25
-    configRelevance: number // 0-20
-    nameRelevance: number // 0-15
-    sizeOptimal: number // 0-10
-    gitActivity: number // 0-10
-  }
-}
-
-export interface FileStats {
-  path: string
-  size: number // bytes
-  mtime: Date // modification time
-  lines?: number // line count
-  imports?: string[] // files this imports
-  importedBy?: string[] // files that import this
-  recentCommits?: number // commits in last 30 days
-}
-
-export interface ScoringContext {
-  allFiles: Map<string, FileStats>
-  configFiles: Set<string>
-  maxFileSize: number
-  maxRecentCommits: number
-  now: Date
-}
-
-// ============================================================================
-// CONSTANTS
-// ============================================================================
-
-export const RELEVANCE_THRESHOLD = 30
-
-// Config file patterns (high importance)
-const CONFIG_PATTERNS = [
-  /^package\.json$/,
-  /^tsconfig.*\.json$/,
-  /^\.env(\..*)?$/,
-  /^\.eslintrc.*$/,
-  /^\.prettierrc.*$/,
-  /^vite\.config\.\w+$/,
-  /^next\.config\.\w+$/,
-  /^webpack\.config\.\w+$/,
-  /^rollup\.config\.\w+$/,
-  /^jest\.config\.\w+$/,
-  /^vitest\.config\.\w+$/,
-  /^tailwind\.config\.\w+$/,
-  /^postcss\.config\.\w+$/,
-  /^Cargo\.toml$/,
-  /^go\.mod$/,
-  /^pyproject\.toml$/,
-  /^requirements\.txt$/,
-  /^Dockerfile$/,
-  /^docker-compose\.ya?ml$/,
-  /^\.github\/workflows\/.*\.ya?ml$/,
-]
-
-// Important filename patterns
-const IMPORTANT_NAME_PATTERNS = [
-  /^index\.\w+$/, // Entry points
-  /^main\.\w+$/, // Main files
-  /^app\.\w+$/, // App files
-  /^server\.\w+$/, // Server files
-  /^router\.\w+$/, // Router files
-  /^routes\.\w+$/, // Routes
-  /^api\.\w+$/, // API files
-  /^schema\.\w+$/, // Schema files
-  /^types?\.\w+$/, // Type definitions
-  /^constants?\.\w+$/, // Constants
-  /^config\.\w+$/, // Config files
-  /^utils?\.\w+$/, // Utilities
-  /^helpers?\.\w+$/, // Helpers
-  /README\.md$/i, // Documentation
-  /CHANGELOG\.md$/i, // Changelog
-]
-
-// ============================================================================
-// FILE SCORER
-// ============================================================================
-
-export class FileScorer {
-  /**
-   * Score a single file
-   */
-  scoreFile(stats: FileStats, context: ScoringContext): FileScore {
-    const factors = {
-      recency: this.calculateRecency(stats, context),
-      centrality: this.calculateCentrality(stats, context),
-      configRelevance: this.calculateConfigRelevance(stats),
-      nameRelevance: this.calculateNameRelevance(stats),
-      sizeOptimal: this.calculateSizeOptimal(stats, context),
-      gitActivity: this.calculateGitActivity(stats, context),
-    }
-
-    const score = Object.values(factors).reduce((sum, v) => sum + v, 0)
-
-    return {
-      path: stats.path,
-      score: Math.min(100, Math.max(0, score)),
-      factors,
-    }
-  }
-
-  /**
-   * Score all files and return sorted by score
-   */
-  scoreAll(context: ScoringContext): FileScore[] {
-    const scores: FileScore[] = []
-
-    for (const stats of context.allFiles.values()) {
-      scores.push(this.scoreFile(stats, context))
-    }
-
-    return scores.sort((a, b) => b.score - a.score)
-  }
-
-  /**
-   * Get relevant files (score > threshold)
-   */
-  getRelevantFiles(context: ScoringContext, threshold: number = RELEVANCE_THRESHOLD): FileScore[] {
-    return this.scoreAll(context).filter((f) => f.score >= threshold)
-  }
-
-  // ==========================================================================
-  // FACTOR CALCULATIONS
-  // ==========================================================================
-
-  /**
-   * Recency factor (0-20)
-   * Files modified recently get higher scores
-   */
-  private calculateRecency(stats: FileStats, context: ScoringContext): number {
-    const daysSinceModified =
-      (context.now.getTime() - stats.mtime.getTime()) / (1000 * 60 * 60 * 24)
-
-    if (daysSinceModified < 1) return 20 // Today
-    if (daysSinceModified < 7) return 15 // This week
-    if (daysSinceModified < 30) return 10 // This month
-    if (daysSinceModified < 90) return 5 // Last 3 months
-    return 0 // Older
-  }
-
-  /**
-   * Centrality factor (0-25)
-   * Files imported by many others are more important
-   */
-  private calculateCentrality(stats: FileStats, context: ScoringContext): number {
-    const importedByCount = stats.importedBy?.length || 0
-    const totalFiles = context.allFiles.size
-
-    if (totalFiles === 0) return 0
-
-    // Normalize: files imported by 20%+ of codebase get max score
-    const ratio = importedByCount / totalFiles
-    if (ratio >= 0.2) return 25
-    if (ratio >= 0.1) return 20
-    if (ratio >= 0.05) return 15
-    if (importedByCount >= 5) return 10
-    if (importedByCount >= 2) return 5
-    return 0
-  }
-
-  /**
-   * Config relevance factor (0-20)
-   * Config files are always important
-   */
-  private calculateConfigRelevance(stats: FileStats): number {
-    const filename = path.basename(stats.path)
-
-    for (const pattern of CONFIG_PATTERNS) {
-      if (pattern.test(filename) || pattern.test(stats.path)) {
-        return 20
-      }
-    }
-
-    return 0
-  }
-
-  /**
-   * Name relevance factor (0-15)
-   * Certain filenames indicate importance
-   */
-  private calculateNameRelevance(stats: FileStats): number {
-    const filename = path.basename(stats.path)
-
-    for (const pattern of IMPORTANT_NAME_PATTERNS) {
-      if (pattern.test(filename)) {
-        return 15
-      }
-    }
-
-    // Directories that suggest importance
-    const dir = path.dirname(stats.path)
-    if (dir.includes('/api/') || dir.includes('/routes/')) return 10
-    if (dir.includes('/components/') && filename.startsWith('index')) return 10
-    if (dir.includes('/pages/') || dir.includes('/app/')) return 8
-
-    return 0
-  }
-
-  /**
-   * Size optimal factor (0-10)
-   * Files that are neither too small nor too large
-   */
-  private calculateSizeOptimal(stats: FileStats, _context: ScoringContext): number {
-    const size = stats.size
-
-    // Too small (likely stub or barrel export)
-    if (size < 100) return 2
-
-    // Optimal range: 500 bytes to 50KB
-    if (size >= 500 && size <= 50_000) return 10
-
-    // Large but not huge (50KB - 200KB)
-    if (size > 50_000 && size <= 200_000) return 5
-
-    // Very large files are less useful as context
-    if (size > 200_000) return 0
-
-    // Small files (100-500 bytes)
-    return 5
-  }
-
-  /**
-   * Git activity factor (0-10)
-   * Files with recent commits are more actively developed
-   */
-  private calculateGitActivity(stats: FileStats, context: ScoringContext): number {
-    const commits = stats.recentCommits || 0
-
-    if (context.maxRecentCommits === 0) return 0
-
-    // Normalize against max
-    const ratio = commits / context.maxRecentCommits
-
-    if (ratio >= 0.5) return 10 // Among most active
-    if (ratio >= 0.25) return 7
-    if (ratio >= 0.1) return 5
-    if (commits > 0) return 2
-    return 0
-  }
-}
-
-export const fileScorer = new FileScorer()
-export default FileScorer