sdlc-agent-4-enterprise-server 1.0.0

package/src/engine/parsers/tree-sitter-indexer.ts

@@ -0,0 +1,263 @@
+/**
+ * KSA-145: Tree-sitter Indexer — Orchestrates file parsing and database storage.
+ * Uses tree-sitter for supported languages, falls back to regex extraction.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import Database from 'better-sqlite3';
+import { GrammarRegistry } from './grammar-registry.js';
+import { extractSymbols } from '../scanner/signature-extractor.js';
+import type { ParseResult, IndexResult } from './types.js';
+
+export class TreeSitterIndexer {
+  private registry: GrammarRegistry;
+  private db: Database.Database;
+  private maxFileSize: number;
+
+  constructor(registry: GrammarRegistry, db: Database.Database, maxFileSize: number = 1_048_576) {
+    this.registry = registry;
+    this.db = db;
+    this.maxFileSize = maxFileSize;
+  }
+
+  /** Index a single file using tree-sitter or regex fallback. */
+  async indexFile(filePath: string, relativePath: string): Promise<IndexResult> {
+    const startTime = Date.now();
+
+    let source: string;
+    try {
+      const stat = fs.statSync(filePath);
+      if (stat.size > this.maxFileSize) {
+        return this.regexFallback(filePath, relativePath, startTime);
+      }
+      source = fs.readFileSync(filePath, 'utf-8');
+    } catch {
+      return {
+        filePath: relativePath,
+        symbolCount: 0,
+        relationshipCount: 0,
+        parseErrors: 1,
+        duration: Date.now() - startTime,
+        method: 'regex-fallback',
+      };
+    }
+
+    // Try tree-sitter first
+    const parser = await this.registry.getParser(filePath);
+    let result: ParseResult;
+    let method: 'tree-sitter' | 'regex-fallback';
+
+    if (parser) {
+      result = parser.parse(source, relativePath);
+      method = 'tree-sitter';
+    } else {
+      return this.regexFallback(filePath, relativePath, startTime);
+    }
+
+    // Atomic database update
+    const symbolIds = this.storeResults(relativePath, result);
+
+    // Extract and store function bodies for embedding-based similarity (KSA-169)
+    this.extractAndStoreBodies(relativePath, source, result, symbolIds);
+
+    return {
+      filePath: relativePath,
+      symbolCount: result.symbols.length,
+      relationshipCount: result.relationships.length,
+      parseErrors: result.errors.length,
+      duration: Date.now() - startTime,
+      method,
+    };
+  }
+
+  /** Batch index multiple files. */
+  async indexFiles(files: { absolutePath: string; relativePath: string }[]): Promise<IndexResult[]> {
+    const results: IndexResult[] = [];
+    for (const file of files) {
+      const result = await this.indexFile(file.absolutePath, file.relativePath);
+      results.push(result);
+    }
+    return results;
+  }
+
+  /** Store parse results in the database atomically. */
+  private storeResults(filePath: string, result: ParseResult): Map<string, number> {
+    const symbolIds = new Map<string, number>();
+    const transaction = this.db.transaction(() => {
+      // Get file_id
+      const fileRow = this.db.prepare(
+        'SELECT id FROM files WHERE relative_path = ?'
+      ).get(filePath) as { id: number } | undefined;
+
+      if (!fileRow) return;
+      const fileId = fileRow.id;
+
+      // Delete old symbols
+      this.db.prepare('DELETE FROM symbols WHERE file_id = ?').run(fileId);
+
+      // Delete old relationships for this file
+      try {
+        this.db.prepare('DELETE FROM relationships WHERE file_path = ?').run(filePath);
+      } catch {
+        // relationships table may not exist yet (pre-migration)
+      }
+
+      // Insert new symbols
+      const insertSym = this.db.prepare(`
+        INSERT INTO symbols (file_id, name, kind, signature, start_line, end_line,
+          parent_symbol, visibility, doc_comment)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+
+      for (const sym of result.symbols) {
+        const info = insertSym.run(
+          fileId, sym.name, sym.kind, sym.signature,
+          sym.startLine, sym.endLine, sym.parentName ?? null,
+          sym.isExported ? 'export' : null, sym.docComment ?? null
+        );
+        symbolIds.set(sym.name, info.lastInsertRowid as number);
+      }
+
+      // Insert relationships (if table exists)
+      try {
+        const insertRel = this.db.prepare(`
+          INSERT INTO relationships (source_symbol_id, target_symbol, target_symbol_id, kind, file_path, line, metadata)
+          VALUES (?, ?, ?, ?, ?, ?, ?)
+        `);
+
+        for (const rel of result.relationships) {
+          const sourceId = symbolIds.get(rel.sourceSymbol);
+          if (!sourceId) continue;
+
+          const targetId = symbolIds.get(rel.targetSymbol) ?? null;
+          insertRel.run(
+            sourceId,
+            rel.targetSymbol,
+            targetId,
+            rel.kind,
+            filePath,
+            rel.line,
+            rel.metadata ? JSON.stringify(rel.metadata) : null
+          );
+        }
+      } catch {
+        // relationships table may not exist yet
+      }
+    });
+
+    transaction();
+    return symbolIds;
+  }
+
+  private regexFallback(
+    filePath: string, relativePath: string, startTime: number
+  ): IndexResult {
+    try {
+      const source = fs.readFileSync(filePath, 'utf-8');
+      const ext = path.extname(filePath).toLowerCase();
+      const language = this.extToLanguage(ext);
+      const symbols = extractSymbols(source, language);
+
+      // Store regex-extracted symbols in the database
+      if (symbols.length > 0) {
+        this.storeRegexResults(relativePath, symbols);
+      }
+
+      return {
+        filePath: relativePath,
+        symbolCount: symbols.length,
+        relationshipCount: 0,
+        parseErrors: 0,
+        duration: Date.now() - startTime,
+        method: 'regex-fallback',
+      };
+    } catch {
+      return {
+        filePath: relativePath,
+        symbolCount: 0,
+        relationshipCount: 0,
+        parseErrors: 1,
+        duration: Date.now() - startTime,
+        method: 'regex-fallback',
+      };
+    }
+  }
+
+  /** Store regex-extracted symbols in the database. */
+  private storeRegexResults(filePath: string, symbols: ReturnType<typeof extractSymbols>): void {
+    const transaction = this.db.transaction(() => {
+      const fileRow = this.db.prepare(
+        'SELECT id FROM files WHERE relative_path = ?'
+      ).get(filePath) as { id: number } | undefined;
+
+      if (!fileRow) return;
+      const fileId = fileRow.id;
+
+      // Delete old symbols
+      this.db.prepare('DELETE FROM symbols WHERE file_id = ?').run(fileId);
+
+      // Insert new symbols
+      const insertSym = this.db.prepare(`
+        INSERT INTO symbols (file_id, name, kind, signature, start_line, end_line,
+          parent_symbol, visibility, doc_comment)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+
+      for (const sym of symbols) {
+        insertSym.run(
+          fileId, sym.name, sym.kind, sym.signature,
+          sym.startLine, sym.endLine, sym.parentSymbol,
+          sym.visibility, sym.docComment
+        );
+      }
+    });
+
+    transaction();
+  }
+
+  private extToLanguage(ext: string): string {
+    const map: Record<string, string> = {
+      '.ts': 'typescript', '.tsx': 'typescript',
+      '.js': 'javascript', '.jsx': 'javascript',
+      '.py': 'python', '.kt': 'kotlin', '.kts': 'kotlin',
+      '.java': 'java', '.go': 'go', '.rs': 'rust',
+      '.cls': 'apex', '.trigger': 'apex',
+    };
+    return map[ext] ?? 'generic';
+  }
+
+  /** Extract and store function body text for embedding-based similarity (KSA-169). */
+  private extractAndStoreBodies(
+    filePath: string, source: string, result: ParseResult, symbolIds: Map<string, number>
+  ): void {
+    try {
+      const lines = source.split('\n');
+      const functionKinds = new Set(['function', 'method', 'arrow_function', 'generator', 'function_declaration']);
+      const minBodyLines = 3;
+
+      const insertBody = this.db.prepare(`
+        INSERT OR REPLACE INTO body_embeddings (symbol_id, chunk_index, embedding, token_count)
+        VALUES (?, ?, ?, ?)
+      `);
+
+      for (const sym of result.symbols) {
+        if (!functionKinds.has(sym.kind)) continue;
+        const symbolId = symbolIds.get(sym.name);
+        if (!symbolId) continue;
+
+        const bodyLines = lines.slice(sym.startLine - 1, sym.endLine);
+        if (bodyLines.length < minBodyLines) continue;
+
+        const bodyText = bodyLines.join('\n');
+        const tokenCount = bodyText.split(/\s+/).filter(Boolean).length;
+
+        // Store body text as raw bytes (embedding generated later by embedding service)
+        const textBuffer = Buffer.from(bodyText, 'utf-8');
+        insertBody.run(symbolId, 0, textBuffer, tokenCount);
+      }
+    } catch {
+      // Body extraction is optional — don't fail indexing
+    }
+  }
+}

package/src/engine/parsers/types.ts

@@ -0,0 +1,78 @@
+/**
+ * KSA-145: Tree-sitter Core Integration — Type definitions.
+ * Shared interfaces for language parsers, parse results, and extracted symbols.
+ */
+
+// web-tree-sitter exports Node class directly from the module
+// We re-export it as SyntaxNode for clarity
+export type { Node as SyntaxNode } from 'web-tree-sitter';
+
+export type SymbolKind =
+  | 'function' | 'class' | 'method' | 'interface'
+  | 'type' | 'enum' | 'variable' | 'namespace'
+  | 'constructor' | 'property' | 'module'
+  | 'trait' | 'struct' | 'constant';
+
+export type RelationshipKind =
+  | 'calls' | 'imports' | 'inherits'
+  | 'implements' | 'uses' | 'decorates'
+  | 'dml' | 'soql' | 'trigger-on' | 'wire' | 'apex-import';
+
+export interface ExtractedSymbol {
+  name: string;
+  kind: SymbolKind;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  signature: string;
+  parameters?: string | null;
+  returnType?: string | null;
+  modifiers?: string[];
+  decorators?: string[];
+  parentName?: string | null;
+  isAsync?: boolean;
+  isExported?: boolean;
+  docComment?: string | null;
+  complexity?: number;
+}
+
+export interface ExtractedRelationship {
+  sourceSymbol: string;
+  targetSymbol: string;
+  kind: RelationshipKind;
+  filePath?: string;
+  line: number;
+  metadata?: Record<string, unknown>;
+}
+
+export interface ParseResult {
+  symbols: ExtractedSymbol[];
+  relationships: ExtractedRelationship[];
+  errors: ParseError[];
+}
+
+export interface ParseError {
+  message: string;
+  line: number;
+  column: number;
+}
+
+export interface ILanguageParser {
+  readonly languageId: string;
+  parse(source: string, filePath: string): ParseResult;
+  getSupportedExtensions(): string[];
+}
+
+export interface IndexResult {
+  filePath: string;
+  symbolCount: number;
+  relationshipCount: number;
+  parseErrors: number;
+  duration: number;
+  method: 'tree-sitter' | 'regex-fallback';
+}
+
+export interface NodeVisitor {
+  enter?(node: import('web-tree-sitter').Node): boolean | void;
+  leave?(node: import('web-tree-sitter').Node): void;
+}

package/src/engine/query/query-layer.ts

@@ -0,0 +1,177 @@
+/**
+ * Query Layer — FTS5 search, symbol lookup, module listing.
+ * Provides the data access layer for all MCP tool handlers.
+ */
+
+import Database from 'better-sqlite3';
+import { DatabaseManager } from '../db/database-manager.js';
+
+export interface SearchResult {
+  name: string;
+  kind: string;
+  signature: string;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  docComment: string | null;
+  rank: number;
+}
+
+export interface SymbolInfo {
+  name: string;
+  kind: string;
+  signature: string;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  visibility: string | null;
+  docComment: string | null;
+  parentSymbol: string | null;
+}
+
+export interface ModuleInfo {
+  name: string;
+  rootPath: string;
+  language: string | null;
+  description: string | null;
+  fileCount: number;
+  symbolCount: number;
+  diStyle: string | null;
+  errorHandling: string | null;
+  namingConvention: string | null;
+  loggingFramework: string | null;
+  testingFramework: string | null;
+  purpose: string | null;
+}
+
+export interface IndexStatus {
+  totalFiles: number;
+  totalSymbols: number;
+  totalModules: number;
+  languages: Record<string, number>;
+  lastIndexed: string | null;
+}
+
+export class QueryLayer {
+  private db: Database.Database;
+
+  constructor(dbManager: DatabaseManager) {
+    this.db = dbManager.getDb();
+  }
+
+  /** Full-text search across symbols using FTS5. */
+  searchCode(query: string, limit: number = 20): SearchResult[] {
+    const ftsQuery = sanitizeFtsQuery(query);
+    const stmt = this.db.prepare(`
+      SELECT s.name, s.kind, s.signature, f.relative_path as filePath,
+             s.start_line as startLine, s.end_line as endLine,
+             s.doc_comment as docComment, rank
+      FROM symbols_fts
+      JOIN symbols s ON symbols_fts.rowid = s.id
+      JOIN files f ON s.file_id = f.id
+      WHERE symbols_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+    return stmt.all(ftsQuery, limit) as SearchResult[];
+  }
+
+  /** Lookup symbols by exact name or prefix. */
+  findSymbols(name: string, kind?: string, limit: number = 50): SymbolInfo[] {
+    let sql = `
+      SELECT s.name, s.kind, s.signature, f.relative_path as filePath,
+             s.start_line as startLine, s.end_line as endLine,
+             s.visibility, s.doc_comment as docComment, s.parent_symbol as parentSymbol
+      FROM symbols s
+      JOIN files f ON s.file_id = f.id
+      WHERE s.name LIKE ?
+    `;
+    const params: any[] = [`${name}%`];
+
+    if (kind) {
+      sql += ' AND s.kind = ?';
+      params.push(kind);
+    }
+    sql += ' ORDER BY s.name LIMIT ?';
+    params.push(limit);
+
+    return this.db.prepare(sql).all(...params) as SymbolInfo[];
+  }
+
+  /** Get symbols in a specific file. */
+  getFileSymbols(relativePath: string): SymbolInfo[] {
+    const stmt = this.db.prepare(`
+      SELECT s.name, s.kind, s.signature, f.relative_path as filePath,
+             s.start_line as startLine, s.end_line as endLine,
+             s.visibility, s.doc_comment as docComment, s.parent_symbol as parentSymbol
+      FROM symbols s
+      JOIN files f ON s.file_id = f.id
+      WHERE f.relative_path = ?
+      ORDER BY s.start_line
+    `);
+    return stmt.all(relativePath) as SymbolInfo[];
+  }
+
+  /** List all modules with stats and pattern metadata. */
+  listModules(): ModuleInfo[] {
+    const stmt = this.db.prepare(`
+      SELECT name, root_path as rootPath, language, description,
+             file_count as fileCount, symbol_count as symbolCount,
+             di_style as diStyle, error_handling as errorHandling,
+             naming_convention as namingConvention,
+             logging_framework as loggingFramework,
+             testing_framework as testingFramework, purpose
+      FROM modules
+      ORDER BY name
+    `);
+    return stmt.all() as ModuleInfo[];
+  }
+
+  /** List modules with pattern metadata, optionally filtered by name. */
+  listModulesWithPatterns(name: string | null): ModuleInfo[] {
+    if (!name) return this.listModules();
+    const stmt = this.db.prepare(`
+      SELECT name, root_path as rootPath, language, description,
+             file_count as fileCount, symbol_count as symbolCount,
+             di_style as diStyle, error_handling as errorHandling,
+             naming_convention as namingConvention,
+             logging_framework as loggingFramework,
+             testing_framework as testingFramework, purpose
+      FROM modules
+      WHERE name LIKE ?
+      ORDER BY name
+    `);
+    return stmt.all(`${name}%`) as ModuleInfo[];
+  }
+
+  /** Get index status and statistics. */
+  getIndexStatus(): IndexStatus {
+    const files = this.db.prepare('SELECT COUNT(*) as c FROM files').get() as { c: number };
+    const symbols = this.db.prepare('SELECT COUNT(*) as c FROM symbols').get() as { c: number };
+    const modules = this.db.prepare('SELECT COUNT(*) as c FROM modules').get() as { c: number };
+    const lastRow = this.db.prepare(
+      'SELECT MAX(last_indexed) as t FROM files'
+    ).get() as { t: string | null };
+
+    const langRows = this.db.prepare(
+      'SELECT language, COUNT(*) as c FROM files GROUP BY language'
+    ).all() as { language: string; c: number }[];
+
+    const languages: Record<string, number> = {};
+    for (const row of langRows) {
+      languages[row.language] = row.c;
+    }
+
+    return {
+      totalFiles: files.c,
+      totalSymbols: symbols.c,
+      totalModules: modules.c,
+      languages,
+      lastIndexed: lastRow.t,
+    };
+  }
+}
+
+function sanitizeFtsQuery(query: string): string {
+  return query.replace(/[^\w\s*"]/g, ' ').trim() || '*';
+}

package/src/engine/scanner/file-scanner.ts

@@ -0,0 +1,173 @@
+/**
+ * File Scanner — Traverses workspace, respects .gitignore, detects language.
+ * Produces a list of scannable files with metadata.
+ * KSA-191: Added Salesforce extensions + compound extension detection.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { AppConfig } from '../config.js';
+import { createIgnoreParser, IgnoreParser } from '../parsers/ignore/index.js';
+
+export interface ScannedFile {
+  absolutePath: string;
+  relativePath: string;
+  language: string;
+  contentHash: string;
+  sizeBytes: number;
+  lineCount: number;
+}
+
+const EXTENSION_LANGUAGE_MAP: Record<string, string> = {
+  '.ts': 'typescript', '.tsx': 'typescript',
+  '.js': 'javascript', '.jsx': 'javascript',
+  '.kt': 'kotlin', '.kts': 'kotlin',
+  '.java': 'java',
+  '.py': 'python',
+  '.go': 'go',
+  '.rs': 'rust',
+  '.c': 'c', '.h': 'c',
+  '.cpp': 'cpp', '.hpp': 'cpp',
+  '.cs': 'csharp',
+  '.rb': 'ruby',
+  '.php': 'php',
+  '.swift': 'swift',
+  '.scala': 'scala',
+  '.sql': 'sql',
+  '.sh': 'bash',
+  '.ps1': 'powershell',
+  '.yaml': 'yaml', '.yml': 'yaml',
+  '.json': 'json',
+  '.toml': 'toml',
+  '.cls': 'apex',
+  '.trigger': 'apex',
+};
+
+/** Scan workspace and return list of indexable files. */
+export function scanWorkspace(config: AppConfig): ScannedFile[] {
+  const results: ScannedFile[] = [];
+  const ignoreParser = createIgnoreParser(config.workspace);
+  traverseDirectory(config.workspace, config, ignoreParser, results);
+  return results;
+}
+
+/** Scan a single file and return metadata. */
+export function scanSingleFile(filePath: string, workspace: string): ScannedFile | null {
+  try {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const relativePath = path.relative(workspace, filePath).replace(/\\/g, '/');
+    const language = detectLanguage(filePath);
+    if (!language) return null;
+
+    return {
+      absolutePath: filePath,
+      relativePath,
+      language,
+      contentHash: hashContent(content),
+      sizeBytes: Buffer.byteLength(content, 'utf-8'),
+      lineCount: content.split('\n').length,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Detect language from file extension — supports compound extensions. */
+export function detectLanguage(filePath: string): string | null {
+  // Check compound extensions first (Salesforce metadata)
+  const lowerPath = filePath.toLowerCase().replace(/\\/g, '/');
+  if (lowerPath.endsWith('.flow-meta.xml') ||
+      lowerPath.endsWith('.object-meta.xml') ||
+      lowerPath.endsWith('.field-meta.xml') ||
+      lowerPath.endsWith('.js-meta.xml') ||
+      lowerPath.endsWith('.component-meta.xml')) {
+    return 'salesforce-meta';
+  }
+
+  const ext = getExtension(filePath);
+  return EXTENSION_LANGUAGE_MAP[ext] ?? null;
+}
+
+function getExtension(filePath: string): string {
+  if (filePath.endsWith('.gradle.kts')) return '.kts';
+  return path.extname(filePath).toLowerCase();
+}
+
+function hashContent(content: string): string {
+  return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+
+function traverseDirectory(
+  dir: string,
+  config: AppConfig,
+  ignoreParser: IgnoreParser,
+  results: ScannedFile[]
+): void {
+  const entries = safeReadDir(dir);
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    const relPath = path.relative(config.workspace, fullPath).replace(/\\/g, '/');
+
+    if (shouldExclude(relPath, entry.name, config.excludePatterns, ignoreParser)) continue;
+
+    if (entry.isDirectory()) {
+      traverseDirectory(fullPath, config, ignoreParser, results);
+    } else if (entry.isFile()) {
+      const file = processFile(fullPath, relPath, config);
+      if (file) results.push(file);
+    }
+  }
+}
+
+function processFile(fullPath: string, relPath: string, config: AppConfig): ScannedFile | null {
+  const language = detectLanguage(fullPath);
+  if (!language) return null;
+
+  const ext = getExtension(fullPath);
+  // Allow through if simple extension matches OR if compound extension detected (salesforce-meta)
+  if (!config.includeExtensions.includes(ext) && ext !== '.kts' && language !== 'salesforce-meta') return null;
+
+  try {
+    const stat = fs.statSync(fullPath);
+    if (stat.size > config.maxFileSize) return null;
+
+    const content = fs.readFileSync(fullPath, 'utf-8');
+    if (isBinary(content)) return null;
+
+    return {
+      absolutePath: fullPath,
+      relativePath: relPath,
+      language,
+      contentHash: hashContent(content),
+      sizeBytes: stat.size,
+      lineCount: content.split('\n').length,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function shouldExclude(
+  relPath: string, name: string, excludes: string[], ignoreParser: IgnoreParser
+): boolean {
+  if (name.startsWith('.') && name !== '.') return true;
+  for (const pattern of excludes) {
+    if (relPath.includes(pattern) || name === pattern) return true;
+  }
+  return ignoreParser.shouldIgnore(relPath);
+}
+
+function isBinary(content: string): boolean {
+  const sample = content.slice(0, 1024);
+  const nullCount = (sample.match(/\0/g) || []).length;
+  return nullCount > 2;
+}
+
+function safeReadDir(dir: string): fs.Dirent[] {
+  try {
+    return fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}