viberag 0.1.0

package/dist/rag/search/hybrid.js

@@ -0,0 +1,58 @@
+/**
+ * Hybrid search combining vector and FTS with RRF reranking.
+ */
+/**
+ * Reciprocal Rank Fusion (RRF) constant.
+ * Higher values give more weight to lower-ranked results.
+ */
+const RRF_K = 60;
+/**
+ * Combine vector and FTS results using Reciprocal Rank Fusion.
+ *
+ * RRF formula: score = sum(1 / (k + rank))
+ * where k is a constant (typically 60) and rank is 1-indexed.
+ *
+ * @param vectorResults - Results from vector search
+ * @param ftsResults - Results from FTS search
+ * @param limit - Maximum number of results to return
+ * @param vectorWeight - Weight for vector results (0.0-1.0, default 0.7)
+ * @returns Combined and reranked results
+ */
+export function hybridRerank(vectorResults, ftsResults, limit, vectorWeight = 0.7) {
+    const ftsWeight = 1 - vectorWeight;
+    const scores = new Map();
+    const resultMap = new Map();
+    const vectorScores = new Map();
+    const ftsScoresMap = new Map();
+    // Score from vector results
+    vectorResults.forEach((result, rank) => {
+        const rrfScore = vectorWeight * (1 / (RRF_K + rank + 1));
+        scores.set(result.id, (scores.get(result.id) ?? 0) + rrfScore);
+        resultMap.set(result.id, result);
+        vectorScores.set(result.id, result.vectorScore ?? result.score);
+    });
+    // Score from FTS results
+    ftsResults.forEach((result, rank) => {
+        const rrfScore = ftsWeight * (1 / (RRF_K + rank + 1));
+        scores.set(result.id, (scores.get(result.id) ?? 0) + rrfScore);
+        ftsScoresMap.set(result.id, result.ftsScore ?? result.score);
+        // Keep the result with more info (prefer vector result if exists)
+        if (!resultMap.has(result.id)) {
+            resultMap.set(result.id, result);
+        }
+    });
+    // Sort by combined RRF score and take top results
+    const sortedIds = [...scores.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([id]) => id);
+    return sortedIds.map(id => {
+        const result = resultMap.get(id);
+        return {
+            ...result,
+            score: scores.get(id),
+            vectorScore: vectorScores.get(id),
+            ftsScore: ftsScoresMap.get(id),
+        };
+    });
+}

package/dist/rag/search/index.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Search module for code search.
+ *
+ * Supports multiple search modes:
+ * - semantic: Dense vector search for conceptual queries
+ * - exact: BM25/FTS for symbol names and exact matches
+ * - hybrid: Combined vector + BM25 with RRF (default)
+ * - definition: Direct metadata lookup for symbol definitions
+ * - similar: Vector search with code snippet as query
+ */
+import type { Logger } from '../logger/index.js';
+import type { SearchOptions, SearchResults } from './types.js';
+export type { SearchDebugInfo, SearchFilters, SearchMode, SearchOptions, SearchResult, SearchResults, } from './types.js';
+export { vectorSearch } from './vector.js';
+export { ftsSearch, ensureFtsIndex } from './fts.js';
+export { hybridRerank } from './hybrid.js';
+/**
+ * Search engine for code search.
+ * Supports vector, FTS, hybrid, definition, and similar search modes.
+ */
+export declare class SearchEngine {
+    private readonly projectRoot;
+    private storage;
+    private embeddings;
+    private logger;
+    private initialized;
+    constructor(projectRoot: string, logger?: Logger);
+    /**
+     * Primary search method. Dispatches to appropriate search mode.
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResults>;
+    /**
+     * Semantic search: Dense vector search only.
+     * Best for conceptual queries like "how does auth work?"
+     */
+    private searchSemantic;
+    /**
+     * Exact search: BM25/FTS only.
+     * Best for symbol names and exact string matches.
+     */
+    private searchExact;
+    /**
+     * Hybrid search: Vector + BM25 with RRF reranking.
+     * Good general-purpose search.
+     *
+     * @param autoBoost - When true, increase BM25 weight and oversample if vector scores are low
+     * @param autoBoostThreshold - Vector score threshold below which auto-boost activates
+     * @param returnDebug - Include debug info in results for AI evaluation
+     */
+    private searchHybrid;
+    /**
+     * Definition search: Direct metadata lookup.
+     * Best for "where is X defined?" queries.
+     */
+    private searchDefinition;
+    /**
+     * Similar search: Vector search with code snippet as query.
+     * Best for "find code like this" queries.
+     */
+    private searchSimilar;
+    /**
+     * Perform vector-only search. (Legacy method)
+     */
+    searchVector(query: string, limit?: number): Promise<SearchResults>;
+    /**
+     * Perform FTS-only search. (Legacy method)
+     */
+    searchFts(query: string, limit?: number): Promise<SearchResults>;
+    /**
+     * Initialize the search engine.
+     */
+    private ensureInitialized;
+    /**
+     * Create the appropriate embedding provider based on config.
+     */
+    private createEmbeddingProvider;
+    /**
+     * Get the code chunks table.
+     */
+    private getTable;
+    /**
+     * Log a message.
+     */
+    private log;
+    /**
+     * Close the search engine and free resources.
+     */
+    close(): void;
+}

package/dist/rag/search/index.js

@@ -0,0 +1,367 @@
+/**
+ * Search module for code search.
+ *
+ * Supports multiple search modes:
+ * - semantic: Dense vector search for conceptual queries
+ * - exact: BM25/FTS for symbol names and exact matches
+ * - hybrid: Combined vector + BM25 with RRF (default)
+ * - definition: Direct metadata lookup for symbol definitions
+ * - similar: Vector search with code snippet as query
+ */
+import { loadConfig } from '../config/index.js';
+import { GeminiEmbeddingProvider, Local4BEmbeddingProvider, LocalEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, } from '../embeddings/index.js';
+import { Storage } from '../storage/index.js';
+import { buildDefinitionFilter, buildFilterClause } from './filters.js';
+import { ftsSearch } from './fts.js';
+import { hybridRerank } from './hybrid.js';
+import { vectorSearch } from './vector.js';
+export { vectorSearch } from './vector.js';
+export { ftsSearch, ensureFtsIndex } from './fts.js';
+export { hybridRerank } from './hybrid.js';
+/** Default search limit */
+const DEFAULT_LIMIT = 10;
+/** Exhaustive mode limit (high but bounded) */
+const EXHAUSTIVE_LIMIT = 500;
+/** Default BM25 weight for hybrid search */
+const DEFAULT_BM25_WEIGHT = 0.3;
+/** Default oversample multiplier for hybrid search */
+const DEFAULT_OVERSAMPLE_MULTIPLIER = 2;
+/** Maximum oversample multiplier (for low vector confidence) */
+const MAX_OVERSAMPLE_MULTIPLIER = 4;
+/**
+ * Search engine for code search.
+ * Supports vector, FTS, hybrid, definition, and similar search modes.
+ */
+export class SearchEngine {
+    constructor(projectRoot, logger) {
+        Object.defineProperty(this, "projectRoot", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "storage", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "embeddings", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.projectRoot = projectRoot;
+        this.logger = logger ?? null;
+    }
+    /**
+     * Primary search method. Dispatches to appropriate search mode.
+     */
+    async search(query, options = {}) {
+        const start = Date.now();
+        const mode = options.mode ?? 'hybrid';
+        const limit = options.exhaustive
+            ? EXHAUSTIVE_LIMIT
+            : (options.limit ?? DEFAULT_LIMIT);
+        const filterClause = buildFilterClause(options.filters);
+        await this.ensureInitialized();
+        const table = await this.getTable();
+        let results;
+        switch (mode) {
+            case 'semantic':
+                results = await this.searchSemantic(table, query, limit, filterClause, options.minScore);
+                break;
+            case 'exact':
+                results = await this.searchExact(table, query, limit, filterClause, options.minScore);
+                break;
+            case 'definition':
+                results = await this.searchDefinition(table, options.symbolName ?? query, limit, options.filters?.type, filterClause);
+                break;
+            case 'similar':
+                results = await this.searchSimilar(table, options.codeSnippet ?? query, limit, filterClause, options.minScore);
+                break;
+            case 'hybrid':
+            default:
+                results = await this.searchHybrid(table, query, limit, options.bm25Weight ?? DEFAULT_BM25_WEIGHT, filterClause, options.minScore, options.autoBoost ?? true, options.autoBoostThreshold ?? 0.3, options.returnDebug ?? false);
+                break;
+        }
+        // Add total matches for exhaustive mode
+        if (options.exhaustive) {
+            results.totalMatches = results.results.length;
+        }
+        results.elapsedMs = Date.now() - start;
+        return results;
+    }
+    /**
+     * Semantic search: Dense vector search only.
+     * Best for conceptual queries like "how does auth work?"
+     */
+    async searchSemantic(table, query, limit, filterClause, minScore) {
+        const queryVector = await this.embeddings.embedSingle(query);
+        const results = await vectorSearch(table, queryVector, {
+            limit,
+            filterClause,
+            minScore,
+        });
+        return {
+            results,
+            query,
+            searchType: 'semantic',
+            elapsedMs: 0,
+        };
+    }
+    /**
+     * Exact search: BM25/FTS only.
+     * Best for symbol names and exact string matches.
+     */
+    async searchExact(table, query, limit, filterClause, minScore) {
+        const results = await ftsSearch(table, query, {
+            limit,
+            filterClause,
+            minScore,
+        });
+        return {
+            results,
+            query,
+            searchType: 'exact',
+            elapsedMs: 0,
+        };
+    }
+    /**
+     * Hybrid search: Vector + BM25 with RRF reranking.
+     * Good general-purpose search.
+     *
+     * @param autoBoost - When true, increase BM25 weight and oversample if vector scores are low
+     * @param autoBoostThreshold - Vector score threshold below which auto-boost activates
+     * @param returnDebug - Include debug info in results for AI evaluation
+     */
+    async searchHybrid(table, query, limit, bm25Weight, filterClause, minScore, autoBoost = true, autoBoostThreshold = 0.3, returnDebug = false) {
+        const queryVector = await this.embeddings.embedSingle(query);
+        // Initial search with default oversample to assess vector confidence
+        const initialOversample = limit * DEFAULT_OVERSAMPLE_MULTIPLIER;
+        const [initialVectorResults, initialFtsResults] = await Promise.all([
+            vectorSearch(table, queryVector, {
+                limit: initialOversample,
+                filterClause,
+            }),
+            ftsSearch(table, query, {
+                limit: initialOversample,
+                filterClause,
+            }),
+        ]);
+        // Calculate confidence metrics
+        const maxVectorScore = Math.max(...initialVectorResults.map(r => r.score), 0);
+        const maxFtsScore = Math.max(...initialFtsResults.map(r => r.ftsScore ?? r.score), 0);
+        // Dynamic oversample: increase when vector confidence is low
+        let oversampleMultiplier = DEFAULT_OVERSAMPLE_MULTIPLIER;
+        let dynamicOversampleApplied = false;
+        if (autoBoost && maxVectorScore < autoBoostThreshold) {
+            // Linear scale from 2x to 4x based on how low vector scores are
+            // At threshold (0.3): 2x, at 0: 4x
+            const boost = 1 - maxVectorScore / autoBoostThreshold;
+            oversampleMultiplier =
+                DEFAULT_OVERSAMPLE_MULTIPLIER +
+                    boost * (MAX_OVERSAMPLE_MULTIPLIER - DEFAULT_OVERSAMPLE_MULTIPLIER);
+            dynamicOversampleApplied =
+                oversampleMultiplier > DEFAULT_OVERSAMPLE_MULTIPLIER;
+        }
+        const effectiveOversample = Math.round(limit * oversampleMultiplier);
+        // If we need more results due to dynamic oversample, fetch additional
+        let vectorResults = initialVectorResults;
+        let ftsResults = initialFtsResults;
+        if (effectiveOversample > initialOversample) {
+            // Re-fetch with higher limit
+            [vectorResults, ftsResults] = await Promise.all([
+                vectorSearch(table, queryVector, {
+                    limit: effectiveOversample,
+                    filterClause,
+                }),
+                ftsSearch(table, query, {
+                    limit: effectiveOversample,
+                    filterClause,
+                }),
+            ]);
+        }
+        // Auto-boost: increase BM25 weight when vector confidence is low
+        let effectiveBm25Weight = bm25Weight;
+        let autoBoostApplied = false;
+        if (autoBoost && maxVectorScore < autoBoostThreshold) {
+            // Calculate boost factor: higher boost when vector scores are lower
+            const boost = (autoBoostThreshold - maxVectorScore) / autoBoostThreshold;
+            // Increase BM25 weight by up to 0.5, capped at 0.9
+            effectiveBm25Weight = Math.min(0.9, bm25Weight + boost * 0.5);
+            autoBoostApplied = effectiveBm25Weight !== bm25Weight;
+        }
+        // Combine with RRF using effective weight
+        const vectorWeight = 1 - effectiveBm25Weight;
+        let results = hybridRerank(vectorResults, ftsResults, limit, vectorWeight);
+        // Apply minScore filter
+        if (minScore) {
+            results = results.filter(r => r.score >= minScore);
+        }
+        // Build debug info if requested
+        const debug = returnDebug
+            ? {
+                maxVectorScore,
+                maxFtsScore,
+                requestedBm25Weight: bm25Weight,
+                effectiveBm25Weight,
+                autoBoostApplied,
+                autoBoostThreshold,
+                vectorResultCount: vectorResults.length,
+                ftsResultCount: ftsResults.length,
+                oversampleMultiplier,
+                dynamicOversampleApplied,
+            }
+            : undefined;
+        return {
+            results,
+            query,
+            searchType: 'hybrid',
+            elapsedMs: 0,
+            debug,
+        };
+    }
+    /**
+     * Definition search: Direct metadata lookup.
+     * Best for "where is X defined?" queries.
+     */
+    async searchDefinition(table, symbolName, limit, typeFilter, additionalFilter) {
+        const definitionFilter = buildDefinitionFilter(symbolName, typeFilter);
+        // Combine with additional filters
+        const fullFilter = additionalFilter
+            ? `(${definitionFilter}) AND (${additionalFilter})`
+            : definitionFilter;
+        // Use table query directly for metadata lookup
+        const queryResults = await table
+            .query()
+            .where(fullFilter)
+            .limit(limit)
+            .toArray();
+        const results = queryResults.map((row, index) => {
+            const chunk = row;
+            return {
+                id: chunk.id,
+                text: chunk.text,
+                filepath: chunk.filepath,
+                filename: chunk.filename,
+                name: chunk.name,
+                type: chunk.type,
+                startLine: chunk.start_line,
+                endLine: chunk.end_line,
+                score: 1 / (index + 1), // Rank-based score
+                signature: chunk.signature,
+                isExported: chunk.is_exported,
+            };
+        });
+        return {
+            results,
+            query: symbolName,
+            searchType: 'definition',
+            elapsedMs: 0,
+        };
+    }
+    /**
+     * Similar search: Vector search with code snippet as query.
+     * Best for "find code like this" queries.
+     */
+    async searchSimilar(table, codeSnippet, limit, filterClause, minScore) {
+        // Embed the code snippet directly
+        const queryVector = await this.embeddings.embedSingle(codeSnippet);
+        const results = await vectorSearch(table, queryVector, {
+            limit,
+            filterClause,
+            minScore,
+        });
+        return {
+            results,
+            query: codeSnippet.substring(0, 100) + (codeSnippet.length > 100 ? '...' : ''),
+            searchType: 'similar',
+            elapsedMs: 0,
+        };
+    }
+    /**
+     * Perform vector-only search. (Legacy method)
+     */
+    async searchVector(query, limit = DEFAULT_LIMIT) {
+        return this.search(query, { mode: 'semantic', limit });
+    }
+    /**
+     * Perform FTS-only search. (Legacy method)
+     */
+    async searchFts(query, limit = DEFAULT_LIMIT) {
+        return this.search(query, { mode: 'exact', limit });
+    }
+    /**
+     * Initialize the search engine.
+     */
+    async ensureInitialized() {
+        if (this.initialized)
+            return;
+        const config = await loadConfig(this.projectRoot);
+        // Initialize storage
+        this.storage = new Storage(this.projectRoot, config.embeddingDimensions);
+        await this.storage.connect();
+        // Initialize embeddings with config (includes apiKey for cloud providers)
+        this.embeddings = this.createEmbeddingProvider(config);
+        await this.embeddings.initialize();
+        this.initialized = true;
+        this.log('info', 'SearchEngine initialized');
+    }
+    /**
+     * Create the appropriate embedding provider based on config.
+     */
+    createEmbeddingProvider(config) {
+        const apiKey = config.apiKey;
+        switch (config.embeddingProvider) {
+            case 'local':
+                return new LocalEmbeddingProvider();
+            case 'local-4b':
+                return new Local4BEmbeddingProvider();
+            case 'gemini':
+                return new GeminiEmbeddingProvider(apiKey);
+            case 'mistral':
+                return new MistralEmbeddingProvider(apiKey);
+            case 'openai':
+                return new OpenAIEmbeddingProvider(apiKey);
+            default:
+                throw new Error(`Unknown embedding provider: ${config.embeddingProvider}`);
+        }
+    }
+    /**
+     * Get the code chunks table.
+     */
+    async getTable() {
+        return this.storage.getChunksTable();
+    }
+    /**
+     * Log a message.
+     */
+    log(level, message) {
+        if (!this.logger)
+            return;
+        this.logger[level]('Search', message);
+    }
+    /**
+     * Close the search engine and free resources.
+     */
+    close() {
+        this.storage?.close();
+        this.embeddings?.close();
+        this.initialized = false;
+        this.log('info', 'SearchEngine closed');
+    }
+}

package/dist/rag/search/types.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Search result types.
+ */
+/**
+ * Search mode determines the search strategy.
+ */
+export type SearchMode = 'semantic' | 'exact' | 'hybrid' | 'definition' | 'similar';
+/**
+ * A single search result.
+ */
+export interface SearchResult {
+    /** Unique ID: "{filepath}:{startLine}" */
+    id: string;
+    /** Source code content */
+    text: string;
+    /** Relative file path */
+    filepath: string;
+    /** Just the filename */
+    filename: string;
+    /** Symbol name */
+    name: string;
+    /** Chunk type: function, class, method, or module */
+    type: string;
+    /** Start line number (1-indexed) */
+    startLine: number;
+    /** End line number (1-indexed) */
+    endLine: number;
+    /** Combined score (for hybrid search) */
+    score: number;
+    /** Vector similarity score (optional) */
+    vectorScore?: number;
+    /** FTS/BM25 score (optional) */
+    ftsScore?: number;
+    /** Function/method signature (if available) */
+    signature?: string | null;
+    /** Whether symbol is exported */
+    isExported?: boolean;
+}
+/**
+ * Debug information for search quality evaluation.
+ * Helps AI agents understand search effectiveness and tune parameters.
+ */
+export interface SearchDebugInfo {
+    /** Maximum vector similarity score across results */
+    maxVectorScore: number;
+    /** Maximum FTS/BM25 score across results */
+    maxFtsScore: number;
+    /** BM25 weight requested by caller */
+    requestedBm25Weight: number;
+    /** Effective BM25 weight after auto-boost */
+    effectiveBm25Weight: number;
+    /** Whether auto-boost was applied */
+    autoBoostApplied: boolean;
+    /** Auto-boost threshold used */
+    autoBoostThreshold: number;
+    /** Number of results from vector search */
+    vectorResultCount: number;
+    /** Number of results from FTS search */
+    ftsResultCount: number;
+    /** Oversample multiplier used (2-4x) */
+    oversampleMultiplier?: number;
+    /** Whether dynamic oversample boost was applied */
+    dynamicOversampleApplied?: boolean;
+}
+/**
+ * Collection of search results with metadata.
+ */
+export interface SearchResults {
+    /** Array of search results */
+    results: SearchResult[];
+    /** Original search query */
+    query: string;
+    /** Type of search performed */
+    searchType: SearchMode;
+    /** Time taken in milliseconds */
+    elapsedMs: number;
+    /** Total matches (when exhaustive=true) */
+    totalMatches?: number;
+    /** Debug info for hybrid search (when return_debug=true) */
+    debug?: SearchDebugInfo;
+}
+/**
+ * Transparent, AI-controlled filters.
+ * AI sees exactly what's being filtered.
+ */
+export interface SearchFilters {
+    /** Scope to files starting with this path prefix (e.g., "src/api/") */
+    pathPrefix?: string;
+    /** Must contain ALL of these strings in path */
+    pathContains?: string[];
+    /** Must not contain ANY of these strings in path */
+    pathNotContains?: string[];
+    /** Filter by chunk type: function, class, method, module */
+    type?: ('function' | 'class' | 'method' | 'module')[];
+    /** Filter by file extension (e.g., [".ts", ".tsx"]) */
+    extension?: string[];
+    /** Only exported/public symbols */
+    isExported?: boolean;
+    /** Decorator name contains this string (e.g., "Get", "route") */
+    decoratorContains?: string;
+    /** Has documentation/docstring */
+    hasDocstring?: boolean;
+}
+/**
+ * Options for search operations.
+ */
+export interface SearchOptions {
+    /** Search mode (default: 'hybrid') */
+    mode?: SearchMode;
+    /** Maximum number of results (default: 10) */
+    limit?: number;
+    /** Weight for BM25 in hybrid search (0.0-1.0, default: 0.3) */
+    bm25Weight?: number;
+    /** Return all matches above threshold (default: false) */
+    exhaustive?: boolean;
+    /** Minimum score threshold 0-1 (default: 0) */
+    minScore?: number;
+    /** Transparent filters */
+    filters?: SearchFilters;
+    /** Code snippet for 'similar' mode */
+    codeSnippet?: string;
+    /** Symbol name for 'definition' mode */
+    symbolName?: string;
+    /** Enable auto-boost of BM25 weight when vector scores are low (default: true) */
+    autoBoost?: boolean;
+    /** Vector score threshold below which auto-boost activates (default: 0.3) */
+    autoBoostThreshold?: number;
+    /** Include debug info in results (default: false) */
+    returnDebug?: boolean;
+}

package/dist/rag/search/types.js

@@ -0,0 +1,4 @@
+/**
+ * Search result types.
+ */
+export {};

package/dist/rag/search/vector.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Vector similarity search using LanceDB.
+ */
+import type { Table } from '@lancedb/lancedb';
+import type { SearchResult } from './types.js';
+/**
+ * Options for vector search.
+ */
+export interface VectorSearchOptions {
+    /** Maximum number of results */
+    limit: number;
+    /** LanceDB WHERE clause filter */
+    filterClause?: string;
+    /** Minimum score threshold (0-1) */
+    minScore?: number;
+}
+/**
+ * Perform vector similarity search.
+ *
+ * @param table - LanceDB table to search
+ * @param queryVector - Query embedding vector
+ * @param options - Search options
+ * @returns Array of search results with vector scores
+ */
+export declare function vectorSearch(table: Table, queryVector: number[], options: VectorSearchOptions | number): Promise<SearchResult[]>;