rag-lite-ts 2.1.0 → 2.2.0

package/dist/esm/core/search-pipeline.js

@@ -0,0 +1,287 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Core search pipeline coordinator
+ * Orchestrates the search pipeline: query processing → vector search → metadata retrieval → optional reranking
+ * Remains completely independent of specific embedding models or transformer libraries
+ */
+export class SearchPipelineCoordinator {
+    embedQueryFn = null;
+    rerankResultsFn = null;
+    indexManager = null;
+    dbConnection = null;
+    defaultContentType = 'text';
+    /**
+     * Set the embedding function for query processing
+     */
+    setEmbedFunction(embedFn) {
+        this.embedQueryFn = embedFn;
+    }
+    /**
+     * Set the reranking function for result reranking
+     */
+    setRerankFunction(rerankFn) {
+        this.rerankResultsFn = rerankFn;
+    }
+    /**
+     * Set the index manager for vector search
+     */
+    setIndexManager(indexManager) {
+        this.indexManager = indexManager;
+    }
+    /**
+     * Set the database connection for metadata retrieval
+     */
+    setDatabaseConnection(dbConnection) {
+        this.dbConnection = dbConnection;
+    }
+    /**
+     * Set the default content type
+     */
+    setDefaultContentType(contentType) {
+        this.defaultContentType = contentType;
+    }
+    /**
+     * Execute the complete search pipeline
+     * Coordinates all steps without knowledge of specific embedding models
+     */
+    async executeSearchPipeline(query, options = {}) {
+        if (!query || query.trim().length === 0) {
+            return [];
+        }
+        const startTime = performance.now();
+        const topK = options.top_k || 10;
+        const shouldRerank = options.rerank !== undefined ? options.rerank : (this.rerankResultsFn !== null);
+        const contentType = options.contentType || this.defaultContentType;
+        // Validate dependencies
+        this.validateDependencies();
+        try {
+            // Step 1: Query processing and embedding
+            const embeddingStartTime = performance.now();
+            const queryEmbedding = await this.embedQuery(query, contentType);
+            const embeddingTime = performance.now() - embeddingStartTime;
+            // Step 2: Vector search
+            const searchStartTime = performance.now();
+            const searchResult = await this.vectorSearch(queryEmbedding.vector, topK);
+            const vectorSearchTime = performance.now() - searchStartTime;
+            if (searchResult.embeddingIds.length === 0) {
+                const totalTime = performance.now() - startTime;
+                console.log(`No similar documents found (${totalTime.toFixed(2)}ms total)`);
+                return [];
+            }
+            // Step 3: Metadata retrieval
+            const retrievalStartTime = performance.now();
+            const chunks = await this.retrieveMetadata(searchResult.embeddingIds);
+            const retrievalTime = performance.now() - retrievalStartTime;
+            // Step 4: Format initial results
+            let results = this.formatResults(chunks, searchResult.distances, searchResult.embeddingIds);
+            // Step 5: Optional reranking
+            let rerankTime = 0;
+            if (shouldRerank && this.rerankResultsFn && results.length > 1) {
+                try {
+                    const rerankStartTime = performance.now();
+                    results = await this.rerankResults(query, results, contentType);
+                    rerankTime = performance.now() - rerankStartTime;
+                }
+                catch (error) {
+                    console.warn(`Reranking failed, using vector search results: ${error instanceof Error ? error.message : 'Unknown error'}`);
+                }
+            }
+            const totalTime = performance.now() - startTime;
+            console.log(`Search pipeline completed: ${results.length} results in ${totalTime.toFixed(2)}ms ` +
+                `(embed: ${embeddingTime.toFixed(2)}ms, vector: ${vectorSearchTime.toFixed(2)}ms, ` +
+                `retrieval: ${retrievalTime.toFixed(2)}ms${rerankTime > 0 ? `, rerank: ${rerankTime.toFixed(2)}ms` : ''})`);
+            return results;
+        }
+        catch (error) {
+            throw new Error(`Search pipeline failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Step 1: Process and embed the query
+     * Uses injected embedding function without knowledge of specific models
+     */
+    async embedQuery(query, contentType) {
+        if (!this.embedQueryFn) {
+            throw new Error('No embedding function provided. Set embedding function before executing pipeline.');
+        }
+        try {
+            return await this.embedQueryFn(query, contentType);
+        }
+        catch (error) {
+            throw new Error(`Query embedding failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Step 2: Perform vector search
+     * Uses index manager without knowledge of specific embedding models
+     */
+    async vectorSearch(queryVector, topK) {
+        if (!this.indexManager) {
+            throw new Error('Index manager not set. Set index manager before executing pipeline.');
+        }
+        try {
+            return this.indexManager.search(queryVector, topK);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {
+                console.warn(`Hash mapping issue detected: ${error.message}`);
+                console.warn('This may indicate index/database synchronization issues. Consider running: raglite rebuild');
+                return { embeddingIds: [], distances: [] };
+            }
+            throw new Error(`Vector search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Step 3: Retrieve metadata from database
+     * Uses database connection without knowledge of specific data formats
+     */
+    async retrieveMetadata(embeddingIds) {
+        if (!this.dbConnection) {
+            throw new Error('Database connection not set. Set database connection before executing pipeline.');
+        }
+        try {
+            const { getChunksByEmbeddingIds } = await import('./db.js');
+            return await getChunksByEmbeddingIds(this.dbConnection, embeddingIds);
+        }
+        catch (error) {
+            throw new Error(`Metadata retrieval failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Step 4: Format initial results
+     * Formats results in core format without knowledge of specific content types
+     */
+    formatResults(chunks, distances, embeddingIds) {
+        const results = [];
+        // Create a map for quick chunk lookup by embedding_id
+        const chunkMap = new Map();
+        chunks.forEach(chunk => {
+            chunkMap.set(chunk.embedding_id, chunk);
+        });
+        // Build results in the order of search results
+        for (let i = 0; i < embeddingIds.length; i++) {
+            const embeddingId = embeddingIds[i];
+            const chunk = chunkMap.get(embeddingId);
+            if (chunk) {
+                // Convert cosine distance to similarity score (1 - distance)
+                const score = Math.max(0, 1 - distances[i]);
+                results.push({
+                    content: chunk.text,
+                    score: score,
+                    contentType: chunk.content_type || this.defaultContentType,
+                    document: {
+                        id: chunk.document_id,
+                        source: chunk.document_source,
+                        title: chunk.document_title,
+                        contentType: chunk.document_content_type || this.defaultContentType
+                    },
+                    metadata: chunk.metadata ? this.parseMetadata(chunk.metadata) : undefined
+                });
+            }
+        }
+        return results;
+    }
+    /**
+     * Step 5: Optional reranking
+     * Uses injected reranking function without knowledge of specific models
+     */
+    async rerankResults(query, results, contentType) {
+        if (!this.rerankResultsFn) {
+            return results; // No reranking function available
+        }
+        try {
+            return await this.rerankResultsFn(query, results, contentType);
+        }
+        catch (error) {
+            console.warn(`Reranking failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            return results; // Return original results on reranking failure
+        }
+    }
+    /**
+     * Validate that all required dependencies are set
+     */
+    validateDependencies() {
+        const missing = [];
+        if (!this.embedQueryFn) {
+            missing.push('embedding function');
+        }
+        if (!this.indexManager) {
+            missing.push('index manager');
+        }
+        if (!this.dbConnection) {
+            missing.push('database connection');
+        }
+        if (missing.length > 0) {
+            throw new Error(`Missing required dependencies: ${missing.join(', ')}. Set all dependencies before executing search pipeline.`);
+        }
+    }
+    /**
+     * Parse metadata safely
+     */
+    parseMetadata(metadata) {
+        try {
+            return JSON.parse(metadata);
+        }
+        catch (error) {
+            console.warn(`Failed to parse metadata: ${metadata}`);
+            return undefined;
+        }
+    }
+    /**
+     * Check if the pipeline is ready to execute
+     */
+    isReady() {
+        return !!(this.embedQueryFn && this.indexManager && this.dbConnection);
+    }
+    /**
+     * Get pipeline status information
+     */
+    getStatus() {
+        return {
+            hasEmbedFunction: this.embedQueryFn !== null,
+            hasRerankFunction: this.rerankResultsFn !== null,
+            hasIndexManager: this.indexManager !== null,
+            hasDatabaseConnection: this.dbConnection !== null,
+            isReady: this.isReady()
+        };
+    }
+    /**
+     * Reset all dependencies (useful for testing or reconfiguration)
+     */
+    reset() {
+        this.embedQueryFn = null;
+        this.rerankResultsFn = null;
+        this.indexManager = null;
+        this.dbConnection = null;
+        this.defaultContentType = 'text';
+    }
+}
+/**
+ * Factory for creating search pipeline coordinators
+ */
+export class SearchPipelineFactory {
+    /**
+     * Create a search pipeline coordinator with all dependencies
+     */
+    static create(embedFn, indexManager, dbConnection, rerankFn, defaultContentType = 'text') {
+        const coordinator = new SearchPipelineCoordinator();
+        coordinator.setEmbedFunction(embedFn);
+        coordinator.setIndexManager(indexManager);
+        coordinator.setDatabaseConnection(dbConnection);
+        coordinator.setDefaultContentType(defaultContentType);
+        if (rerankFn) {
+            coordinator.setRerankFunction(rerankFn);
+        }
+        return coordinator;
+    }
+    /**
+     * Create an empty coordinator for manual configuration
+     */
+    static createEmpty() {
+        return new SearchPipelineCoordinator();
+    }
+}
+//# sourceMappingURL=search-pipeline.js.map

package/dist/esm/core/search.d.ts

@@ -0,0 +1,141 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+import { IndexManager } from '../index-manager.js';
+import { DatabaseConnection } from './db.js';
+import type { SearchResult, SearchOptions } from './types.js';
+import type { EmbedFunction, RerankFunction } from './interfaces.js';
+/**
+ * Search engine that provides semantic search capabilities
+ * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
+ * Uses explicit dependency injection for clean architecture
+ */
+export declare class SearchEngine {
+    private embedFn;
+    private indexManager;
+    private db;
+    private rerankFn?;
+    private contentResolver?;
+    /**
+     * Creates a new SearchEngine with explicit dependency injection
+     *
+     * DEPENDENCY INJECTION PATTERN:
+     * This constructor requires all dependencies to be explicitly provided, enabling:
+     * - Clean separation between core logic and implementation-specific components
+     * - Support for different embedding models (text-only, multimodal, custom)
+     * - Testability through mock injection
+     * - Future extensibility without core changes
+     *
+     * @param embedFn - Function to embed queries into vectors
+     *   - Signature: (query: string, contentType?: string) => Promise<EmbeddingResult>
+     *   - Examples:
+     *     - Text: const embedFn = (query) => textEmbedder.embedSingle(query)
+     *     - Multimodal: const embedFn = (query, type) => type === 'image' ? clipEmbedder.embedImage(query) : clipEmbedder.embedText(query)
+     *     - Custom: const embedFn = (query) => customModel.embed(query)
+     *
+     * @param indexManager - Vector index manager for similarity search
+     *   - Handles vector storage and retrieval operations
+     *   - Works with any embedding dimensions (384, 512, 768, etc.)
+     *   - Example: new IndexManager('./index.bin')
+     *
+     * @param db - Database connection for metadata retrieval
+     *   - Provides access to document and chunk metadata
+     *   - Supports different content types through metadata fields
+     *   - Example: await openDatabase('./db.sqlite')
+     *
+     * @param rerankFn - Optional function to rerank search results
+     *   - Signature: (query: string, results: SearchResult[], contentType?: string) => Promise<SearchResult[]>
+     *   - Examples:
+     *     - Text: const rerankFn = (query, results) => textReranker.rerank(query, results)
+     *     - Custom: const rerankFn = (query, results) => customReranker.rerank(query, results)
+     *     - Disabled: undefined (no reranking)
+     *
+     * USAGE EXAMPLES:
+     * ```typescript
+     * // Text-only search engine
+     * const textEmbedFn = createTextEmbedFunction();
+     * const textRerankFn = createTextRerankFunction();
+     * const indexManager = new IndexManager('./index.bin');
+     * const db = await openDatabase('./db.sqlite');
+     * const search = new SearchEngine(textEmbedFn, indexManager, db, textRerankFn);
+     *
+     * // Search engine without reranking
+     * const search = new SearchEngine(textEmbedFn, indexManager, db);
+     *
+     * // Custom embedding implementation
+     * const customEmbedFn = async (query) => ({
+     *   embedding_id: generateId(),
+     *   vector: await myCustomModel.embed(query)
+     * });
+     * const search = new SearchEngine(customEmbedFn, indexManager, db);
+     * ```
+     */
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver);
+    /**
+     * Perform semantic search on the indexed documents
+     * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
+     * @param query - Search query string
+     * @param options - Search options including top_k and rerank settings
+     * @returns Promise resolving to array of search results
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Perform semantic search using a pre-computed embedding vector
+     * Useful for image-based search or when embedding is computed externally
+     * @param queryVector - Pre-computed query embedding vector
+     * @param options - Search options including top_k and rerank settings
+     * @param originalQuery - Optional original query for reranking (text or image path)
+     * @param embeddingTime - Optional embedding time for logging
+     * @returns Promise resolving to array of search results
+     */
+    searchWithVector(queryVector: Float32Array, options?: SearchOptions, originalQuery?: string, embeddingTime?: number): Promise<SearchResult[]>;
+    /**
+     * Format search results with proper structure
+     * @param chunks - Database chunks with metadata
+     * @param distances - Similarity distances from vector search
+     * @param embeddingIds - Embedding IDs in search result order
+     * @returns Formatted search results
+     */
+    private formatSearchResults;
+    /**
+     * Get search engine statistics
+     * @returns Object with current search engine stats
+     */
+    getStats(): Promise<{
+        totalChunks: number;
+        indexSize: number;
+        rerankingEnabled: boolean;
+    }>;
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    getContent(contentId: string, format?: 'file' | 'base64'): Promise<string>;
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    getContentBatch(contentIds: string[], format?: 'file' | 'base64'): Promise<string[]>;
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    getContentMetadata(contentId: string): Promise<import('./content-resolver.js').ContentMetadata>;
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    verifyContentExists(contentId: string): Promise<boolean>;
+    /**
+     * Clean up resources - explicit cleanup method
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=search.d.ts.map