rag-lite-ts 1.0.2 → 2.0.1

package/dist/core/content-resolver.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Content Resolver - Handles content retrieval and format adaptation for unified content system
+ * Resolves content IDs to actual content locations and adapts format based on client needs
+ * Supports efficient batch retrieval operations and handles missing content gracefully
+ */
+import { DatabaseConnection, type ContentMetadata } from './db.js';
+export type { ContentMetadata };
+/**
+ * Content request for batch operations
+ */
+export interface ContentRequest {
+    contentId: string;
+    format: 'file' | 'base64';
+}
+/**
+ * Content result for batch operations
+ */
+export interface ContentResult {
+    contentId: string;
+    success: boolean;
+    content?: string;
+    error?: string;
+}
+/**
+ * ContentResolver class for handling content retrieval and format conversion
+ * Implements format-adaptive content retrieval for CLI and MCP clients
+ */
+export declare class ContentResolver {
+    private db;
+    private streamingOps;
+    private performanceOptimizer;
+    constructor(db: DatabaseConnection);
+    /**
+     * Retrieves content by ID and adapts format based on client needs
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    getContent(contentId: string, format?: 'file' | 'base64'): Promise<string>;
+    /**
+     * Retrieves multiple content items efficiently in batch with performance optimizations
+     * @param requests - Array of content requests with IDs and formats
+     * @returns Promise that resolves to array of content results
+     */
+    getContentBatch(requests: ContentRequest[]): Promise<ContentResult[]>;
+    /**
+     * Fallback batch processing implementation
+     * @param requests - Array of content requests
+     * @returns Promise that resolves to array of content results
+     */
+    private getContentBatchFallback;
+    /**
+     * Retrieves content metadata without loading the actual content
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    getContentMetadata(contentId: string): Promise<ContentMetadata>;
+    /**
+     * Verifies that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    verifyContentExists(contentId: string): Promise<boolean>;
+    /**
+     * Gets performance statistics for batch operations and content retrieval
+     * @returns Performance statistics
+     */
+    getPerformanceStats(): {
+        batchOperations: {
+            totalOperations: number;
+            averageDuration: number;
+            totalBytesProcessed: number;
+            averageSpeed: number;
+            errorRate: number;
+        };
+        contentRetrieval: {
+            totalRetrievals: number;
+            averageDuration: number;
+            cacheHitRate: number;
+        };
+    };
+    /**
+     * Clears performance caches and resets metrics
+     */
+    clearPerformanceCaches(): void;
+    /**
+     * Returns content as file path for CLI clients
+     * @param metadata - Content metadata
+     * @returns File path that can be accessed directly
+     */
+    private getContentAsFilePath;
+    /**
+     * Returns content as base64 string for MCP clients with optimized I/O
+     * @param metadata - Content metadata
+     * @returns Base64-encoded content ready for display
+     */
+    private getContentAsBase64;
+    /**
+     * Cleanup resources to prevent memory leaks and hanging processes
+     * Should be called when ContentResolver is no longer needed
+     */
+    cleanup(): void;
+}
+//# sourceMappingURL=content-resolver.d.ts.map

package/dist/core/content-resolver.js

@@ -0,0 +1,285 @@
+/**
+ * Content Resolver - Handles content retrieval and format adaptation for unified content system
+ * Resolves content IDs to actual content locations and adapts format based on client needs
+ * Supports efficient batch retrieval operations and handles missing content gracefully
+ */
+import { promises as fs } from 'fs';
+import { getContentMetadata } from './db.js';
+import { ContentNotFoundError, ContentRetrievalError, ContentErrorHandler } from './content-errors.js';
+import { withTimeout, SafeBuffer } from './resource-cleanup.js';
+import { createStreamingOperations, formatBytes, formatProcessingTime, calculateProcessingSpeed } from './streaming-operations.js';
+import { createContentPerformanceOptimizer, formatProcessingSpeed } from './content-performance-optimizer.js';
+/**
+ * ContentResolver class for handling content retrieval and format conversion
+ * Implements format-adaptive content retrieval for CLI and MCP clients
+ */
+export class ContentResolver {
+    db;
+    streamingOps;
+    performanceOptimizer;
+    constructor(db) {
+        this.db = db;
+        // Initialize streaming operations for content retrieval
+        this.streamingOps = createStreamingOperations({
+            chunkSize: 256 * 1024, // 256KB chunks for retrieval operations
+            enableProgress: false,
+            enableHashing: false,
+            timeout: 300000 // 5 minutes
+        });
+        // Initialize performance optimizer for batch operations and caching
+        this.performanceOptimizer = createContentPerformanceOptimizer({
+            hashCacheSize: 500, // Smaller cache for resolver
+            hashCacheTTL: 30 * 60 * 1000, // 30 minutes TTL
+            maxConcurrentOperations: 15, // Higher concurrency for retrieval
+            batchSize: 25,
+            fileBufferSize: 256 * 1024,
+            enableAsyncIO: true,
+            enableMetrics: true,
+            metricsRetentionTime: 12 * 60 * 60 * 1000 // 12 hours
+        });
+    }
+    /**
+     * Retrieves content by ID and adapts format based on client needs
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    async getContent(contentId, format = 'file') {
+        try {
+            // Validate format parameter
+            if (format !== 'file' && format !== 'base64') {
+                throw new ContentRetrievalError(contentId, format, 'Format must be either "file" or "base64"', 'format_validation');
+            }
+            // Get content metadata with timeout
+            const metadata = await withTimeout(getContentMetadata(this.db, contentId), 10000, // 10 second timeout for database query
+            'Database query for content metadata timed out');
+            if (!metadata) {
+                throw new ContentNotFoundError(contentId, undefined, 'metadata_lookup');
+            }
+            // Check if content file exists with timeout
+            const contentExists = await withTimeout(this.verifyContentExists(contentId), 5000, // 5 second timeout for file verification
+            'Content file verification timed out');
+            if (!contentExists) {
+                throw new ContentNotFoundError(contentId, metadata.displayName, 'file_verification');
+            }
+            // Return content in requested format with timeout
+            if (format === 'file') {
+                return await withTimeout(this.getContentAsFilePath(metadata), 5000, // 5 second timeout for file path resolution
+                'File path resolution timed out');
+            }
+            else {
+                return await withTimeout(this.getContentAsBase64(metadata), 30000, // 30 second timeout for base64 conversion (can be slow for large files)
+                'Base64 conversion timed out');
+            }
+        }
+        catch (error) {
+            if (error instanceof ContentNotFoundError || error instanceof ContentRetrievalError) {
+                throw error; // Re-throw content-specific errors
+            }
+            ContentErrorHandler.handleContentError(error, 'content retrieval', 'getContent');
+        }
+    }
+    /**
+     * Retrieves multiple content items efficiently in batch with performance optimizations
+     * @param requests - Array of content requests with IDs and formats
+     * @returns Promise that resolves to array of content results
+     */
+    async getContentBatch(requests) {
+        const startTime = Date.now();
+        try {
+            // Use performance optimizer for batch processing
+            const batchResult = await this.performanceOptimizer.processBatchOptimized(requests, async (request) => {
+                try {
+                    const content = await this.getContent(request.contentId, request.format);
+                    return {
+                        contentId: request.contentId,
+                        success: true,
+                        content
+                    };
+                }
+                catch (error) {
+                    return {
+                        contentId: request.contentId,
+                        success: false,
+                        error: error instanceof Error ? error.message : 'Unknown error'
+                    };
+                }
+            }, {
+                batchSize: 25, // Optimized batch size for content retrieval
+                maxConcurrency: 15, // Higher concurrency for I/O operations
+                enableMetrics: true
+            });
+            // Log performance metrics for large batches
+            if (requests.length > 10) {
+                const duration = Date.now() - startTime;
+                const speed = batchResult.averageSpeed;
+                console.log(`Batch retrieval completed: ${requests.length} items in ${formatProcessingTime(duration)} ` +
+                    `(${batchResult.successCount} success, ${batchResult.errorCount} errors, ${formatProcessingSpeed(speed)})`);
+            }
+            return batchResult.results;
+        }
+        catch (error) {
+            // Fallback to original implementation if optimization fails
+            console.warn('Batch optimization failed, using fallback:', error);
+            return this.getContentBatchFallback(requests);
+        }
+    }
+    /**
+     * Fallback batch processing implementation
+     * @param requests - Array of content requests
+     * @returns Promise that resolves to array of content results
+     */
+    async getContentBatchFallback(requests) {
+        const results = [];
+        const concurrencyLimit = 10;
+        const batches = [];
+        for (let i = 0; i < requests.length; i += concurrencyLimit) {
+            batches.push(requests.slice(i, i + concurrencyLimit));
+        }
+        for (const batch of batches) {
+            const batchPromises = batch.map(async (request) => {
+                try {
+                    const content = await this.getContent(request.contentId, request.format);
+                    return {
+                        contentId: request.contentId,
+                        success: true,
+                        content
+                    };
+                }
+                catch (error) {
+                    return {
+                        contentId: request.contentId,
+                        success: false,
+                        error: error instanceof Error ? error.message : 'Unknown error'
+                    };
+                }
+            });
+            const batchResults = await Promise.all(batchPromises);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    /**
+     * Retrieves content metadata without loading the actual content
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    async getContentMetadata(contentId) {
+        const metadata = await getContentMetadata(this.db, contentId);
+        if (!metadata) {
+            throw new ContentNotFoundError(contentId, undefined, 'metadata_retrieval');
+        }
+        return metadata;
+    }
+    /**
+     * Verifies that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    async verifyContentExists(contentId) {
+        try {
+            const metadata = await getContentMetadata(this.db, contentId);
+            if (!metadata) {
+                return false;
+            }
+            // Check if the content file exists and is accessible
+            try {
+                const stats = await fs.stat(metadata.contentPath);
+                return stats.isFile();
+            }
+            catch {
+                return false;
+            }
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Gets performance statistics for batch operations and content retrieval
+     * @returns Performance statistics
+     */
+    getPerformanceStats() {
+        const batchStats = this.performanceOptimizer.getPerformanceStats('batch_processing');
+        const retrievalStats = this.performanceOptimizer.getPerformanceStats('file_read');
+        return {
+            batchOperations: batchStats,
+            contentRetrieval: {
+                totalRetrievals: retrievalStats.totalOperations,
+                averageDuration: retrievalStats.averageDuration,
+                cacheHitRate: retrievalStats.cacheHitRate
+            }
+        };
+    }
+    /**
+     * Clears performance caches and resets metrics
+     */
+    clearPerformanceCaches() {
+        this.performanceOptimizer.clearHashCache();
+    }
+    // =============================================================================
+    // PRIVATE METHODS
+    // =============================================================================
+    /**
+     * Returns content as file path for CLI clients
+     * @param metadata - Content metadata
+     * @returns File path that can be accessed directly
+     */
+    async getContentAsFilePath(metadata) {
+        // For both filesystem and content_dir storage, return the content path
+        // CLI clients can access files directly regardless of storage type
+        return metadata.contentPath;
+    }
+    /**
+     * Returns content as base64 string for MCP clients with optimized I/O
+     * @param metadata - Content metadata
+     * @returns Base64-encoded content ready for display
+     */
+    async getContentAsBase64(metadata) {
+        let safeBuffer = null;
+        try {
+            // Use optimized file reading for better performance
+            if (metadata.fileSize > 10 * 1024 * 1024) { // Use streaming for files > 10MB
+                const startTime = Date.now();
+                const base64Content = await withTimeout(this.streamingOps.readFileAsBase64Streaming(metadata.contentPath), 300000, // 5 minute timeout for large file base64 conversion
+                'Streaming base64 conversion timed out');
+                const processingTime = Date.now() - startTime;
+                const speed = calculateProcessingSpeed(metadata.fileSize, processingTime);
+                // Log performance metrics for large files
+                if (metadata.fileSize > 50 * 1024 * 1024) {
+                    console.log(`Optimized base64 conversion completed: ${formatBytes(metadata.fileSize)} in ${formatProcessingTime(processingTime)} (${formatProcessingSpeed(speed)})`);
+                }
+                return base64Content;
+            }
+            else {
+                // For smaller files, use traditional method with memory management
+                const content = await fs.readFile(metadata.contentPath);
+                // Use safe buffer for memory management (don't clear original for normal operations)
+                safeBuffer = new SafeBuffer(content, { clearOriginal: false });
+                // Convert to base64
+                const base64Content = safeBuffer.get().toString('base64');
+                return base64Content;
+            }
+        }
+        catch (error) {
+            throw new ContentRetrievalError(metadata.id, 'base64', `Failed to read content file: ${error instanceof Error ? error.message : 'Unknown error'}`, 'file_reading');
+        }
+        finally {
+            // Clear sensitive buffer data
+            if (safeBuffer) {
+                safeBuffer.clear();
+            }
+        }
+    }
+    /**
+     * Cleanup resources to prevent memory leaks and hanging processes
+     * Should be called when ContentResolver is no longer needed
+     */
+    cleanup() {
+        // Clean up performance optimizer interval that prevents process exit
+        if (this.performanceOptimizer && typeof this.performanceOptimizer.cleanup === 'function') {
+            this.performanceOptimizer.cleanup();
+        }
+    }
+}
+//# sourceMappingURL=content-resolver.js.map

package/dist/core/cross-modal-search.d.ts

@@ -0,0 +1,164 @@
+/**
+ * Cross-Modal Search Implementation
+ *
+ * This module extends the core SearchEngine to provide cross-modal search capabilities
+ * that enable text queries to find images and image queries to find text content.
+ *
+ * Task 4.2: Implement cross-modal search functionality
+ * - Update search logic to handle mixed content type results
+ * - Ensure ranking works properly across text and image content
+ * - Test text queries finding relevant images and vice versa
+ *
+ * Requirements addressed:
+ * - 6.1: Enable text queries to find relevant image content
+ * - 6.2: Enable image queries to find relevant text content
+ * - 6.3: Rank mixed content types by semantic similarity
+ */
+import { SearchEngine } from './search.js';
+import type { SearchResult, SearchOptions } from './types.js';
+import type { EmbedFunction, RerankFunction } from './interfaces.js';
+/**
+ * Extended search options for cross-modal search
+ */
+export interface CrossModalSearchOptions extends SearchOptions {
+    /** Content types to include in search results */
+    includeContentTypes?: string[];
+    /** Whether to enable cross-modal ranking */
+    enableCrossModalRanking?: boolean;
+    /** Minimum similarity threshold for cross-modal results */
+    crossModalThreshold?: number;
+}
+/**
+ * Cross-modal search result with enhanced metadata
+ */
+export interface CrossModalSearchResult extends SearchResult {
+    /** Whether this result is from a different modality than the query */
+    isCrossModal?: boolean;
+    /** Semantic similarity score for cross-modal ranking */
+    semanticSimilarity?: number;
+    /** Original vector search score before cross-modal adjustments */
+    originalScore?: number;
+}
+/**
+ * Cross-Modal Search Engine
+ *
+ * Extends the core SearchEngine to provide cross-modal search capabilities.
+ * This implementation enables:
+ * - Text queries finding semantically similar images
+ * - Image queries finding semantically similar text
+ * - Mixed content type results ranked by semantic similarity
+ * - Unified embedding space leveraging CLIP models
+ */
+export declare class CrossModalSearchEngine extends SearchEngine {
+    private embedder?;
+    constructor(embedFn: EmbedFunction, indexManager: any, db: any, rerankFn?: RerankFunction, contentResolver?: any, embedder?: any);
+    /**
+     * Perform cross-modal search that can find content across different modalities
+     *
+     * This method extends the base search functionality to:
+     * 1. Detect query content type (text or image path)
+     * 2. Generate appropriate embeddings for the query
+     * 3. Search across all content types in the unified embedding space
+     * 4. Rank results by semantic similarity regardless of content type
+     * 5. Apply cross-modal ranking adjustments
+     *
+     * @param query - Search query (text string or image path)
+     * @param options - Cross-modal search options
+     * @returns Promise resolving to cross-modal search results
+     */
+    crossModalSearch(query: string, options?: CrossModalSearchOptions): Promise<CrossModalSearchResult[]>;
+    /**
+     * Search for images using text queries
+     *
+     * @param textQuery - Text description to search for
+     * @param options - Search options
+     * @returns Promise resolving to image search results
+     */
+    searchImagesWithText(textQuery: string, options?: CrossModalSearchOptions): Promise<CrossModalSearchResult[]>;
+    /**
+     * Search for text using image queries
+     *
+     * @param imagePath - Path to image file to search with
+     * @param options - Search options
+     * @returns Promise resolving to text search results
+     */
+    searchTextWithImage(imagePath: string, options?: CrossModalSearchOptions): Promise<CrossModalSearchResult[]>;
+    /**
+     * Search across all content types with unified ranking
+     *
+     * @param query - Search query (text or image path)
+     * @param options - Search options
+     * @returns Promise resolving to mixed content type results
+     */
+    searchUnified(query: string, options?: CrossModalSearchOptions): Promise<CrossModalSearchResult[]>;
+    /**
+     * Detect the content type of a query
+     * @private
+     */
+    private detectQueryContentType;
+    /**
+     * Enhance search results with cross-modal information
+     * @private
+     */
+    private enhanceResultsWithCrossModalInfo;
+    /**
+     * Apply cross-modal ranking to results
+     * @private
+     */
+    private applyCrossModalRanking;
+    /**
+     * Calculate semantic similarity between query and result content
+     * @private
+     */
+    private calculateSemanticSimilarity;
+    /**
+     * Get cross-modal search statistics
+     */
+    getCrossModalStats(): Promise<{
+        totalChunks: number;
+        textChunks: number;
+        imageChunks: number;
+        crossModalCapable: boolean;
+        supportedContentTypes: string[];
+    }>;
+}
+/**
+ * Factory function to create a cross-modal search engine
+ *
+ * @param embedFn - Embedding function that supports multiple content types
+ * @param indexManager - Vector index manager
+ * @param db - Database connection
+ * @param rerankFn - Optional reranking function
+ * @param contentResolver - Content resolver for unified content system
+ * @param embedder - Reference to the embedder for cross-modal operations
+ * @returns CrossModalSearchEngine instance
+ */
+export declare function createCrossModalSearchEngine(embedFn: EmbedFunction, indexManager: any, db: any, rerankFn?: RerankFunction, contentResolver?: any, embedder?: any): CrossModalSearchEngine;
+/**
+ * Utility function to check if a search engine supports cross-modal search
+ *
+ * @param searchEngine - Search engine to check
+ * @returns True if the engine supports cross-modal search
+ */
+export declare function supportsCrossModalSearch(searchEngine: any): searchEngine is CrossModalSearchEngine;
+/**
+ * Cross-modal search result analyzer
+ * Provides utilities for analyzing cross-modal search results
+ */
+export declare class CrossModalResultAnalyzer {
+    /**
+     * Analyze cross-modal search results
+     */
+    static analyzeResults(results: CrossModalSearchResult[]): {
+        totalResults: number;
+        crossModalResults: number;
+        sameModalResults: number;
+        averageSemanticSimilarity: number;
+        contentTypeDistribution: Record<string, number>;
+    };
+    /**
+     * Generate a summary report of cross-modal search results
+     */
+    static generateReport(results: CrossModalSearchResult[], query: string): string;
+}
+//# sourceMappingURL=cross-modal-search.d.ts.map