rag-lite-ts 2.1.1 → 2.3.0

package/dist/esm/core/streaming-operations.js

@@ -0,0 +1,409 @@
+/**
+ * Streaming Operations for Large Content - Task 9.1 Implementation
+ * Provides memory-efficient streaming operations for content ingestion and retrieval
+ * Minimizes memory usage for large files through streaming algorithms
+ */
+import { createHash } from 'crypto';
+import { createReadStream, createWriteStream, promises as fs } from 'fs';
+import { pipeline } from 'stream/promises';
+import { Transform, Readable } from 'stream';
+import { dirname } from 'path';
+/**
+ * Default streaming configuration
+ */
+const DEFAULT_STREAMING_CONFIG = {
+    chunkSize: 64 * 1024, // 64KB chunks
+    enableProgress: false,
+    enableHashing: false,
+    timeout: 300000 // 5 minutes
+};
+/**
+ * StreamingOperations class provides memory-efficient operations for large content
+ */
+export class StreamingOperations {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_STREAMING_CONFIG, ...config };
+    }
+    /**
+     * Calculates SHA-256 hash of a file using streaming to minimize memory usage
+     * @param filePath - Path to the file to hash
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to hash result
+     */
+    async calculateFileHashStreaming(filePath, progressCallback) {
+        const startTime = Date.now();
+        let bytesProcessed = 0;
+        let totalBytes;
+        try {
+            // Get file size for progress reporting
+            if (this.config.enableProgress || progressCallback) {
+                const stats = await fs.stat(filePath);
+                totalBytes = stats.size;
+            }
+            const hash = createHash('sha256');
+            const readStream = createReadStream(filePath, {
+                highWaterMark: this.config.chunkSize
+            });
+            // Use promise-based approach instead of pipeline for better control
+            return new Promise((resolve, reject) => {
+                const timeoutId = setTimeout(() => {
+                    readStream.destroy();
+                    reject(new Error('File hash calculation timed out'));
+                }, this.config.timeout);
+                readStream.on('data', (chunk) => {
+                    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                    hash.update(buffer);
+                    bytesProcessed += buffer.length;
+                    // Report progress if callback provided
+                    if (progressCallback) {
+                        progressCallback(bytesProcessed, totalBytes);
+                    }
+                });
+                readStream.on('end', () => {
+                    clearTimeout(timeoutId);
+                    const processingTimeMs = Date.now() - startTime;
+                    resolve({
+                        hash: hash.digest('hex'),
+                        bytesProcessed,
+                        processingTimeMs
+                    });
+                });
+                readStream.on('error', (error) => {
+                    clearTimeout(timeoutId);
+                    reject(new Error(`Failed to read file: ${error.message}`));
+                });
+            });
+        }
+        catch (error) {
+            throw new Error(`Failed to calculate file hash: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Calculates SHA-256 hash of a buffer using streaming to minimize memory usage
+     * @param content - Buffer to hash
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to hash result
+     */
+    async calculateBufferHashStreaming(content, progressCallback) {
+        const startTime = Date.now();
+        let bytesProcessed = 0;
+        const totalBytes = content.length;
+        try {
+            const hash = createHash('sha256');
+            // Process buffer in chunks to avoid memory spikes
+            const chunkSize = this.config.chunkSize;
+            for (let offset = 0; offset < content.length; offset += chunkSize) {
+                const chunk = content.subarray(offset, Math.min(offset + chunkSize, content.length));
+                hash.update(chunk);
+                bytesProcessed += chunk.length;
+                // Report progress if callback provided
+                if (progressCallback) {
+                    progressCallback(bytesProcessed, totalBytes);
+                }
+                // Yield control to event loop to prevent blocking
+                if (offset % (chunkSize * 10) === 0) {
+                    await new Promise(resolve => setImmediate(resolve));
+                }
+            }
+            const processingTimeMs = Date.now() - startTime;
+            return {
+                hash: hash.digest('hex'),
+                bytesProcessed,
+                processingTimeMs
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to calculate buffer hash: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Copies a file using streaming operations with optional hashing
+     * @param sourcePath - Source file path
+     * @param destinationPath - Destination file path
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to copy result
+     */
+    async copyFileStreaming(sourcePath, destinationPath, progressCallback) {
+        const startTime = Date.now();
+        let bytesWritten = 0;
+        let totalBytes;
+        let hash;
+        try {
+            // Get file size for progress reporting
+            if (this.config.enableProgress || progressCallback) {
+                const stats = await fs.stat(sourcePath);
+                totalBytes = stats.size;
+            }
+            // Ensure destination directory exists
+            await fs.mkdir(dirname(destinationPath), { recursive: true });
+            const readStream = createReadStream(sourcePath, {
+                highWaterMark: this.config.chunkSize
+            });
+            const writeStream = createWriteStream(destinationPath);
+            let hashCalculator;
+            if (this.config.enableHashing) {
+                hashCalculator = createHash('sha256');
+            }
+            // Create transform stream for progress tracking and optional hashing
+            const progressTransform = new Transform({
+                transform(chunk, encoding, callback) {
+                    bytesWritten += chunk.length;
+                    // Update hash if enabled
+                    if (hashCalculator) {
+                        hashCalculator.update(chunk);
+                    }
+                    // Report progress if callback provided
+                    if (progressCallback) {
+                        progressCallback(bytesWritten, totalBytes);
+                    }
+                    callback(null, chunk);
+                }
+            });
+            // Use pipeline for proper error handling and cleanup
+            await this.withTimeout(pipeline(readStream, progressTransform, writeStream), this.config.timeout, 'File copy operation timed out');
+            const processingTimeMs = Date.now() - startTime;
+            if (hashCalculator) {
+                hash = hashCalculator.digest('hex');
+            }
+            return {
+                bytesWritten,
+                processingTimeMs,
+                hash
+            };
+        }
+        catch (error) {
+            // Clean up destination file if copy failed
+            try {
+                await fs.unlink(destinationPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+            throw new Error(`Failed to copy file: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Writes buffer content to file using streaming operations
+     * @param content - Buffer to write
+     * @param destinationPath - Destination file path
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to write result
+     */
+    async writeBufferStreaming(content, destinationPath, progressCallback) {
+        const startTime = Date.now();
+        let bytesWritten = 0;
+        const totalBytes = content.length;
+        let hash;
+        try {
+            // Ensure destination directory exists
+            await fs.mkdir(dirname(destinationPath), { recursive: true });
+            const writeStream = createWriteStream(destinationPath);
+            let hashCalculator;
+            if (this.config.enableHashing) {
+                hashCalculator = createHash('sha256');
+            }
+            // Create readable stream from buffer
+            const readableStream = Readable.from(this.bufferToChunks(content));
+            // Create transform stream for progress tracking and optional hashing
+            const progressTransform = new Transform({
+                transform(chunk, encoding, callback) {
+                    bytesWritten += chunk.length;
+                    // Update hash if enabled
+                    if (hashCalculator) {
+                        hashCalculator.update(chunk);
+                    }
+                    // Report progress if callback provided
+                    if (progressCallback) {
+                        progressCallback(bytesWritten, totalBytes);
+                    }
+                    callback(null, chunk);
+                }
+            });
+            // Use pipeline for proper error handling and cleanup
+            await this.withTimeout(pipeline(readableStream, progressTransform, writeStream), this.config.timeout, 'Buffer write operation timed out');
+            const processingTimeMs = Date.now() - startTime;
+            if (hashCalculator) {
+                hash = hashCalculator.digest('hex');
+            }
+            return {
+                bytesWritten,
+                processingTimeMs,
+                hash
+            };
+        }
+        catch (error) {
+            // Clean up destination file if write failed
+            try {
+                await fs.unlink(destinationPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+            throw new Error(`Failed to write buffer: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Reads file content and converts to base64 using streaming to minimize memory usage
+     * @param filePath - Path to the file to read
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to base64 string
+     */
+    async readFileAsBase64Streaming(filePath, progressCallback) {
+        let bytesProcessed = 0;
+        let totalBytes;
+        try {
+            // Get file size for progress reporting
+            if (this.config.enableProgress || progressCallback) {
+                const stats = await fs.stat(filePath);
+                totalBytes = stats.size;
+            }
+            // For base64 conversion, we need to read the entire file to get correct encoding
+            // Streaming base64 conversion chunk by chunk doesn't work correctly because
+            // base64 encoding requires complete byte sequences
+            const content = await fs.readFile(filePath);
+            bytesProcessed = content.length;
+            // Report progress if callback provided
+            if (progressCallback) {
+                progressCallback(bytesProcessed, totalBytes);
+            }
+            // Convert to base64
+            return content.toString('base64');
+        }
+        catch (error) {
+            throw new Error(`Failed to read file as base64: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Validates file integrity by comparing streaming hash with expected hash
+     * @param filePath - Path to the file to validate
+     * @param expectedHash - Expected SHA-256 hash
+     * @param progressCallback - Optional callback for progress reporting
+     * @returns Promise that resolves to validation result
+     */
+    async validateFileIntegrityStreaming(filePath, expectedHash, progressCallback) {
+        try {
+            const result = await this.calculateFileHashStreaming(filePath, progressCallback);
+            return {
+                isValid: result.hash === expectedHash.toLowerCase(),
+                actualHash: result.hash,
+                bytesProcessed: result.bytesProcessed
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to validate file integrity: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Gets file information without loading content into memory
+     * @param filePath - Path to the file
+     * @returns Promise that resolves to file information
+     */
+    async getFileInfo(filePath) {
+        try {
+            const stats = await fs.stat(filePath);
+            // Check permissions
+            let canRead = false;
+            let canWrite = false;
+            try {
+                await fs.access(filePath, fs.constants.R_OK);
+                canRead = true;
+            }
+            catch {
+                // Cannot read
+            }
+            try {
+                await fs.access(filePath, fs.constants.W_OK);
+                canWrite = true;
+            }
+            catch {
+                // Cannot write
+            }
+            return {
+                size: stats.size,
+                isFile: stats.isFile(),
+                isDirectory: stats.isDirectory(),
+                lastModified: stats.mtime,
+                canRead,
+                canWrite
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get file info: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    // =============================================================================
+    // PRIVATE HELPER METHODS
+    // =============================================================================
+    /**
+     * Converts buffer to chunks for streaming
+     * @param buffer - Buffer to chunk
+     * @returns Generator that yields buffer chunks
+     */
+    *bufferToChunks(buffer) {
+        const chunkSize = this.config.chunkSize;
+        for (let offset = 0; offset < buffer.length; offset += chunkSize) {
+            yield buffer.subarray(offset, Math.min(offset + chunkSize, buffer.length));
+        }
+    }
+    /**
+     * Wraps a promise with timeout functionality
+     * @param promise - Promise to wrap
+     * @param timeoutMs - Timeout in milliseconds
+     * @param errorMessage - Error message for timeout
+     * @returns Promise that rejects if timeout is reached
+     */
+    async withTimeout(promise, timeoutMs, errorMessage) {
+        const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => reject(new Error(errorMessage)), timeoutMs);
+        });
+        return Promise.race([promise, timeoutPromise]);
+    }
+}
+/**
+ * Creates a StreamingOperations instance with default configuration
+ * @param config - Optional configuration overrides
+ * @returns StreamingOperations instance
+ */
+export function createStreamingOperations(config) {
+    return new StreamingOperations(config);
+}
+/**
+ * Utility function to format bytes for progress reporting
+ * @param bytes - Number of bytes
+ * @returns Formatted string (e.g., "1.5 MB")
+ */
+export function formatBytes(bytes) {
+    if (bytes === 0)
+        return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
+}
+/**
+ * Utility function to format processing time
+ * @param milliseconds - Processing time in milliseconds
+ * @returns Formatted string (e.g., "1.5s" or "150ms")
+ */
+export function formatProcessingTime(milliseconds) {
+    if (milliseconds < 1000) {
+        return `${Math.round(milliseconds)}ms`;
+    }
+    else {
+        return `${(milliseconds / 1000).toFixed(1)}s`;
+    }
+}
+/**
+ * Utility function to calculate processing speed
+ * @param bytes - Number of bytes processed
+ * @param milliseconds - Processing time in milliseconds
+ * @returns Speed in MB/s
+ */
+export function calculateProcessingSpeed(bytes, milliseconds) {
+    if (milliseconds === 0)
+        return 0;
+    const bytesPerSecond = (bytes / milliseconds) * 1000;
+    return bytesPerSecond / (1024 * 1024); // Convert to MB/s
+}
+//# sourceMappingURL=streaming-operations.js.map

package/dist/esm/core/types.d.ts

@@ -0,0 +1,66 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+export interface ContentDocument {
+    source: string;
+    title: string;
+    content: string;
+    contentType: string;
+    metadata?: Record<string, any>;
+}
+export interface ContentChunk {
+    text: string;
+    chunkIndex: number;
+    contentType: string;
+    metadata?: Record<string, any>;
+}
+export interface EmbeddingResult {
+    embedding_id: string;
+    vector: Float32Array;
+    contentType?: string;
+    metadata?: Record<string, any>;
+}
+export type EmbedFunction = (query: string, contentType?: string) => Promise<EmbeddingResult>;
+export interface SearchResult {
+    content: string;
+    score: number;
+    contentType: string;
+    document: {
+        id: number;
+        source: string;
+        title: string;
+        contentType: string;
+        contentId?: string;
+    };
+    metadata?: Record<string, any>;
+}
+export type RerankFunction = (query: string, results: SearchResult[], contentType?: string) => Promise<SearchResult[]>;
+export interface EmbeddingQueryInterface {
+    embedQuery: EmbedFunction;
+    supportedContentTypes: string[];
+    embeddingDimensions: number;
+}
+export interface RerankingInterface {
+    rerankResults: RerankFunction;
+    supportedContentTypes: string[];
+    isEnabled: boolean;
+}
+export interface SearchOptions {
+    top_k?: number;
+    rerank?: boolean;
+    contentType?: 'text' | 'image' | 'combined';
+}
+export interface Chunk {
+    text: string;
+    chunk_index: number;
+}
+export interface Document {
+    source: string;
+    title: string;
+    content: string;
+    metadata?: Record<string, any>;
+}
+export type { DatabaseConnection } from './db.js';
+export type { ContentChunk as ChunkResult } from './db.js';
+//# sourceMappingURL=types.d.ts.map

package/dist/esm/core/types.js

@@ -0,0 +1,6 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/esm/core/universal-embedder.d.ts

@@ -0,0 +1,177 @@
+/**
+ * CORE MODULE — Universal Embedder Interface for Chameleon Architecture
+ *
+ * Model-agnostic interfaces supporting both text and multimodal models.
+ * Designed for runtime polymorphism and extensibility.
+ *
+ * ARCHITECTURAL ROLE:
+ * This file contains ONLY interfaces, types, and utility functions - no implementation logic.
+ * It defines the contract that all embedder implementations must follow.
+ *
+ * CONTENTS:
+ * - UniversalEmbedder interface: Core contract for all embedders
+ * - Type definitions: ModelType, ContentType, etc.
+ * - Error classes: ModelValidationError, ContentTypeError, etc.
+ * - Utility functions: Type guards and validation helpers
+ * - Constants: Default capabilities and content types
+ *
+ * USAGE:
+ * - Implementation layers (text, multimodal) implement these interfaces
+ * - Core layer uses these types for dependency injection
+ * - Public API exports these types for external use
+ */
+import type { EmbeddingResult } from '../types.js';
+/**
+ * Universal embedder interface supporting both text and multimodal models
+ * Provides a consistent API across different model types and content types
+ */
+export interface UniversalEmbedder {
+    readonly modelName: string;
+    readonly modelType: ModelType;
+    readonly dimensions: number;
+    readonly supportedContentTypes: readonly string[];
+    embedText(text: string): Promise<EmbeddingResult>;
+    embedImage?(imagePath: string): Promise<EmbeddingResult>;
+    embedBatch(items: EmbeddingBatchItem[]): Promise<EmbeddingResult[]>;
+    loadModel(): Promise<void>;
+    isLoaded(): boolean;
+    getModelInfo(): ModelInfo;
+    cleanup(): Promise<void>;
+}
+/**
+ * Enhanced embedding result with content type and metadata support
+ * Extends the existing EmbeddingResult interface for multimodal compatibility
+ */
+export interface EnhancedEmbeddingResult extends EmbeddingResult {
+    contentType: string;
+    metadata?: Record<string, any>;
+}
+/**
+ * Batch embedding item for efficient processing
+ */
+export interface EmbeddingBatchItem {
+    content: string;
+    contentType: string;
+    metadata?: Record<string, any>;
+}
+/**
+ * Model information interface for runtime introspection
+ */
+export interface ModelInfo {
+    name: string;
+    type: ModelType;
+    dimensions: number;
+    version: string;
+    supportedContentTypes: readonly string[];
+    capabilities: ModelCapabilities;
+    requirements: ModelRequirements;
+}
+/**
+ * Model capabilities for feature detection
+ */
+export interface ModelCapabilities {
+    supportsText: boolean;
+    supportsImages: boolean;
+    supportsBatchProcessing: boolean;
+    supportsMetadata: boolean;
+    maxBatchSize?: number;
+    maxTextLength?: number;
+    supportedImageFormats?: readonly string[];
+    supportsMultimodal?: boolean;
+    supportsCrossModalSearch?: boolean;
+    unifiedEmbeddingSpace?: boolean;
+    reliableImplementation?: boolean;
+}
+/**
+ * Model requirements for validation and compatibility checking
+ */
+export interface ModelRequirements {
+    transformersJsVersion: string;
+    minimumMemory?: number;
+    requiredFeatures?: readonly string[];
+    platformSupport?: readonly string[];
+}
+/**
+ * Supported model types in the Chameleon architecture
+ */
+export type ModelType = 'sentence-transformer' | 'clip';
+/**
+ * Content types supported by the system
+ */
+export type ContentType = 'text' | 'image' | 'pdf' | 'docx';
+/**
+ * Model validation result for compatibility checking
+ */
+export interface ModelValidationResult {
+    isValid: boolean;
+    errors: string[];
+    warnings: string[];
+    suggestions: string[];
+}
+/**
+ * Simple embedder creation function type
+ * Replaces complex factory patterns with a straightforward function approach
+ */
+export type CreateEmbedderFunction = (modelName: string, options?: EmbedderCreationOptions) => Promise<UniversalEmbedder>;
+/**
+ * Options for creating embedder instances
+ */
+export interface EmbedderCreationOptions {
+    cachePath?: string;
+    maxBatchSize?: number;
+    timeout?: number;
+    enableGPU?: boolean;
+    customConfig?: Record<string, any>;
+}
+/**
+ * Model validation error for unsupported or incompatible models
+ */
+export declare class ModelValidationError extends Error {
+    readonly modelName: string;
+    readonly availableModels: readonly string[];
+    constructor(modelName: string, availableModels: readonly string[], message: string);
+}
+/**
+ * Transformers.js compatibility error for version mismatches
+ */
+export declare class TransformersCompatibilityError extends Error {
+    readonly modelName: string;
+    readonly requiredVersion: string;
+    readonly currentVersion: string;
+    constructor(modelName: string, requiredVersion: string, currentVersion: string, message: string);
+}
+/**
+ * Content type error for unsupported content types
+ */
+export declare class ContentTypeError extends Error {
+    readonly contentType: string;
+    readonly supportedTypes: readonly string[];
+    constructor(contentType: string, supportedTypes: readonly string[], message: string);
+}
+/**
+ * Type guard for checking if an embedder supports images
+ */
+export declare function supportsImages(embedder: UniversalEmbedder): embedder is UniversalEmbedder & {
+    embedImage(imagePath: string): Promise<EmbeddingResult>;
+};
+/**
+ * Type guard for checking if an embedder supports a specific content type
+ */
+export declare function supportsContentType(embedder: UniversalEmbedder, contentType: string): boolean;
+/**
+ * Utility function to create enhanced embedding results
+ */
+export declare function createEnhancedEmbeddingResult(embeddingId: string, vector: Float32Array, contentType: string, metadata?: Record<string, any>): EnhancedEmbeddingResult;
+/**
+ * Utility function to validate content type against supported types
+ */
+export declare function validateContentType(contentType: string, supportedTypes: readonly string[]): void;
+/**
+ * Default supported content types for different model types
+ */
+export declare const DEFAULT_CONTENT_TYPES: Record<ModelType, readonly string[]>;
+/**
+ * Default model capabilities for different model types
+ */
+export declare const DEFAULT_CAPABILITIES: Record<ModelType, ModelCapabilities>;
+//# sourceMappingURL=universal-embedder.d.ts.map