rag-lite-ts 1.0.1 → 2.0.0

package/dist/config.js

@@ -1,281 +1,76 @@
-import { homedir } from 'os';
-import { join } from 'path';
 /**
- * Default preprocessing configuration - "balanced" mode
+ * Main configuration file with text-specific settings
+ * Extends core configuration with implementation-specific properties
  */
-export const defaultPreprocessingConfig = {
-    mode: 'balanced'
-};
+import { getDefaultModelCachePath } from './core/config.js';
 /**
- * Get the default model cache path as specified in the requirements
- * @returns Default cache path (~/.raglite/models/)
+ * Default configuration object with both core and text-specific settings
  */
-function getDefaultModelCachePath() {
-    return join(homedir(), '.raglite', 'models');
-}
-/**
- * Returns model-specific default configuration values
- * @param modelName - The embedding model name
- * @returns Model-specific defaults for dimensions, chunk_size, chunk_overlap, and batch_size
- */
-export function getModelDefaults(modelName) {
-    if (modelName === 'Xenova/all-mpnet-base-v2') {
-        return {
-            dimensions: 768,
-            chunk_size: 400,
-            chunk_overlap: 80,
-            batch_size: 8
-        };
-    }
-    // Default to sentence-transformers/all-MiniLM-L6-v2 settings (current defaults)
-    return {
-        dimensions: 384,
-        chunk_size: 250,
-        chunk_overlap: 50,
-        batch_size: 16
-    };
-}
-// Create config with model-specific defaults
-const embeddingModel = process.env.RAG_EMBEDDING_MODEL || 'sentence-transformers/all-MiniLM-L6-v2';
-const modelDefaults = getModelDefaults(embeddingModel);
 export const config = {
-    embedding_model: embeddingModel,
-    chunk_size: parseInt(process.env.RAG_CHUNK_SIZE || modelDefaults.chunk_size.toString(), 10),
-    chunk_overlap: parseInt(process.env.RAG_CHUNK_OVERLAP || modelDefaults.chunk_overlap.toString(), 10),
-    batch_size: parseInt(process.env.RAG_BATCH_SIZE || modelDefaults.batch_size.toString(), 10),
+    // Core settings
+    chunk_size: parseInt(process.env.RAG_CHUNK_SIZE || '250', 10),
+    chunk_overlap: parseInt(process.env.RAG_CHUNK_OVERLAP || '50', 10),
+    batch_size: parseInt(process.env.RAG_BATCH_SIZE || '16', 10),
     top_k: parseInt(process.env.RAG_TOP_K || '10', 10),
     db_file: process.env.RAG_DB_FILE || 'db.sqlite',
     index_file: process.env.RAG_INDEX_FILE || 'vector-index.bin',
-    rerank_enabled: process.env.RAG_RERANK_ENABLED === 'true',
-    preprocessing: defaultPreprocessingConfig,
     model_cache_path: process.env.RAG_MODEL_CACHE_PATH || getDefaultModelCachePath(),
-    path_storage_strategy: process.env.RAG_PATH_STORAGE_STRATEGY || 'relative'
-};
-/**
- * Standard exit codes for different error conditions
- */
-export const EXIT_CODES = {
-    SUCCESS: 0,
-    GENERAL_ERROR: 1,
-    INVALID_ARGUMENTS: 2,
-    CONFIGURATION_ERROR: 3,
-    FILE_NOT_FOUND: 4,
-    DATABASE_ERROR: 5,
-    MODEL_ERROR: 6,
-    INDEX_ERROR: 7,
-    PERMISSION_ERROR: 8
-};
-/**
- * Configuration validation error with specific exit code
- */
-export class ConfigurationError extends Error {
-    exitCode;
-    constructor(message, exitCode = EXIT_CODES.CONFIGURATION_ERROR) {
-        super(message);
-        this.exitCode = exitCode;
-        this.name = 'ConfigurationError';
-    }
-}
-/**
- * Validates preprocessing configuration
- * @param config - Preprocessing configuration to validate
- * @throws {ConfigurationError} If preprocessing configuration is invalid
- */
-export function validatePreprocessingConfig(config) {
-    if (!config || typeof config !== 'object') {
-        throw new ConfigurationError('Preprocessing configuration must be an object');
-    }
-    // Validate mode
-    const validModes = ['strict', 'balanced', 'rich'];
-    if (!config.mode || !validModes.includes(config.mode)) {
-        throw new ConfigurationError(`Configuration error: preprocessing.mode must be one of: ${validModes.join(', ')}.\n` +
-            `Current value: ${JSON.stringify(config.mode)}\n` +
-            `Please set it to 'strict', 'balanced', or 'rich'.`);
+    path_storage_strategy: process.env.RAG_PATH_STORAGE_STRATEGY || 'relative',
+    // Text-specific settings
+    embedding_model: process.env.RAG_EMBEDDING_MODEL || 'sentence-transformers/all-MiniLM-L6-v2',
+    rerank_enabled: process.env.RAG_RERANK_ENABLED !== 'false', // Default to true unless explicitly disabled
+    rerank_model: process.env.RAG_RERANK_MODEL || 'cross-encoder/ms-marco-MiniLM-L-6-v2',
+    // Preprocessing settings
+    preprocessing: {
+        enabled: process.env.RAG_PREPROCESSING_ENABLED !== 'false',
+        mdx: process.env.RAG_PREPROCESSING_MDX !== 'false',
+        mermaid: process.env.RAG_PREPROCESSING_MERMAID !== 'false',
+        code_blocks: process.env.RAG_PREPROCESSING_CODE_BLOCKS !== 'false'
     }
-    // Validate overrides if present
-    if (config.overrides !== undefined) {
-        if (typeof config.overrides !== 'object' || config.overrides === null) {
-            throw new ConfigurationError(`Configuration error: preprocessing.overrides must be an object.\n` +
-                `Current value: ${JSON.stringify(config.overrides)}`);
-        }
-        // Validate MDX override
-        if (config.overrides.mdx !== undefined) {
-            const validMdxOptions = ['strip', 'keep', 'placeholder'];
-            if (!validMdxOptions.includes(config.overrides.mdx)) {
-                throw new ConfigurationError(`Configuration error: preprocessing.overrides.mdx must be one of: ${validMdxOptions.join(', ')}.\n` +
-                    `Current value: ${JSON.stringify(config.overrides.mdx)}`);
-            }
-        }
-        // Validate Mermaid override
-        if (config.overrides.mermaid !== undefined) {
-            const validMermaidOptions = ['strip', 'extract', 'placeholder'];
-            if (!validMermaidOptions.includes(config.overrides.mermaid)) {
-                throw new ConfigurationError(`Configuration error: preprocessing.overrides.mermaid must be one of: ${validMermaidOptions.join(', ')}.\n` +
-                    `Current value: ${JSON.stringify(config.overrides.mermaid)}`);
-            }
-        }
-        // Validate code override
-        if (config.overrides.code !== undefined) {
-            const validCodeOptions = ['strip', 'keep', 'placeholder'];
-            if (!validCodeOptions.includes(config.overrides.code)) {
-                throw new ConfigurationError(`Configuration error: preprocessing.overrides.code must be one of: ${validCodeOptions.join(', ')}.\n` +
-                    `Current value: ${JSON.stringify(config.overrides.code)}`);
-            }
-        }
-    }
-}
-/**
- * Merges preprocessing mode with overrides to create final configuration
- * @param config - Base preprocessing configuration
- * @returns Resolved preprocessing options for each content type
- */
-export function mergePreprocessingConfig(config) {
-    // Define mode defaults
-    const modeDefaults = {
-        strict: {
-            mdx: 'strip',
-            mermaid: 'strip',
-            code: 'strip'
-        },
-        balanced: {
-            mdx: 'placeholder',
-            mermaid: 'placeholder',
-            code: 'keep'
-        },
-        rich: {
-            mdx: 'keep',
-            mermaid: 'extract',
-            code: 'keep'
-        }
-    };
-    // Start with mode defaults
-    const result = { ...modeDefaults[config.mode] };
-    // Apply overrides (shallow override only)
-    if (config.overrides) {
-        if (config.overrides.mdx !== undefined) {
-            result.mdx = config.overrides.mdx;
-        }
-        if (config.overrides.mermaid !== undefined) {
-            result.mermaid = config.overrides.mermaid;
-        }
-        if (config.overrides.code !== undefined) {
-            result.code = config.overrides.code;
-        }
-    }
-    return result;
-}
+};
+// Re-export everything from core config
+export * from './core/config.js';
 /**
- * Validates the configuration object
- * @param config - Configuration object to validate
- * @throws {ConfigurationError} If configuration is invalid
+ * Validate complete configuration including text-specific settings
  */
 export function validateConfig(config) {
-    if (!config || typeof config !== 'object') {
-        throw new ConfigurationError('Configuration must be an object');
-    }
-    // Check required string fields
-    const requiredStrings = ['embedding_model', 'db_file', 'index_file'];
-    for (const field of requiredStrings) {
-        if (!config[field] || typeof config[field] !== 'string') {
-            throw new ConfigurationError(`Configuration error: '${field}' must be a non-empty string.\n` +
-                `Current value: ${JSON.stringify(config[field])}\n` +
-                `Please check your configuration file.`);
-        }
-    }
-    // Validate path_storage_strategy
-    if (!['absolute', 'relative'].includes(config.path_storage_strategy)) {
-        throw new ConfigurationError(`Configuration error: 'path_storage_strategy' must be either 'absolute' or 'relative'.\n` +
-            `Current value: ${JSON.stringify(config.path_storage_strategy)}\n` +
-            `Please set it to 'absolute' or 'relative'.`);
+    // First validate core config
+    const { validateCoreConfig } = require('./core/config.js');
+    validateCoreConfig(config);
+    // Validate text-specific settings
+    if (!config.embedding_model || typeof config.embedding_model !== 'string') {
+        throw new Error('embedding_model must be a non-empty string');
     }
-    // Check required numeric fields are positive
-    const requiredNumbers = ['chunk_size', 'chunk_overlap', 'batch_size', 'top_k'];
-    for (const field of requiredNumbers) {
-        if (typeof config[field] !== 'number' || config[field] <= 0) {
-            throw new ConfigurationError(`Configuration error: '${field}' must be a positive number.\n` +
-                `Current value: ${JSON.stringify(config[field])}\n` +
-                `Please ensure all numeric values are greater than 0.`);
-        }
-    }
-    // Check boolean fields
     if (typeof config.rerank_enabled !== 'boolean') {
-        throw new ConfigurationError(`Configuration error: 'rerank_enabled' must be a boolean (true or false).\n` +
-            `Current value: ${JSON.stringify(config.rerank_enabled)}\n` +
-            `Please set it to either true or false.`);
-    }
-    // Validate preprocessing configuration
-    validatePreprocessingConfig(config.preprocessing);
-    // Validate optional model_cache_path field
-    if (config.model_cache_path !== undefined && (typeof config.model_cache_path !== 'string' || config.model_cache_path.trim() === '')) {
-        throw new ConfigurationError(`Configuration error: 'model_cache_path' must be a non-empty string when provided.\n` +
-            `Current value: ${JSON.stringify(config.model_cache_path)}\n` +
-            `Please provide a valid directory path or remove the field to use default caching.`);
-    }
-    // Validate chunk_overlap is less than chunk_size
-    if (config.chunk_overlap >= config.chunk_size) {
-        throw new ConfigurationError(`Configuration error: chunk_overlap (${config.chunk_overlap}) must be less than chunk_size (${config.chunk_size}).\n` +
-            `Recommended: Set chunk_overlap to about 20% of chunk_size (e.g., chunk_size: 250, chunk_overlap: 50).`);
+        throw new Error('rerank_enabled must be a boolean');
     }
-    // Validate reasonable ranges for performance
-    if (config.chunk_size > 1000) {
-        console.warn(`Warning: Large chunk_size (${config.chunk_size}) may impact performance. Recommended range: 200-400 tokens.`);
+    if (!config.rerank_model || typeof config.rerank_model !== 'string') {
+        throw new Error('rerank_model must be a non-empty string');
     }
-    if (config.batch_size > 64) {
-        console.warn(`Warning: Large batch_size (${config.batch_size}) may cause memory issues. Recommended range: 8-32.`);
+    if (!config.preprocessing || typeof config.preprocessing !== 'object') {
+        throw new Error('preprocessing must be an object');
     }
 }
 /**
- * Utility function to handle unrecoverable errors with descriptive messages
- * Logs error and exits immediately with appropriate exit code
- * @param error - Error object or message
- * @param context - Context where the error occurred
- * @param exitCode - Exit code to use (defaults to GENERAL_ERROR)
+ * Validate preprocessing configuration
  */
-export function handleUnrecoverableError(error, context, exitCode = EXIT_CODES.GENERAL_ERROR) {
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    console.error(`\nFatal Error in ${context}:`);
-    console.error(errorMessage);
-    console.error('\nThe system cannot continue and will exit immediately.');
-    // Provide context-specific guidance
-    switch (exitCode) {
-        case EXIT_CODES.CONFIGURATION_ERROR:
-            console.error('\nPlease check your configuration and try again.');
-            break;
-        case EXIT_CODES.DATABASE_ERROR:
-            console.error('\nTry running "raglite rebuild" to fix database issues.');
-            break;
-        case EXIT_CODES.MODEL_ERROR:
-            console.error('\nEnsure you have internet connection for model download and sufficient disk space.');
-            break;
-        case EXIT_CODES.INDEX_ERROR:
-            console.error('\nTry running "raglite rebuild" to recreate the vector index.');
-            break;
-        case EXIT_CODES.FILE_NOT_FOUND:
-            console.error('\nPlease check that the specified files or directories exist and are accessible.');
-            break;
-        case EXIT_CODES.PERMISSION_ERROR:
-            console.error('\nPlease check file and directory permissions.');
-            break;
-        default:
-            console.error('\nIf this problem persists, please report it as a bug.');
-    }
-    process.exit(exitCode);
+export function validatePreprocessingConfig(config) {
+    return config &&
+        typeof config === 'object' &&
+        typeof config.enabled === 'boolean' &&
+        typeof config.mdx === 'boolean' &&
+        typeof config.mermaid === 'boolean' &&
+        typeof config.code_blocks === 'boolean';
 }
 /**
- * Utility function for safe error logging with context
- * @param error - Error to log
- * @param context - Context where error occurred
- * @param skipError - Whether to skip this error and continue (default: false)
+ * Merge preprocessing configurations
  */
-export function logError(error, context, skipError = false) {
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    if (skipError) {
-        console.error(`Warning in ${context}: ${errorMessage} (skipping and continuing)`);
-    }
-    else {
-        console.error(`Error in ${context}: ${errorMessage}`);
-    }
+export function mergePreprocessingConfig(base, override) {
+    return {
+        enabled: override.enabled !== undefined ? override.enabled : base.enabled,
+        mdx: override.mdx !== undefined ? override.mdx : base.mdx,
+        mermaid: override.mermaid !== undefined ? override.mermaid : base.mermaid,
+        code_blocks: override.code_blocks !== undefined ? override.code_blocks : base.code_blocks
+    };
 }
-// Validate the default config on module load
-validateConfig(config);
 //# sourceMappingURL=config.js.map

package/dist/core/abstract-embedder.d.ts

@@ -0,0 +1,125 @@
+/**
+ * CORE MODULE — Abstract Base Embedder
+ *
+ * Provides model-agnostic base functionality for all embedder implementations.
+ * This is an abstract base class, not a concrete implementation.
+ *
+ * ARCHITECTURAL NOTE:
+ * While this contains implementation logic, it remains in the core layer because:
+ * 1. It's model-agnostic (no knowledge of specific models or transformers.js)
+ * 2. It's shared by multiple implementation layers (text, multimodal)
+ * 3. It provides common infrastructure (lifecycle, validation, batch processing)
+ * 4. Moving it would create awkward cross-layer dependencies
+ *
+ * This follows the "shared base class" pattern common in framework design,
+ * similar to React.Component, Django Model, or other framework base classes.
+ *
+ * RESPONSIBILITIES:
+ * - Model lifecycle management (loading, cleanup, disposal)
+ * - Batch processing coordination
+ * - Input validation and text truncation
+ * - Error handling with helpful messages
+ * - Embedding ID generation
+ * - Common utility methods
+ *
+ * IMPLEMENTATION LAYERS:
+ * - Text: SentenceTransformerEmbedder extends this class
+ * - Multimodal: CLIPEmbedder extends this class
+ */
+import type { UniversalEmbedder, ModelInfo, ModelType, EmbeddingBatchItem } from './universal-embedder.js';
+import type { EmbeddingResult } from '../types.js';
+/**
+ * Abstract base class for universal embedders
+ * Provides common functionality and lifecycle management
+ */
+export declare abstract class BaseUniversalEmbedder implements UniversalEmbedder {
+    readonly modelName: string;
+    protected readonly options: EmbedderOptions;
+    protected _isLoaded: boolean;
+    protected _modelInfo: ModelInfo;
+    constructor(modelName: string, options?: EmbedderOptions);
+    get modelType(): ModelType;
+    get dimensions(): number;
+    get supportedContentTypes(): readonly string[];
+    isLoaded(): boolean;
+    getModelInfo(): ModelInfo;
+    /**
+     * Load the model - must be implemented by subclasses
+     */
+    abstract loadModel(): Promise<void>;
+    /**
+     * Embed text content - must be implemented by subclasses
+     */
+    abstract embedText(text: string): Promise<EmbeddingResult>;
+    /**
+     * Clean up resources - must be implemented by subclasses
+     */
+    abstract cleanup(): Promise<void>;
+    /**
+     * Dispose of all resources and prepare for garbage collection
+     * This method should be called when the embedder is no longer needed
+     */
+    dispose(): Promise<void>;
+    /**
+     * Embed image content - optional, only implemented by multimodal embedders
+     */
+    embedImage?(imagePath: string): Promise<EmbeddingResult>;
+    /**
+     * Batch embedding with default implementation
+     * Subclasses can override for more efficient batch processing
+     */
+    embedBatch(items: EmbeddingBatchItem[]): Promise<EmbeddingResult[]>;
+    /**
+     * Process a single batch of items
+     * Can be overridden by subclasses for more efficient batch processing
+     */
+    protected processBatch(batch: EmbeddingBatchItem[]): Promise<EmbeddingResult[]>;
+    /**
+     * Validate that the model is loaded before operations
+     */
+    protected ensureLoaded(): void;
+    /**
+     * Generate a unique embedding ID
+     */
+    protected generateEmbeddingId(content: string, contentType?: string): string;
+    /**
+     * Simple hash function for content identification
+     */
+    private simpleHash;
+    /**
+     * Validate text length against model constraints
+     */
+    protected validateTextLength(text: string): void;
+    /**
+     * Truncate text to model's maximum length
+     */
+    protected truncateText(text: string): string;
+    /**
+     * Log model loading progress
+     */
+    protected logModelLoading(stage: string, details?: string): void;
+    /**
+     * Handle model loading errors with helpful messages
+     */
+    protected handleLoadingError(error: Error): Error;
+}
+/**
+ * Options for configuring embedder instances
+ */
+export interface EmbedderOptions {
+    cachePath?: string;
+    maxBatchSize?: number;
+    timeout?: number;
+    enableGPU?: boolean;
+    customConfig?: Record<string, any>;
+    logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'silent';
+}
+/**
+ * Create embedder options with defaults
+ */
+export declare function createEmbedderOptions(options?: Partial<EmbedderOptions>): EmbedderOptions;
+/**
+ * Validate embedder options
+ */
+export declare function validateEmbedderOptions(options: EmbedderOptions): void;
+//# sourceMappingURL=abstract-embedder.d.ts.map