rag-lite-ts 2.1.0 → 2.2.0

package/dist/esm/core/cli-database-utils.js

@@ -0,0 +1,239 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+import { DatabaseConnectionManager } from './database-connection-manager.js';
+import { existsSync } from 'fs';
+/**
+ * Default options for CLI database access
+ */
+const DEFAULT_CLI_OPTIONS = {
+    maxWaitMs: 10000, // 10 seconds
+    retryIntervalMs: 500, // 0.5 seconds
+    showProgress: true,
+    commandName: 'CLI command'
+};
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export async function waitForCLIDatabaseAccess(dbPath, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    // Check if database file exists
+    if (!existsSync(dbPath)) {
+        throw new Error(`Database file not found: ${dbPath}\n` +
+            `Please run 'raglite ingest <path>' first to create the database.`);
+    }
+    const startTime = Date.now();
+    let attempts = 0;
+    let lastError = null;
+    while (Date.now() - startTime < opts.maxWaitMs) {
+        attempts++;
+        try {
+            // Try to get database access
+            await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 1000);
+            if (opts.showProgress && attempts > 1) {
+                console.log(`✅ Database is now available (after ${attempts} attempts)`);
+            }
+            return; // Success!
+        }
+        catch (error) {
+            lastError = error;
+            if (lastError.message.includes('SQLITE_BUSY') || lastError.message.includes('busy')) {
+                // Database is busy - show progress and retry
+                if (opts.showProgress) {
+                    if (attempts === 1) {
+                        console.log(`⏳ Database is busy, waiting for access...`);
+                        console.log(`   This usually happens when another process is using the database.`);
+                        console.log(`   Common causes:`);
+                        console.log(`   • MCP server is running`);
+                        console.log(`   • Another CLI command is in progress`);
+                        console.log(`   • Long-running ingestion process`);
+                        console.log('');
+                    }
+                    else if (attempts % 4 === 0) {
+                        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+                        console.log(`   Still waiting... (${elapsed}s elapsed, attempt ${attempts})`);
+                    }
+                }
+                // Wait before retrying
+                await new Promise(resolve => setTimeout(resolve, opts.retryIntervalMs));
+                continue;
+            }
+            else {
+                // Other error - don't retry
+                throw new Error(`Failed to access database: ${lastError.message}\n` +
+                    `Please check that the database file is not corrupted and you have proper permissions.`);
+            }
+        }
+    }
+    // Timeout reached
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    throw new Error(`Database is still busy after ${elapsed} seconds.\n` +
+        `\n` +
+        `This might be because:\n` +
+        `• Another process is using the database (MCP server, long ingestion, etc.)\n` +
+        `• The database is locked due to an interrupted operation\n` +
+        `\n` +
+        `Solutions:\n` +
+        `• Wait for other operations to complete\n` +
+        `• Stop the MCP server if running\n` +
+        `• Restart your terminal/process\n` +
+        `• As a last resort, restart your computer\n` +
+        `\n` +
+        `Last error: ${lastError?.message || 'Unknown error'}`);
+}
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export async function withCLIDatabaseAccess(dbPath, operation, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    try {
+        // Wait for database access
+        await waitForCLIDatabaseAccess(dbPath, opts);
+        // Execute the operation
+        return await operation();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            // Enhance error message with CLI context
+            const enhancedMessage = `${opts.commandName} failed: ${error.message}\n` +
+                `\n` +
+                `If this error persists:\n` +
+                `• Check that no other RAG-lite processes are running\n` +
+                `• Verify database file permissions\n` +
+                `• Try running the command again\n`;
+            throw new Error(enhancedMessage);
+        }
+        throw error;
+    }
+}
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export async function isDatabaseBusy(dbPath) {
+    try {
+        await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 100);
+        return { isBusy: false };
+    }
+    catch (error) {
+        if (error instanceof Error && (error.message.includes('SQLITE_BUSY') ||
+            error.message.includes('busy'))) {
+            return {
+                isBusy: true,
+                reason: 'Database is currently in use by another process',
+                suggestions: [
+                    'Wait for other operations to complete',
+                    'Stop MCP server if running',
+                    'Check for other CLI commands in progress'
+                ]
+            };
+        }
+        return {
+            isBusy: true,
+            reason: `Database access error: ${error instanceof Error ? error.message : String(error)}`,
+            suggestions: [
+                'Check database file permissions',
+                'Verify database file is not corrupted',
+                'Ensure you have read/write access to the database directory'
+            ]
+        };
+    }
+}
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export async function showDatabaseStatus(dbPath) {
+    console.log(`📊 Database Status: ${dbPath}`);
+    console.log('');
+    // Check file existence
+    if (!existsSync(dbPath)) {
+        console.log('❌ Database file does not exist');
+        console.log('   Run "raglite ingest <path>" to create the database');
+        return;
+    }
+    // Check file stats
+    try {
+        const fs = await import('fs');
+        const stats = fs.statSync(dbPath);
+        console.log(`📁 File size: ${(stats.size / 1024).toFixed(1)} KB`);
+        console.log(`📅 Last modified: ${stats.mtime.toLocaleString()}`);
+    }
+    catch (error) {
+        console.log(`⚠️  Cannot read file stats: ${error}`);
+    }
+    // Check database access
+    const busyStatus = await isDatabaseBusy(dbPath);
+    if (busyStatus.isBusy) {
+        console.log(`🔒 Status: BUSY`);
+        console.log(`   Reason: ${busyStatus.reason}`);
+        if (busyStatus.suggestions) {
+            console.log('   Suggestions:');
+            busyStatus.suggestions.forEach(suggestion => {
+                console.log(`   • ${suggestion}`);
+            });
+        }
+    }
+    else {
+        console.log(`✅ Status: AVAILABLE`);
+    }
+    // Show connection manager stats
+    const connectionStats = DatabaseConnectionManager.getConnectionStats();
+    if (connectionStats.totalConnections > 0) {
+        console.log('');
+        console.log(`🔗 Active connections: ${connectionStats.totalConnections}`);
+        connectionStats.connections.forEach((conn, index) => {
+            console.log(`   ${index + 1}. ${conn.path}`);
+            console.log(`      References: ${conn.refCount}`);
+            console.log(`      Last accessed: ${conn.lastAccessed.toLocaleString()}`);
+            console.log(`      Idle time: ${(conn.idleTime / 1000).toFixed(1)}s`);
+        });
+    }
+    console.log('');
+}
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export async function forceCleanupDatabase(dbPath) {
+    console.log(`🚨 Force cleaning up database connections: ${dbPath}`);
+    try {
+        await DatabaseConnectionManager.forceCloseConnection(dbPath);
+        console.log('✅ Force cleanup completed');
+    }
+    catch (error) {
+        console.log(`⚠️  Force cleanup failed: ${error}`);
+        console.log('You may need to restart the process or reboot your system');
+    }
+}
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export function setupCLICleanup(dbPath) {
+    const cleanup = async () => {
+        console.log('\n🛑 Shutting down gracefully...');
+        if (dbPath) {
+            try {
+                await DatabaseConnectionManager.releaseConnection(dbPath);
+            }
+            catch (error) {
+                // Ignore cleanup errors during shutdown
+            }
+        }
+        await DatabaseConnectionManager.closeAllConnections();
+        process.exit(0);
+    };
+    // Only set up handlers if they haven't been set up already
+    if (!process.listenerCount('SIGINT')) {
+        process.on('SIGINT', cleanup);
+    }
+    if (!process.listenerCount('SIGTERM')) {
+        process.on('SIGTERM', cleanup);
+    }
+}
+//# sourceMappingURL=cli-database-utils.js.map

package/dist/esm/core/config.d.ts

@@ -0,0 +1,102 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Core configuration interface - model-agnostic settings
+ */
+export interface CoreConfig {
+    chunk_size: number;
+    chunk_overlap: number;
+    batch_size: number;
+    top_k: number;
+    db_file: string;
+    index_file: string;
+    model_cache_path?: string;
+    path_storage_strategy: 'absolute' | 'relative';
+    embedding_model: string;
+    rerank_enabled: boolean;
+    preprocessing?: any;
+}
+/**
+ * Extensible configuration interface that can be extended by implementations
+ */
+export interface ExtensibleConfig<T = {}> extends CoreConfig {
+    implementation: T;
+}
+/**
+ * Standard exit codes for different error conditions
+ */
+export declare const EXIT_CODES: {
+    readonly SUCCESS: 0;
+    readonly GENERAL_ERROR: 1;
+    readonly INVALID_ARGUMENTS: 2;
+    readonly CONFIGURATION_ERROR: 3;
+    readonly FILE_NOT_FOUND: 4;
+    readonly DATABASE_ERROR: 5;
+    readonly MODEL_ERROR: 6;
+    readonly INDEX_ERROR: 7;
+    readonly PERMISSION_ERROR: 8;
+};
+/**
+ * Configuration validation error with specific exit code
+ */
+export declare class ConfigurationError extends Error {
+    exitCode: number;
+    constructor(message: string, exitCode?: number);
+}
+/**
+ * Get the default model cache path as specified in the requirements
+ * @returns Default cache path (~/.raglite/models/)
+ */
+export declare function getDefaultModelCachePath(): string;
+/**
+ * Validates core configuration fields
+ * @param config - Configuration object to validate
+ * @throws {ConfigurationError} If configuration is invalid
+ */
+export declare function validateCoreConfig(config: any): asserts config is CoreConfig;
+/**
+ * Model defaults interface for different embedding models
+ */
+export interface ModelDefaults {
+    dimensions: number;
+    chunk_size: number;
+    chunk_overlap: number;
+    batch_size: number;
+}
+/**
+ * Get default configuration for different embedding models
+ * @param modelName - Name of the embedding model
+ * @returns Model-specific defaults
+ */
+export declare function getModelDefaults(modelName?: string): ModelDefaults;
+/**
+ * Default core configuration object
+ * Model-agnostic settings that can be used by core modules
+ */
+export declare const config: CoreConfig;
+/**
+ * Validate preprocessing configuration
+ */
+export declare function validatePreprocessingConfig(config: any): asserts config is any;
+/**
+ * Merge preprocessing configurations with mode defaults
+ */
+export declare function mergePreprocessingConfig(config: any): any;
+/**
+ * Utility function to handle unrecoverable errors with descriptive messages
+ * Logs error and exits immediately with appropriate exit code
+ * @param error - Error object or message
+ * @param context - Context where the error occurred
+ * @param exitCode - Exit code to use (defaults to GENERAL_ERROR)
+ */
+export declare function handleUnrecoverableError(error: Error | string, context: string, exitCode?: number): never;
+/**
+ * Utility function for safe error logging with context
+ * @param error - Error to log
+ * @param context - Context where error occurred
+ * @param skipError - Whether to skip this error and continue (default: false)
+ */
+export declare function logError(error: Error | string, context: string, skipError?: boolean): void;
+//# sourceMappingURL=config.d.ts.map

package/dist/esm/core/config.js

@@ -0,0 +1,247 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+import { homedir } from 'os';
+import { join } from 'path';
+/**
+ * Standard exit codes for different error conditions
+ */
+export const EXIT_CODES = {
+    SUCCESS: 0,
+    GENERAL_ERROR: 1,
+    INVALID_ARGUMENTS: 2,
+    CONFIGURATION_ERROR: 3,
+    FILE_NOT_FOUND: 4,
+    DATABASE_ERROR: 5,
+    MODEL_ERROR: 6,
+    INDEX_ERROR: 7,
+    PERMISSION_ERROR: 8
+};
+/**
+ * Configuration validation error with specific exit code
+ */
+export class ConfigurationError extends Error {
+    exitCode;
+    constructor(message, exitCode = EXIT_CODES.CONFIGURATION_ERROR) {
+        super(message);
+        this.exitCode = exitCode;
+        this.name = 'ConfigurationError';
+    }
+}
+/**
+ * Get the default model cache path as specified in the requirements
+ * @returns Default cache path (~/.raglite/models/)
+ */
+export function getDefaultModelCachePath() {
+    return join(homedir(), '.raglite', 'models');
+}
+/**
+ * Validates core configuration fields
+ * @param config - Configuration object to validate
+ * @throws {ConfigurationError} If configuration is invalid
+ */
+export function validateCoreConfig(config) {
+    if (!config || typeof config !== 'object') {
+        throw new ConfigurationError('Configuration must be an object');
+    }
+    // Check required string fields
+    const requiredStrings = ['db_file', 'index_file'];
+    for (const field of requiredStrings) {
+        if (!config[field] || typeof config[field] !== 'string') {
+            throw new ConfigurationError(`Configuration error: '${field}' must be a non-empty string.\n` +
+                `Current value: ${JSON.stringify(config[field])}\n` +
+                `Please check your configuration file.`);
+        }
+    }
+    // Validate path_storage_strategy
+    if (!['absolute', 'relative'].includes(config.path_storage_strategy)) {
+        throw new ConfigurationError(`Configuration error: 'path_storage_strategy' must be either 'absolute' or 'relative'.\n` +
+            `Current value: ${JSON.stringify(config.path_storage_strategy)}\n` +
+            `Please set it to 'absolute' or 'relative'.`);
+    }
+    // Check required numeric fields are positive
+    const requiredNumbers = ['chunk_size', 'chunk_overlap', 'batch_size', 'top_k'];
+    for (const field of requiredNumbers) {
+        if (typeof config[field] !== 'number' || config[field] <= 0) {
+            throw new ConfigurationError(`Configuration error: '${field}' must be a positive number.\n` +
+                `Current value: ${JSON.stringify(config[field])}\n` +
+                `Please ensure all numeric values are greater than 0.`);
+        }
+    }
+    // Validate optional model_cache_path field
+    if (config.model_cache_path !== undefined && (typeof config.model_cache_path !== 'string' || config.model_cache_path.trim() === '')) {
+        throw new ConfigurationError(`Configuration error: 'model_cache_path' must be a non-empty string when provided.\n` +
+            `Current value: ${JSON.stringify(config.model_cache_path)}\n` +
+            `Please provide a valid directory path or remove the field to use default caching.`);
+    }
+    // Validate chunk_overlap is less than chunk_size
+    if (config.chunk_overlap >= config.chunk_size) {
+        throw new ConfigurationError(`Configuration error: chunk_overlap (${config.chunk_overlap}) must be less than chunk_size (${config.chunk_size}).\n` +
+            `Recommended: Set chunk_overlap to about 20% of chunk_size (e.g., chunk_size: 250, chunk_overlap: 50).`);
+    }
+    // Validate reasonable ranges for performance
+    if (config.chunk_size > 1000) {
+        console.warn(`Warning: Large chunk_size (${config.chunk_size}) may impact performance. Recommended range: 200-400 tokens.`);
+    }
+    if (config.batch_size > 64) {
+        console.warn(`Warning: Large batch_size (${config.batch_size}) may cause memory issues. Recommended range: 8-32.`);
+    }
+}
+/**
+ * Get default configuration for different embedding models
+ * @param modelName - Name of the embedding model
+ * @returns Model-specific defaults
+ */
+export function getModelDefaults(modelName) {
+    // Default configuration for most models
+    const defaults = {
+        dimensions: 384, // Most common dimension for sentence transformers
+        chunk_size: 250,
+        chunk_overlap: 50,
+        batch_size: 16
+    };
+    // Model-specific overrides based on model name heuristics
+    if (modelName) {
+        const normalizedName = modelName.toLowerCase();
+        // CLIP models - 512 dimensions
+        if (normalizedName.includes('clip')) {
+            defaults.dimensions = 512;
+            defaults.batch_size = 8;
+        }
+        // MPNet models - 768 dimensions
+        else if (normalizedName.includes('all-mpnet-base-v2')) {
+            defaults.dimensions = 768;
+            defaults.chunk_size = 400;
+            defaults.chunk_overlap = 80;
+            defaults.batch_size = 8;
+        }
+        else if (normalizedName.includes('mpnet') || normalizedName.includes('768')) {
+            defaults.dimensions = 768;
+        }
+        // Models with 512 in the name
+        else if (normalizedName.includes('512')) {
+            defaults.dimensions = 512;
+        }
+        // MiniLM and other 384-dimensional models (default)
+        else if (normalizedName.includes('384') || normalizedName.includes('minilm')) {
+            defaults.dimensions = 384;
+        }
+    }
+    return defaults;
+}
+/**
+ * Default core configuration object
+ * Model-agnostic settings that can be used by core modules
+ */
+export const config = {
+    chunk_size: parseInt(process.env.RAG_CHUNK_SIZE || '250', 10),
+    chunk_overlap: parseInt(process.env.RAG_CHUNK_OVERLAP || '50', 10),
+    batch_size: parseInt(process.env.RAG_BATCH_SIZE || '16', 10),
+    top_k: parseInt(process.env.RAG_TOP_K || '10', 10),
+    db_file: process.env.RAG_DB_FILE || 'db.sqlite',
+    index_file: process.env.RAG_INDEX_FILE || 'vector-index.bin',
+    model_cache_path: process.env.RAG_MODEL_CACHE_PATH || getDefaultModelCachePath(),
+    path_storage_strategy: process.env.RAG_PATH_STORAGE_STRATEGY || 'relative',
+    embedding_model: process.env.RAG_EMBEDDING_MODEL || 'sentence-transformers/all-MiniLM-L6-v2',
+    rerank_enabled: process.env.RAG_RERANK_ENABLED === 'true',
+    preprocessing: { mode: 'balanced' }
+};
+/**
+ * Validate preprocessing configuration
+ */
+export function validatePreprocessingConfig(config) {
+    if (!config || typeof config !== 'object') {
+        throw new ConfigurationError('Preprocessing configuration must be an object');
+    }
+    if (!config.mode || !['strict', 'balanced', 'rich'].includes(config.mode)) {
+        throw new ConfigurationError('preprocessing.mode must be one of: strict, balanced, rich');
+    }
+    // Validate overrides if present
+    if (config.overrides !== undefined) {
+        if (!config.overrides || typeof config.overrides !== 'object') {
+            throw new ConfigurationError('preprocessing.overrides must be an object when provided');
+        }
+        const validValues = ['strip', 'keep', 'placeholder', 'extract'];
+        // Validate MDX override
+        if (config.overrides.mdx !== undefined && !validValues.includes(config.overrides.mdx)) {
+            throw new ConfigurationError('preprocessing.overrides.mdx must be one of: strip, keep, placeholder');
+        }
+        // Validate Mermaid override
+        if (config.overrides.mermaid !== undefined && !validValues.includes(config.overrides.mermaid)) {
+            throw new ConfigurationError('preprocessing.overrides.mermaid must be one of: strip, keep, placeholder, extract');
+        }
+        // Validate code override
+        if (config.overrides.code !== undefined && !validValues.includes(config.overrides.code)) {
+            throw new ConfigurationError('preprocessing.overrides.code must be one of: strip, keep, placeholder');
+        }
+    }
+}
+/**
+ * Merge preprocessing configurations with mode defaults
+ */
+export function mergePreprocessingConfig(config) {
+    const modeDefaults = {
+        strict: { mdx: 'strip', mermaid: 'strip', code: 'strip' },
+        balanced: { mdx: 'placeholder', mermaid: 'placeholder', code: 'keep' },
+        rich: { mdx: 'keep', mermaid: 'extract', code: 'keep' }
+    };
+    const defaults = modeDefaults[config.mode] || modeDefaults.balanced;
+    return {
+        ...defaults,
+        ...config.overrides
+    };
+}
+/**
+ * Utility function to handle unrecoverable errors with descriptive messages
+ * Logs error and exits immediately with appropriate exit code
+ * @param error - Error object or message
+ * @param context - Context where the error occurred
+ * @param exitCode - Exit code to use (defaults to GENERAL_ERROR)
+ */
+export function handleUnrecoverableError(error, context, exitCode = EXIT_CODES.GENERAL_ERROR) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    console.error(`\nFatal Error in ${context}:`);
+    console.error(errorMessage);
+    console.error('\nThe system cannot continue and will exit immediately.');
+    // Provide context-specific guidance
+    switch (exitCode) {
+        case EXIT_CODES.CONFIGURATION_ERROR:
+            console.error('\nPlease check your configuration and try again.');
+            break;
+        case EXIT_CODES.DATABASE_ERROR:
+            console.error('\nTry running "raglite rebuild" to fix database issues.');
+            break;
+        case EXIT_CODES.MODEL_ERROR:
+            console.error('\nEnsure you have internet connection for model download and sufficient disk space.');
+            break;
+        case EXIT_CODES.INDEX_ERROR:
+            console.error('\nTry running "raglite rebuild" to recreate the vector index.');
+            break;
+        case EXIT_CODES.FILE_NOT_FOUND:
+            console.error('\nPlease check that the specified files or directories exist and are accessible.');
+            break;
+        case EXIT_CODES.PERMISSION_ERROR:
+            console.error('\nPlease check file and directory permissions.');
+            break;
+        default:
+            console.error('\nIf this problem persists, please report it as a bug.');
+    }
+    process.exit(exitCode);
+}
+/**
+ * Utility function for safe error logging with context
+ * @param error - Error to log
+ * @param context - Context where error occurred
+ * @param skipError - Whether to skip this error and continue (default: false)
+ */
+export function logError(error, context, skipError = false) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    if (skipError) {
+        console.error(`Warning in ${context}: ${errorMessage} (skipping and continuing)`);
+    }
+    else {
+        console.error(`Error in ${context}: ${errorMessage}`);
+    }
+}
+//# sourceMappingURL=config.js.map