rag-lite-ts 1.0.1 → 2.0.0

package/dist/core/chunker.d.ts

@@ -0,0 +1,119 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Configuration for chunking behavior
+ */
+export interface ChunkConfig {
+    /** Target chunk size in tokens (200-300 recommended) */
+    chunkSize: number;
+    /** Overlap between chunks in tokens (50 recommended) */
+    chunkOverlap: number;
+}
+/**
+ * Generic document interface that can represent different content types
+ */
+export interface GenericDocument {
+    /** Source path or identifier */
+    source: string;
+    /** Document title */
+    title: string;
+    /** Content (text, image path, etc.) */
+    content: string;
+    /** Content type identifier (text, image, etc.) */
+    contentType: string;
+    /** Optional metadata for the document */
+    metadata?: Record<string, any>;
+}
+/**
+ * Generic chunk interface that can represent different content types
+ */
+export interface GenericChunk {
+    /** The content of the chunk (text, image path, etc.) */
+    content: string;
+    /** Content type identifier (text, image, etc.) */
+    contentType: string;
+    /** Index of this chunk within the document */
+    chunkIndex: number;
+    /** Optional metadata for the chunk */
+    metadata?: Record<string, any>;
+}
+/**
+ * Strategy interface for chunking different content types
+ */
+export interface ChunkingStrategy {
+    /**
+     * Check if this strategy applies to the given content type
+     */
+    appliesTo(contentType: string): boolean;
+    /**
+     * Chunk a document using this strategy
+     */
+    chunk(document: GenericDocument, config: ChunkConfig): Promise<GenericChunk[]>;
+}
+/**
+ * Registry for chunking strategies
+ */
+export declare class ChunkingStrategyRegistry {
+    private strategies;
+    /**
+     * Register a chunking strategy
+     */
+    register(strategy: ChunkingStrategy): void;
+    /**
+     * Find the appropriate strategy for a content type
+     */
+    findStrategy(contentType: string): ChunkingStrategy | undefined;
+    /**
+     * Get all registered strategies
+     */
+    getStrategies(): ChunkingStrategy[];
+}
+/**
+ * Default chunking configuration
+ */
+export declare const DEFAULT_CHUNK_CONFIG: ChunkConfig;
+/**
+ * Global chunking strategy registry
+ */
+export declare const chunkingRegistry: ChunkingStrategyRegistry;
+/**
+ * Generic chunking function that uses registered strategies
+ */
+export declare function chunkGenericDocument(document: GenericDocument, config?: ChunkConfig): Promise<GenericChunk[]>;
+/**
+ * Document interface for text chunking
+ */
+export interface Document {
+    /** Source path or identifier */
+    source: string;
+    /** Document title */
+    title: string;
+    /** Full text content */
+    content: string;
+    /** Optional metadata */
+    metadata?: Record<string, any>;
+}
+/**
+ * Chunk interface for text chunking results
+ */
+export interface Chunk {
+    /** The text content of the chunk */
+    text: string;
+    /** Index of this chunk within the document */
+    chunkIndex: number;
+    /** Number of tokens in this chunk */
+    tokenCount: number;
+}
+/**
+ * Text document chunking function
+ * Uses the text chunking strategy from the text implementation layer
+ */
+export declare function chunkDocument(document: Document, config?: ChunkConfig): Promise<Chunk[]>;
+/**
+ * Register the text chunking strategy with the global registry
+ * This should be called during application initialization
+ */
+export declare function registerTextChunkingStrategy(): Promise<void>;
+//# sourceMappingURL=chunker.d.ts.map

package/dist/core/chunker.js

@@ -0,0 +1,73 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Registry for chunking strategies
+ */
+export class ChunkingStrategyRegistry {
+    strategies = [];
+    /**
+     * Register a chunking strategy
+     */
+    register(strategy) {
+        this.strategies.push(strategy);
+    }
+    /**
+     * Find the appropriate strategy for a content type
+     */
+    findStrategy(contentType) {
+        return this.strategies.find(strategy => strategy.appliesTo(contentType));
+    }
+    /**
+     * Get all registered strategies
+     */
+    getStrategies() {
+        return [...this.strategies];
+    }
+}
+/**
+ * Default chunking configuration
+ */
+export const DEFAULT_CHUNK_CONFIG = {
+    chunkSize: 250, // Target 200-300 tokens
+    chunkOverlap: 50
+};
+/**
+ * Global chunking strategy registry
+ */
+export const chunkingRegistry = new ChunkingStrategyRegistry();
+/**
+ * Generic chunking function that uses registered strategies
+ */
+export async function chunkGenericDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    const strategy = chunkingRegistry.findStrategy(document.contentType);
+    if (!strategy) {
+        throw new Error(`No chunking strategy found for content type: ${document.contentType}`);
+    }
+    return strategy.chunk(document, config);
+}
+/**
+ * Text document chunking function
+ * Uses the text chunking strategy from the text implementation layer
+ */
+export async function chunkDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    // Import the text chunker implementation dynamically to avoid circular dependencies
+    const { chunkDocument: textChunkDocument } = await import('../text/chunker.js');
+    return textChunkDocument(document, config);
+}
+/**
+ * Register the text chunking strategy with the global registry
+ * This should be called during application initialization
+ */
+export async function registerTextChunkingStrategy() {
+    const { TextChunkingStrategy } = await import('../text/chunker.js');
+    const textStrategy = new TextChunkingStrategy();
+    chunkingRegistry.register(textStrategy);
+}
+// Auto-register the text strategy when this module is loaded
+// This ensures text chunking works out of the box
+registerTextChunkingStrategy().catch(error => {
+    console.warn('Failed to register text chunking strategy:', error);
+});
+//# sourceMappingURL=chunker.js.map

package/dist/core/cli-database-utils.d.ts

@@ -0,0 +1,53 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+/**
+ * CLI-specific database access options
+ */
+export interface CLIDatabaseOptions {
+    /** Maximum time to wait for database access (ms) */
+    maxWaitMs?: number;
+    /** Retry interval (ms) */
+    retryIntervalMs?: number;
+    /** Show progress messages to user */
+    showProgress?: boolean;
+    /** Command name for better error messages */
+    commandName?: string;
+}
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export declare function waitForCLIDatabaseAccess(dbPath: string, options?: CLIDatabaseOptions): Promise<void>;
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export declare function withCLIDatabaseAccess<T>(dbPath: string, operation: () => Promise<T>, options?: CLIDatabaseOptions): Promise<T>;
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export declare function isDatabaseBusy(dbPath: string): Promise<{
+    isBusy: boolean;
+    reason?: string;
+    suggestions?: string[];
+}>;
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export declare function showDatabaseStatus(dbPath: string): Promise<void>;
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export declare function forceCleanupDatabase(dbPath: string): Promise<void>;
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export declare function setupCLICleanup(dbPath?: string): void;
+//# sourceMappingURL=cli-database-utils.d.ts.map

package/dist/core/cli-database-utils.js

@@ -0,0 +1,239 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+import { DatabaseConnectionManager } from './database-connection-manager.js';
+import { existsSync } from 'fs';
+/**
+ * Default options for CLI database access
+ */
+const DEFAULT_CLI_OPTIONS = {
+    maxWaitMs: 10000, // 10 seconds
+    retryIntervalMs: 500, // 0.5 seconds
+    showProgress: true,
+    commandName: 'CLI command'
+};
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export async function waitForCLIDatabaseAccess(dbPath, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    // Check if database file exists
+    if (!existsSync(dbPath)) {
+        throw new Error(`Database file not found: ${dbPath}\n` +
+            `Please run 'raglite ingest <path>' first to create the database.`);
+    }
+    const startTime = Date.now();
+    let attempts = 0;
+    let lastError = null;
+    while (Date.now() - startTime < opts.maxWaitMs) {
+        attempts++;
+        try {
+            // Try to get database access
+            await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 1000);
+            if (opts.showProgress && attempts > 1) {
+                console.log(`✅ Database is now available (after ${attempts} attempts)`);
+            }
+            return; // Success!
+        }
+        catch (error) {
+            lastError = error;
+            if (lastError.message.includes('SQLITE_BUSY') || lastError.message.includes('busy')) {
+                // Database is busy - show progress and retry
+                if (opts.showProgress) {
+                    if (attempts === 1) {
+                        console.log(`⏳ Database is busy, waiting for access...`);
+                        console.log(`   This usually happens when another process is using the database.`);
+                        console.log(`   Common causes:`);
+                        console.log(`   • MCP server is running`);
+                        console.log(`   • Another CLI command is in progress`);
+                        console.log(`   • Long-running ingestion process`);
+                        console.log('');
+                    }
+                    else if (attempts % 4 === 0) {
+                        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+                        console.log(`   Still waiting... (${elapsed}s elapsed, attempt ${attempts})`);
+                    }
+                }
+                // Wait before retrying
+                await new Promise(resolve => setTimeout(resolve, opts.retryIntervalMs));
+                continue;
+            }
+            else {
+                // Other error - don't retry
+                throw new Error(`Failed to access database: ${lastError.message}\n` +
+                    `Please check that the database file is not corrupted and you have proper permissions.`);
+            }
+        }
+    }
+    // Timeout reached
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    throw new Error(`Database is still busy after ${elapsed} seconds.\n` +
+        `\n` +
+        `This might be because:\n` +
+        `• Another process is using the database (MCP server, long ingestion, etc.)\n` +
+        `• The database is locked due to an interrupted operation\n` +
+        `\n` +
+        `Solutions:\n` +
+        `• Wait for other operations to complete\n` +
+        `• Stop the MCP server if running\n` +
+        `• Restart your terminal/process\n` +
+        `• As a last resort, restart your computer\n` +
+        `\n` +
+        `Last error: ${lastError?.message || 'Unknown error'}`);
+}
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export async function withCLIDatabaseAccess(dbPath, operation, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    try {
+        // Wait for database access
+        await waitForCLIDatabaseAccess(dbPath, opts);
+        // Execute the operation
+        return await operation();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            // Enhance error message with CLI context
+            const enhancedMessage = `${opts.commandName} failed: ${error.message}\n` +
+                `\n` +
+                `If this error persists:\n` +
+                `• Check that no other RAG-lite processes are running\n` +
+                `• Verify database file permissions\n` +
+                `• Try running the command again\n`;
+            throw new Error(enhancedMessage);
+        }
+        throw error;
+    }
+}
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export async function isDatabaseBusy(dbPath) {
+    try {
+        await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 100);
+        return { isBusy: false };
+    }
+    catch (error) {
+        if (error instanceof Error && (error.message.includes('SQLITE_BUSY') ||
+            error.message.includes('busy'))) {
+            return {
+                isBusy: true,
+                reason: 'Database is currently in use by another process',
+                suggestions: [
+                    'Wait for other operations to complete',
+                    'Stop MCP server if running',
+                    'Check for other CLI commands in progress'
+                ]
+            };
+        }
+        return {
+            isBusy: true,
+            reason: `Database access error: ${error instanceof Error ? error.message : String(error)}`,
+            suggestions: [
+                'Check database file permissions',
+                'Verify database file is not corrupted',
+                'Ensure you have read/write access to the database directory'
+            ]
+        };
+    }
+}
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export async function showDatabaseStatus(dbPath) {
+    console.log(`📊 Database Status: ${dbPath}`);
+    console.log('');
+    // Check file existence
+    if (!existsSync(dbPath)) {
+        console.log('❌ Database file does not exist');
+        console.log('   Run "raglite ingest <path>" to create the database');
+        return;
+    }
+    // Check file stats
+    try {
+        const fs = await import('fs');
+        const stats = fs.statSync(dbPath);
+        console.log(`📁 File size: ${(stats.size / 1024).toFixed(1)} KB`);
+        console.log(`📅 Last modified: ${stats.mtime.toLocaleString()}`);
+    }
+    catch (error) {
+        console.log(`⚠️  Cannot read file stats: ${error}`);
+    }
+    // Check database access
+    const busyStatus = await isDatabaseBusy(dbPath);
+    if (busyStatus.isBusy) {
+        console.log(`🔒 Status: BUSY`);
+        console.log(`   Reason: ${busyStatus.reason}`);
+        if (busyStatus.suggestions) {
+            console.log('   Suggestions:');
+            busyStatus.suggestions.forEach(suggestion => {
+                console.log(`   • ${suggestion}`);
+            });
+        }
+    }
+    else {
+        console.log(`✅ Status: AVAILABLE`);
+    }
+    // Show connection manager stats
+    const connectionStats = DatabaseConnectionManager.getConnectionStats();
+    if (connectionStats.totalConnections > 0) {
+        console.log('');
+        console.log(`🔗 Active connections: ${connectionStats.totalConnections}`);
+        connectionStats.connections.forEach((conn, index) => {
+            console.log(`   ${index + 1}. ${conn.path}`);
+            console.log(`      References: ${conn.refCount}`);
+            console.log(`      Last accessed: ${conn.lastAccessed.toLocaleString()}`);
+            console.log(`      Idle time: ${(conn.idleTime / 1000).toFixed(1)}s`);
+        });
+    }
+    console.log('');
+}
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export async function forceCleanupDatabase(dbPath) {
+    console.log(`🚨 Force cleaning up database connections: ${dbPath}`);
+    try {
+        await DatabaseConnectionManager.forceCloseConnection(dbPath);
+        console.log('✅ Force cleanup completed');
+    }
+    catch (error) {
+        console.log(`⚠️  Force cleanup failed: ${error}`);
+        console.log('You may need to restart the process or reboot your system');
+    }
+}
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export function setupCLICleanup(dbPath) {
+    const cleanup = async () => {
+        console.log('\n🛑 Shutting down gracefully...');
+        if (dbPath) {
+            try {
+                await DatabaseConnectionManager.releaseConnection(dbPath);
+            }
+            catch (error) {
+                // Ignore cleanup errors during shutdown
+            }
+        }
+        await DatabaseConnectionManager.closeAllConnections();
+        process.exit(0);
+    };
+    // Only set up handlers if they haven't been set up already
+    if (!process.listenerCount('SIGINT')) {
+        process.on('SIGINT', cleanup);
+    }
+    if (!process.listenerCount('SIGTERM')) {
+        process.on('SIGTERM', cleanup);
+    }
+}
+//# sourceMappingURL=cli-database-utils.js.map

package/dist/core/config.d.ts

@@ -0,0 +1,102 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Core configuration interface - model-agnostic settings
+ */
+export interface CoreConfig {
+    chunk_size: number;
+    chunk_overlap: number;
+    batch_size: number;
+    top_k: number;
+    db_file: string;
+    index_file: string;
+    model_cache_path?: string;
+    path_storage_strategy: 'absolute' | 'relative';
+    embedding_model: string;
+    rerank_enabled: boolean;
+    preprocessing?: any;
+}
+/**
+ * Extensible configuration interface that can be extended by implementations
+ */
+export interface ExtensibleConfig<T = {}> extends CoreConfig {
+    implementation: T;
+}
+/**
+ * Standard exit codes for different error conditions
+ */
+export declare const EXIT_CODES: {
+    readonly SUCCESS: 0;
+    readonly GENERAL_ERROR: 1;
+    readonly INVALID_ARGUMENTS: 2;
+    readonly CONFIGURATION_ERROR: 3;
+    readonly FILE_NOT_FOUND: 4;
+    readonly DATABASE_ERROR: 5;
+    readonly MODEL_ERROR: 6;
+    readonly INDEX_ERROR: 7;
+    readonly PERMISSION_ERROR: 8;
+};
+/**
+ * Configuration validation error with specific exit code
+ */
+export declare class ConfigurationError extends Error {
+    exitCode: number;
+    constructor(message: string, exitCode?: number);
+}
+/**
+ * Get the default model cache path as specified in the requirements
+ * @returns Default cache path (~/.raglite/models/)
+ */
+export declare function getDefaultModelCachePath(): string;
+/**
+ * Validates core configuration fields
+ * @param config - Configuration object to validate
+ * @throws {ConfigurationError} If configuration is invalid
+ */
+export declare function validateCoreConfig(config: any): asserts config is CoreConfig;
+/**
+ * Model defaults interface for different embedding models
+ */
+export interface ModelDefaults {
+    dimensions: number;
+    chunk_size: number;
+    chunk_overlap: number;
+    batch_size: number;
+}
+/**
+ * Get default configuration for different embedding models
+ * @param modelName - Name of the embedding model
+ * @returns Model-specific defaults
+ */
+export declare function getModelDefaults(modelName?: string): ModelDefaults;
+/**
+ * Default core configuration object
+ * Model-agnostic settings that can be used by core modules
+ */
+export declare const config: CoreConfig;
+/**
+ * Validate preprocessing configuration
+ */
+export declare function validatePreprocessingConfig(config: any): asserts config is any;
+/**
+ * Merge preprocessing configurations with mode defaults
+ */
+export declare function mergePreprocessingConfig(config: any): any;
+/**
+ * Utility function to handle unrecoverable errors with descriptive messages
+ * Logs error and exits immediately with appropriate exit code
+ * @param error - Error object or message
+ * @param context - Context where the error occurred
+ * @param exitCode - Exit code to use (defaults to GENERAL_ERROR)
+ */
+export declare function handleUnrecoverableError(error: Error | string, context: string, exitCode?: number): never;
+/**
+ * Utility function for safe error logging with context
+ * @param error - Error to log
+ * @param context - Context where error occurred
+ * @param skipError - Whether to skip this error and continue (default: false)
+ */
+export declare function logError(error: Error | string, context: string, skipError?: boolean): void;
+//# sourceMappingURL=config.d.ts.map