rag-lite-ts 1.0.2 → 2.0.1

package/dist/core/binary-index-format.js

@@ -0,0 +1,122 @@
+/**
+ * Binary Index Format Module
+ *
+ * Provides efficient binary serialization for HNSW vector indices.
+ *
+ * Format Specification:
+ * - Header: 24 bytes (6 × uint32)
+ * - Vectors: N × (4 + D × 4) bytes
+ * - Little-endian encoding for cross-platform compatibility
+ * - 4-byte alignment for Float32Array zero-copy views
+ *
+ * Performance:
+ * - 3.66x smaller than JSON format
+ * - 3.5x faster loading
+ * - Zero-copy Float32Array views
+ */
+import { readFileSync, writeFileSync } from 'fs';
+export class BinaryIndexFormat {
+    /**
+     * Save index data to binary format
+     *
+     * File structure:
+     * - Header (24 bytes): dimensions, maxElements, M, efConstruction, seed, currentSize
+     * - Vectors: For each vector: id (4 bytes) + vector data (dimensions × 4 bytes)
+     *
+     * @param indexPath Path to save the binary index file
+     * @param data Index data to serialize
+     */
+    static async save(indexPath, data) {
+        // Calculate total size
+        const headerSize = 24; // 6 uint32 fields
+        const vectorSize = 4 + (data.dimensions * 4); // id + vector
+        const totalSize = headerSize + (data.currentSize * vectorSize);
+        const buffer = new ArrayBuffer(totalSize);
+        const view = new DataView(buffer);
+        let offset = 0;
+        // Write header (24 bytes, all little-endian)
+        view.setUint32(offset, data.dimensions, true);
+        offset += 4;
+        view.setUint32(offset, data.maxElements, true);
+        offset += 4;
+        view.setUint32(offset, data.M, true);
+        offset += 4;
+        view.setUint32(offset, data.efConstruction, true);
+        offset += 4;
+        view.setUint32(offset, data.seed, true);
+        offset += 4;
+        view.setUint32(offset, data.currentSize, true);
+        offset += 4;
+        // Write vectors
+        for (const item of data.vectors) {
+            // Ensure 4-byte alignment (should always be true with our format)
+            if (offset % 4 !== 0) {
+                throw new Error(`Offset ${offset} is not 4-byte aligned`);
+            }
+            // Write vector ID
+            view.setUint32(offset, item.id, true);
+            offset += 4;
+            // Write vector data
+            for (let i = 0; i < item.vector.length; i++) {
+                view.setFloat32(offset, item.vector[i], true);
+                offset += 4;
+            }
+        }
+        // Write to file
+        writeFileSync(indexPath, Buffer.from(buffer));
+    }
+    /**
+     * Load index data from binary format
+     *
+     * Uses zero-copy Float32Array views for efficient loading.
+     * Copies the views to ensure data persistence after buffer lifecycle.
+     *
+     * @param indexPath Path to the binary index file
+     * @returns Deserialized index data
+     */
+    static async load(indexPath) {
+        const buffer = readFileSync(indexPath);
+        const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+        let offset = 0;
+        // Read header (24 bytes, all little-endian)
+        const dimensions = view.getUint32(offset, true);
+        offset += 4;
+        const maxElements = view.getUint32(offset, true);
+        offset += 4;
+        const M = view.getUint32(offset, true);
+        offset += 4;
+        const efConstruction = view.getUint32(offset, true);
+        offset += 4;
+        const seed = view.getUint32(offset, true);
+        offset += 4;
+        const currentSize = view.getUint32(offset, true);
+        offset += 4;
+        // Read vectors
+        const vectors = [];
+        for (let i = 0; i < currentSize; i++) {
+            // Ensure 4-byte alignment (should always be true with our format)
+            if (offset % 4 !== 0) {
+                throw new Error(`Offset ${offset} is not 4-byte aligned`);
+            }
+            // Read vector ID
+            const id = view.getUint32(offset, true);
+            offset += 4;
+            // Zero-copy Float32Array view (fast!)
+            const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
+            // Copy to avoid buffer lifecycle issues
+            const vector = new Float32Array(vectorView);
+            offset += dimensions * 4;
+            vectors.push({ id, vector });
+        }
+        return {
+            dimensions,
+            maxElements,
+            M,
+            efConstruction,
+            seed,
+            currentSize,
+            vectors
+        };
+    }
+}
+//# sourceMappingURL=binary-index-format.js.map

package/dist/core/chunker.d.ts

@@ -92,6 +92,8 @@ export interface Document {
     title: string;
     /** Full text content */
     content: string;
+    /** Optional metadata */
+    metadata?: Record<string, any>;
 }
 /**
  * Chunk interface for text chunking results

package/dist/core/cli-database-utils.d.ts

@@ -0,0 +1,53 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+/**
+ * CLI-specific database access options
+ */
+export interface CLIDatabaseOptions {
+    /** Maximum time to wait for database access (ms) */
+    maxWaitMs?: number;
+    /** Retry interval (ms) */
+    retryIntervalMs?: number;
+    /** Show progress messages to user */
+    showProgress?: boolean;
+    /** Command name for better error messages */
+    commandName?: string;
+}
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export declare function waitForCLIDatabaseAccess(dbPath: string, options?: CLIDatabaseOptions): Promise<void>;
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export declare function withCLIDatabaseAccess<T>(dbPath: string, operation: () => Promise<T>, options?: CLIDatabaseOptions): Promise<T>;
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export declare function isDatabaseBusy(dbPath: string): Promise<{
+    isBusy: boolean;
+    reason?: string;
+    suggestions?: string[];
+}>;
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export declare function showDatabaseStatus(dbPath: string): Promise<void>;
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export declare function forceCleanupDatabase(dbPath: string): Promise<void>;
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export declare function setupCLICleanup(dbPath?: string): void;
+//# sourceMappingURL=cli-database-utils.d.ts.map

package/dist/core/cli-database-utils.js

@@ -0,0 +1,239 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+import { DatabaseConnectionManager } from './database-connection-manager.js';
+import { existsSync } from 'fs';
+/**
+ * Default options for CLI database access
+ */
+const DEFAULT_CLI_OPTIONS = {
+    maxWaitMs: 10000, // 10 seconds
+    retryIntervalMs: 500, // 0.5 seconds
+    showProgress: true,
+    commandName: 'CLI command'
+};
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export async function waitForCLIDatabaseAccess(dbPath, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    // Check if database file exists
+    if (!existsSync(dbPath)) {
+        throw new Error(`Database file not found: ${dbPath}\n` +
+            `Please run 'raglite ingest <path>' first to create the database.`);
+    }
+    const startTime = Date.now();
+    let attempts = 0;
+    let lastError = null;
+    while (Date.now() - startTime < opts.maxWaitMs) {
+        attempts++;
+        try {
+            // Try to get database access
+            await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 1000);
+            if (opts.showProgress && attempts > 1) {
+                console.log(`✅ Database is now available (after ${attempts} attempts)`);
+            }
+            return; // Success!
+        }
+        catch (error) {
+            lastError = error;
+            if (lastError.message.includes('SQLITE_BUSY') || lastError.message.includes('busy')) {
+                // Database is busy - show progress and retry
+                if (opts.showProgress) {
+                    if (attempts === 1) {
+                        console.log(`⏳ Database is busy, waiting for access...`);
+                        console.log(`   This usually happens when another process is using the database.`);
+                        console.log(`   Common causes:`);
+                        console.log(`   • MCP server is running`);
+                        console.log(`   • Another CLI command is in progress`);
+                        console.log(`   • Long-running ingestion process`);
+                        console.log('');
+                    }
+                    else if (attempts % 4 === 0) {
+                        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+                        console.log(`   Still waiting... (${elapsed}s elapsed, attempt ${attempts})`);
+                    }
+                }
+                // Wait before retrying
+                await new Promise(resolve => setTimeout(resolve, opts.retryIntervalMs));
+                continue;
+            }
+            else {
+                // Other error - don't retry
+                throw new Error(`Failed to access database: ${lastError.message}\n` +
+                    `Please check that the database file is not corrupted and you have proper permissions.`);
+            }
+        }
+    }
+    // Timeout reached
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    throw new Error(`Database is still busy after ${elapsed} seconds.\n` +
+        `\n` +
+        `This might be because:\n` +
+        `• Another process is using the database (MCP server, long ingestion, etc.)\n` +
+        `• The database is locked due to an interrupted operation\n` +
+        `\n` +
+        `Solutions:\n` +
+        `• Wait for other operations to complete\n` +
+        `• Stop the MCP server if running\n` +
+        `• Restart your terminal/process\n` +
+        `• As a last resort, restart your computer\n` +
+        `\n` +
+        `Last error: ${lastError?.message || 'Unknown error'}`);
+}
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export async function withCLIDatabaseAccess(dbPath, operation, options = {}) {
+    const opts = { ...DEFAULT_CLI_OPTIONS, ...options };
+    try {
+        // Wait for database access
+        await waitForCLIDatabaseAccess(dbPath, opts);
+        // Execute the operation
+        return await operation();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            // Enhance error message with CLI context
+            const enhancedMessage = `${opts.commandName} failed: ${error.message}\n` +
+                `\n` +
+                `If this error persists:\n` +
+                `• Check that no other RAG-lite processes are running\n` +
+                `• Verify database file permissions\n` +
+                `• Try running the command again\n`;
+            throw new Error(enhancedMessage);
+        }
+        throw error;
+    }
+}
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export async function isDatabaseBusy(dbPath) {
+    try {
+        await DatabaseConnectionManager.waitForDatabaseAccess(dbPath, 100);
+        return { isBusy: false };
+    }
+    catch (error) {
+        if (error instanceof Error && (error.message.includes('SQLITE_BUSY') ||
+            error.message.includes('busy'))) {
+            return {
+                isBusy: true,
+                reason: 'Database is currently in use by another process',
+                suggestions: [
+                    'Wait for other operations to complete',
+                    'Stop MCP server if running',
+                    'Check for other CLI commands in progress'
+                ]
+            };
+        }
+        return {
+            isBusy: true,
+            reason: `Database access error: ${error instanceof Error ? error.message : String(error)}`,
+            suggestions: [
+                'Check database file permissions',
+                'Verify database file is not corrupted',
+                'Ensure you have read/write access to the database directory'
+            ]
+        };
+    }
+}
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export async function showDatabaseStatus(dbPath) {
+    console.log(`📊 Database Status: ${dbPath}`);
+    console.log('');
+    // Check file existence
+    if (!existsSync(dbPath)) {
+        console.log('❌ Database file does not exist');
+        console.log('   Run "raglite ingest <path>" to create the database');
+        return;
+    }
+    // Check file stats
+    try {
+        const fs = await import('fs');
+        const stats = fs.statSync(dbPath);
+        console.log(`📁 File size: ${(stats.size / 1024).toFixed(1)} KB`);
+        console.log(`📅 Last modified: ${stats.mtime.toLocaleString()}`);
+    }
+    catch (error) {
+        console.log(`⚠️  Cannot read file stats: ${error}`);
+    }
+    // Check database access
+    const busyStatus = await isDatabaseBusy(dbPath);
+    if (busyStatus.isBusy) {
+        console.log(`🔒 Status: BUSY`);
+        console.log(`   Reason: ${busyStatus.reason}`);
+        if (busyStatus.suggestions) {
+            console.log('   Suggestions:');
+            busyStatus.suggestions.forEach(suggestion => {
+                console.log(`   • ${suggestion}`);
+            });
+        }
+    }
+    else {
+        console.log(`✅ Status: AVAILABLE`);
+    }
+    // Show connection manager stats
+    const connectionStats = DatabaseConnectionManager.getConnectionStats();
+    if (connectionStats.totalConnections > 0) {
+        console.log('');
+        console.log(`🔗 Active connections: ${connectionStats.totalConnections}`);
+        connectionStats.connections.forEach((conn, index) => {
+            console.log(`   ${index + 1}. ${conn.path}`);
+            console.log(`      References: ${conn.refCount}`);
+            console.log(`      Last accessed: ${conn.lastAccessed.toLocaleString()}`);
+            console.log(`      Idle time: ${(conn.idleTime / 1000).toFixed(1)}s`);
+        });
+    }
+    console.log('');
+}
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export async function forceCleanupDatabase(dbPath) {
+    console.log(`🚨 Force cleaning up database connections: ${dbPath}`);
+    try {
+        await DatabaseConnectionManager.forceCloseConnection(dbPath);
+        console.log('✅ Force cleanup completed');
+    }
+    catch (error) {
+        console.log(`⚠️  Force cleanup failed: ${error}`);
+        console.log('You may need to restart the process or reboot your system');
+    }
+}
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export function setupCLICleanup(dbPath) {
+    const cleanup = async () => {
+        console.log('\n🛑 Shutting down gracefully...');
+        if (dbPath) {
+            try {
+                await DatabaseConnectionManager.releaseConnection(dbPath);
+            }
+            catch (error) {
+                // Ignore cleanup errors during shutdown
+            }
+        }
+        await DatabaseConnectionManager.closeAllConnections();
+        process.exit(0);
+    };
+    // Only set up handlers if they haven't been set up already
+    if (!process.listenerCount('SIGINT')) {
+        process.on('SIGINT', cleanup);
+    }
+    if (!process.listenerCount('SIGTERM')) {
+        process.on('SIGTERM', cleanup);
+    }
+}
+//# sourceMappingURL=cli-database-utils.js.map

package/dist/core/config.js

@@ -101,11 +101,16 @@ export function getModelDefaults(modelName) {
         chunk_overlap: 50,
         batch_size: 16
     };
-    // Model-specific overrides
+    // Model-specific overrides based on model name heuristics
     if (modelName) {
         const normalizedName = modelName.toLowerCase();
-        // Specific model configurations
-        if (normalizedName.includes('all-mpnet-base-v2')) {
+        // CLIP models - 512 dimensions
+        if (normalizedName.includes('clip')) {
+            defaults.dimensions = 512;
+            defaults.batch_size = 8;
+        }
+        // MPNet models - 768 dimensions
+        else if (normalizedName.includes('all-mpnet-base-v2')) {
             defaults.dimensions = 768;
             defaults.chunk_size = 400;
             defaults.chunk_overlap = 80;
@@ -114,9 +119,11 @@ export function getModelDefaults(modelName) {
         else if (normalizedName.includes('mpnet') || normalizedName.includes('768')) {
             defaults.dimensions = 768;
         }
+        // Models with 512 in the name
         else if (normalizedName.includes('512')) {
             defaults.dimensions = 512;
         }
+        // MiniLM and other 384-dimensional models (default)
         else if (normalizedName.includes('384') || normalizedName.includes('minilm')) {
             defaults.dimensions = 384;
         }

package/dist/core/content-errors.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Content System Error Handling - Comprehensive error handling for unified content system
+ * Implements task 8.1: Add specific error types for content operations with clear guidance
+ * Requirements: 8.1, 8.2, 8.3, 8.6
+ */
+import { APIError, ErrorFactory } from '../api-errors.js';
+import { ErrorSeverity } from './error-handler.js';
+/**
+ * Content-specific error types for unified content system operations
+ */
+export declare class ContentNotFoundError extends APIError {
+    constructor(contentId: string, displayName?: string, context?: string);
+}
+/**
+ * Error when storage limits are exceeded
+ */
+export declare class StorageLimitExceededError extends APIError {
+    constructor(currentUsageMB: number, limitMB: number, contentSizeMB: number, context?: string);
+}
+/**
+ * Error when content format is invalid or unsupported
+ */
+export declare class InvalidContentFormatError extends APIError {
+    constructor(contentType: string, reason: string, context?: string);
+    private static getSuggestionsForContentType;
+}
+/**
+ * Error when content ingestion fails
+ */
+export declare class ContentIngestionError extends APIError {
+    constructor(operation: string, reason: string, context?: string);
+}
+/**
+ * Error when content retrieval fails
+ */
+export declare class ContentRetrievalError extends APIError {
+    constructor(contentId: string, format: string, reason: string, context?: string);
+}
+/**
+ * Error when content directory operations fail
+ */
+export declare class ContentDirectoryError extends APIError {
+    constructor(operation: string, reason: string, context?: string);
+}
+/**
+ * Error when content deduplication fails
+ */
+export declare class ContentDeduplicationError extends APIError {
+    constructor(reason: string, context?: string);
+}
+/**
+ * Enhanced error factory for content system operations
+ * Extends the base ErrorFactory with content-specific error handling
+ */
+export declare class ContentErrorFactory extends ErrorFactory {
+    /**
+     * Create content-specific error from generic error
+     */
+    static createContentError(error: unknown, operation: string, context: string): APIError;
+    /**
+     * Create storage-related error with enhanced guidance
+     */
+    static createStorageError(error: unknown, context: string): APIError;
+    /**
+     * Create format validation error with specific guidance
+     */
+    static createFormatError(contentType: string, reason: string, context: string): InvalidContentFormatError;
+}
+/**
+ * Content system error handler with categorized error management
+ */
+export declare class ContentErrorHandler {
+    /**
+     * Handle content operation errors with appropriate categorization
+     */
+    static handleContentError(error: unknown, operation: string, context: string, options?: {
+        severity?: ErrorSeverity;
+        skipError?: boolean;
+        showStack?: boolean;
+    }): never;
+    /**
+     * Get appropriate error category for content errors
+     */
+    private static getCategoryForError;
+    /**
+     * Validate content operation parameters and throw appropriate errors
+     */
+    static validateContentOperation(contentId?: string, format?: string, contentType?: string): void;
+    /**
+     * Create user-friendly error message for common content scenarios
+     */
+    static createUserFriendlyMessage(error: APIError): string;
+}
+/**
+ * Utility functions for content error handling
+ */
+export declare const ContentErrorUtils: {
+    /**
+     * Check if an error is content-related
+     */
+    isContentError(error: unknown): error is APIError;
+    /**
+     * Extract content ID from error message
+     */
+    extractContentId(error: APIError): string | null;
+    /**
+     * Get recovery action for content error
+     */
+    getRecoveryAction(error: APIError): string;
+};
+//# sourceMappingURL=content-errors.d.ts.map