rag-lite-ts 1.0.1 → 2.0.0

package/dist/core/database-connection-manager.js

@@ -0,0 +1,304 @@
+/**
+ * Database Connection Manager - Centralized connection handling
+ * Prevents database locking issues by managing shared connections
+ * Addresses production issues with MCP server + CLI concurrent usage
+ */
+import { openDatabase } from './db.js';
+/**
+ * Database Connection Manager
+ * Manages shared database connections to prevent locking issues
+ */
+export class DatabaseConnectionManager {
+    static connections = new Map();
+    static cleanupInterval = null;
+    static CLEANUP_INTERVAL = 30000; // 30 seconds
+    static MAX_IDLE_TIME = 60000; // 1 minute
+    /**
+     * Get a shared database connection
+     * Creates new connection if none exists, otherwise returns existing
+     */
+    static async getConnection(dbPath) {
+        const normalizedPath = this.normalizePath(dbPath);
+        let connectionInfo = this.connections.get(normalizedPath);
+        if (!connectionInfo || connectionInfo.isClosing) {
+            // Create new connection
+            const connection = await openDatabase(dbPath);
+            connectionInfo = {
+                connection,
+                refCount: 1,
+                lastAccessed: Date.now(),
+                isClosing: false
+            };
+            this.connections.set(normalizedPath, connectionInfo);
+            // Start cleanup timer if this is the first connection
+            if (this.connections.size === 1 && !this.cleanupInterval) {
+                this.startCleanupTimer();
+            }
+            console.log(`📊 Database connection created: ${normalizedPath} (total: ${this.connections.size})`);
+        }
+        else {
+            // Reuse existing connection
+            connectionInfo.refCount++;
+            connectionInfo.lastAccessed = Date.now();
+            console.log(`🔄 Database connection reused: ${normalizedPath} (refs: ${connectionInfo.refCount})`);
+        }
+        return connectionInfo.connection;
+    }
+    /**
+     * Release a database connection reference
+     * Connection is kept alive for potential reuse
+     */
+    static async releaseConnection(dbPath) {
+        const normalizedPath = this.normalizePath(dbPath);
+        const connectionInfo = this.connections.get(normalizedPath);
+        if (connectionInfo && !connectionInfo.isClosing) {
+            connectionInfo.refCount = Math.max(0, connectionInfo.refCount - 1);
+            connectionInfo.lastAccessed = Date.now();
+            console.log(`📉 Database connection released: ${normalizedPath} (refs: ${connectionInfo.refCount})`);
+            // Don't immediately close - let cleanup timer handle it
+            // This prevents rapid open/close cycles
+        }
+    }
+    /**
+     * Force close a specific database connection
+     * Use with caution - only for cleanup or error recovery
+     */
+    static async forceCloseConnection(dbPath) {
+        const normalizedPath = this.normalizePath(dbPath);
+        const connectionInfo = this.connections.get(normalizedPath);
+        if (connectionInfo && !connectionInfo.isClosing) {
+            connectionInfo.isClosing = true;
+            try {
+                await connectionInfo.connection.close();
+                console.log(`🔒 Database connection force closed: ${normalizedPath}`);
+            }
+            catch (error) {
+                console.warn(`⚠️  Warning: Error force closing connection ${normalizedPath}:`, error);
+            }
+            finally {
+                this.connections.delete(normalizedPath);
+            }
+        }
+    }
+    /**
+     * Close all database connections
+     * Used during application shutdown
+     */
+    static async closeAllConnections() {
+        if (this.connections.size === 0) {
+            return; // Nothing to close
+        }
+        const isCLI = process.env.RAG_CLI_MODE === 'true';
+        if (!isCLI) {
+            console.log(`🧹 Closing all database connections (${this.connections.size} active)`);
+        }
+        const closePromises = [];
+        for (const [path, connectionInfo] of this.connections.entries()) {
+            if (!connectionInfo.isClosing) {
+                connectionInfo.isClosing = true;
+                closePromises.push(connectionInfo.connection.close()
+                    .then(() => {
+                    if (!isCLI) {
+                        console.log(`✅ Closed connection: ${path}`);
+                    }
+                })
+                    .catch(error => {
+                    if (!isCLI) {
+                        console.warn(`⚠️  Error closing connection ${path}:`, error);
+                    }
+                }));
+            }
+        }
+        await Promise.all(closePromises);
+        this.connections.clear();
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+        if (!isCLI) {
+            console.log('✅ All database connections closed');
+        }
+    }
+    /**
+     * Get connection statistics for monitoring
+     */
+    static getConnectionStats() {
+        const now = Date.now();
+        const connections = Array.from(this.connections.entries()).map(([path, info]) => ({
+            path,
+            refCount: info.refCount,
+            lastAccessed: new Date(info.lastAccessed),
+            idleTime: now - info.lastAccessed
+        }));
+        return {
+            totalConnections: this.connections.size,
+            connections
+        };
+    }
+    /**
+     * Check if a connection exists for a given path
+     */
+    static hasConnection(dbPath) {
+        const normalizedPath = this.normalizePath(dbPath);
+        const connectionInfo = this.connections.get(normalizedPath);
+        return connectionInfo !== undefined && !connectionInfo.isClosing;
+    }
+    /**
+     * Wait for database to become available
+     * Useful for handling temporary locking issues
+     */
+    static async waitForDatabaseAccess(dbPath, maxWaitMs = 5000) {
+        const startTime = Date.now();
+        while (Date.now() - startTime < maxWaitMs) {
+            try {
+                // Try to get a connection
+                const connection = await this.getConnection(dbPath);
+                await this.releaseConnection(dbPath);
+                return; // Success
+            }
+            catch (error) {
+                if (error instanceof Error && error.message.includes('SQLITE_BUSY')) {
+                    // Wait a bit and retry
+                    await new Promise(resolve => setTimeout(resolve, 100));
+                    continue;
+                }
+                // Other errors should be thrown immediately
+                throw error;
+            }
+        }
+        throw new Error(`Database ${dbPath} is busy after ${maxWaitMs}ms. Please try again later.`);
+    }
+    // =============================================================================
+    // PRIVATE METHODS
+    // =============================================================================
+    /**
+     * Normalize database path for consistent key usage
+     */
+    static normalizePath(dbPath) {
+        // Convert to absolute path and normalize separators
+        // Use URL constructor for cross-platform path normalization
+        try {
+            const url = new URL(`file://${dbPath.replace(/\\/g, '/')}`);
+            return url.pathname;
+        }
+        catch {
+            // Fallback: simple normalization
+            return dbPath.replace(/\\/g, '/');
+        }
+    }
+    /**
+     * Start the cleanup timer for idle connections
+     * Only start if not in CLI mode to prevent hanging
+     */
+    static startCleanupTimer() {
+        // Don't start cleanup timer for CLI commands - they should exit quickly
+        const isCLI = process.argv.some(arg => arg.includes('cli.js') ||
+            arg.includes('raglite') ||
+            process.env.RAG_CLI_MODE === 'true');
+        if (!isCLI) {
+            this.cleanupInterval = setInterval(() => {
+                this.cleanupIdleConnections();
+            }, this.CLEANUP_INTERVAL);
+        }
+    }
+    /**
+     * Clean up idle connections that haven't been used recently
+     */
+    static async cleanupIdleConnections() {
+        const now = Date.now();
+        const connectionsToClose = [];
+        for (const [path, connectionInfo] of this.connections.entries()) {
+            const idleTime = now - connectionInfo.lastAccessed;
+            // Close connections that are idle and have no active references
+            if (connectionInfo.refCount === 0 && idleTime > this.MAX_IDLE_TIME && !connectionInfo.isClosing) {
+                connectionsToClose.push(path);
+            }
+        }
+        if (connectionsToClose.length > 0) {
+            console.log(`🧹 Cleaning up ${connectionsToClose.length} idle database connections`);
+            for (const path of connectionsToClose) {
+                await this.forceCloseConnection(path);
+            }
+        }
+        // Stop cleanup timer if no connections remain
+        if (this.connections.size === 0 && this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+    }
+}
+/**
+ * Convenience function to get a managed database connection
+ * Use this instead of openDatabase() directly
+ */
+export async function getManagedConnection(dbPath) {
+    return DatabaseConnectionManager.getConnection(dbPath);
+}
+/**
+ * Convenience function to release a managed database connection
+ * Use this instead of connection.close() directly
+ */
+export async function releaseManagedConnection(dbPath) {
+    return DatabaseConnectionManager.releaseConnection(dbPath);
+}
+/**
+ * Enhanced database connection wrapper
+ * Automatically manages connection lifecycle
+ */
+export class ManagedDatabaseConnection {
+    dbPath;
+    connection = null;
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    /**
+     * Get the underlying database connection
+     * Automatically acquires managed connection on first use
+     */
+    async getConnection() {
+        if (!this.connection) {
+            this.connection = await DatabaseConnectionManager.getConnection(this.dbPath);
+        }
+        return this.connection;
+    }
+    /**
+     * Execute a database operation with automatic connection management
+     */
+    async execute(operation) {
+        const connection = await this.getConnection();
+        return operation(connection);
+    }
+    /**
+     * Release the managed connection
+     * Connection may be kept alive for reuse by other components
+     */
+    async release() {
+        if (this.connection) {
+            await DatabaseConnectionManager.releaseConnection(this.dbPath);
+            this.connection = null;
+        }
+    }
+    /**
+     * Force close the connection
+     * Use only during cleanup or error recovery
+     */
+    async forceClose() {
+        if (this.connection) {
+            await DatabaseConnectionManager.forceCloseConnection(this.dbPath);
+            this.connection = null;
+        }
+    }
+}
+// =============================================================================
+// PROCESS CLEANUP HANDLERS
+// =============================================================================
+/**
+ * Ensure all connections are closed on process exit
+ */
+process.on('exit', () => {
+    // Synchronous cleanup only
+    console.log('🔄 Process exiting, cleaning up database connections...');
+});
+// Don't set up global process handlers - let applications handle their own cleanup
+// This prevents CLI commands from hanging due to active event listeners
+//# sourceMappingURL=database-connection-manager.js.map

package/dist/core/db.d.ts

@@ -0,0 +1,245 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+import sqlite3 from 'sqlite3';
+import type { SystemInfo } from '../types.js';
+export interface DatabaseConnection {
+    db: sqlite3.Database;
+    run: (sql: string, params?: any[]) => Promise<sqlite3.RunResult>;
+    get: (sql: string, params?: any[]) => Promise<any>;
+    all: (sql: string, params?: any[]) => Promise<any[]>;
+    close: () => Promise<void>;
+}
+export interface ContentChunk {
+    id: number;
+    embedding_id: string;
+    document_id: number;
+    content: string;
+    content_type: string;
+    chunk_index: number;
+    metadata?: Record<string, any>;
+    created_at: string;
+    document_source: string;
+    document_title: string;
+    document_content_type: string;
+    document_content_id?: string;
+}
+/**
+ * Opens a SQLite database connection with promisified methods
+ * @param dbPath - Path to the SQLite database file
+ * @returns Promise that resolves to a database connection object
+ */
+export declare function openDatabase(dbPath: string): Promise<DatabaseConnection>;
+/**
+ * Initializes the database schema with all required tables and indexes
+ * Enhanced to support content types for multimodal use
+ * @param connection - Database connection object
+ */
+export declare function initializeSchema(connection: DatabaseConnection): Promise<void>;
+/**
+ * Inserts a new document into the database with content type support
+ * @param connection - Database connection object
+ * @param source - Source path of the document
+ * @param title - Title of the document
+ * @param contentType - Type of content ('text', 'image', etc.)
+ * @param metadata - Optional metadata object
+ * @param contentId - Optional content ID referencing content_metadata table
+ * @returns Promise that resolves to the document ID
+ */
+export declare function insertDocument(connection: DatabaseConnection, source: string, title: string, contentType?: string, metadata?: Record<string, any>, contentId?: string): Promise<number>;
+/**
+ * Inserts or updates a chunk in the database with content type support (upsert operation)
+ * @param connection - Database connection object
+ * @param embeddingId - Unique embedding ID for the chunk
+ * @param documentId - ID of the parent document
+ * @param content - Content of the chunk (text, image path, etc.)
+ * @param chunkIndex - Index of the chunk within the document
+ * @param contentType - Type of content ('text', 'image', etc.)
+ * @param metadata - Optional metadata object
+ */
+export declare function insertChunk(connection: DatabaseConnection, embeddingId: string, documentId: number, content: string, chunkIndex: number, contentType?: string, metadata?: Record<string, any>): Promise<void>;
+/**
+ * Inserts a new document or returns existing document ID if it already exists
+ * Enhanced with content type support
+ * @param connection - Database connection object
+ * @param source - Source path of the document
+ * @param title - Title of the document
+ * @param contentType - Type of content ('text', 'image', etc.)
+ * @param metadata - Optional metadata object
+ * @param contentId - Optional content ID referencing content_metadata table
+ * @returns Promise that resolves to the document ID
+ */
+export declare function upsertDocument(connection: DatabaseConnection, source: string, title: string, contentType?: string, metadata?: Record<string, any>, contentId?: string): Promise<number>;
+/**
+ * Retrieves chunks by their embedding IDs with document metadata
+ * Enhanced to include content type information
+ * @param connection - Database connection object
+ * @param embeddingIds - Array of embedding IDs to retrieve
+ * @returns Promise that resolves to an array of chunk results with document metadata
+ */
+export declare function getChunksByEmbeddingIds(connection: DatabaseConnection, embeddingIds: string[]): Promise<ContentChunk[]>;
+/**
+ * Gets the complete system information from system_info table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to SystemInfo object or null if not set
+ */
+export declare function getSystemInfo(connection: DatabaseConnection): Promise<SystemInfo | null>;
+/**
+ * Sets the complete system information in system_info table
+ * @param connection - Database connection object
+ * @param systemInfo - SystemInfo object to store
+ */
+export declare function setSystemInfo(connection: DatabaseConnection, systemInfo: Partial<SystemInfo>): Promise<void>;
+/**
+ * @deprecated Use getSystemInfo() instead. This function is kept for existing code compatibility.
+ * Gets the current model version from system_info table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to the model version string or null if not set
+ */
+export declare function getModelVersion(connection: DatabaseConnection): Promise<string | null>;
+/**
+ * @deprecated Use setSystemInfo() instead. This function is kept for existing code compatibility.
+ * Sets the model version in system_info table
+ * @param connection - Database connection object
+ * @param modelVersion - Model version string to store
+ */
+export declare function setModelVersion(connection: DatabaseConnection, modelVersion: string): Promise<void>;
+/**
+ * @deprecated Use getSystemInfo() instead. This function is kept for existing code compatibility.
+ * Gets the stored model information from system_info table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to model info object or null if not set
+ */
+export declare function getStoredModelInfo(connection: DatabaseConnection): Promise<{
+    modelName: string;
+    dimensions: number;
+} | null>;
+/**
+ * @deprecated Use setSystemInfo() instead. This function is kept for existing code compatibility.
+ * Sets the model information in system_info table
+ * @param connection - Database connection object
+ * @param modelName - Name of the embedding model
+ * @param dimensions - Number of dimensions for the model
+ */
+export declare function setStoredModelInfo(connection: DatabaseConnection, modelName: string, dimensions: number): Promise<void>;
+/**
+ * Retrieves documents by content type
+ * @param connection - Database connection object
+ * @param contentType - Content type to filter by
+ * @returns Promise that resolves to an array of documents
+ */
+export declare function getDocumentsByContentType(connection: DatabaseConnection, contentType: string): Promise<Array<{
+    id: number;
+    source: string;
+    title: string;
+    content_type: string;
+    metadata?: Record<string, any>;
+    created_at: string;
+}>>;
+/**
+ * Retrieves chunks by content type
+ * @param connection - Database connection object
+ * @param contentType - Content type to filter by
+ * @returns Promise that resolves to an array of chunks with document metadata
+ */
+export declare function getChunksByContentType(connection: DatabaseConnection, contentType: string): Promise<ContentChunk[]>;
+/**
+ * Gets content type statistics from the database
+ * @param connection - Database connection object
+ * @returns Promise that resolves to content type statistics
+ */
+export declare function getContentTypeStatistics(connection: DatabaseConnection): Promise<{
+    documents: Record<string, number>;
+    chunks: Record<string, number>;
+    total: {
+        documents: number;
+        chunks: number;
+    };
+}>;
+/**
+ * Updates document metadata
+ * @param connection - Database connection object
+ * @param documentId - ID of the document to update
+ * @param metadata - New metadata object
+ */
+export declare function updateDocumentMetadata(connection: DatabaseConnection, documentId: number, metadata: Record<string, any>): Promise<void>;
+/**
+ * Updates chunk metadata
+ * @param connection - Database connection object
+ * @param chunkId - ID of the chunk to update
+ * @param metadata - New metadata object
+ */
+export declare function updateChunkMetadata(connection: DatabaseConnection, chunkId: number, metadata: Record<string, any>): Promise<void>;
+/**
+ * Content metadata interface for unified content system
+ */
+export interface ContentMetadata {
+    id: string;
+    storageType: 'filesystem' | 'content_dir';
+    originalPath?: string;
+    contentPath: string;
+    displayName: string;
+    contentType: string;
+    fileSize: number;
+    contentHash: string;
+    createdAt: Date;
+}
+/**
+ * Inserts content metadata into the content_metadata table
+ * @param connection - Database connection object
+ * @param contentMetadata - Content metadata to insert
+ */
+export declare function insertContentMetadata(connection: DatabaseConnection, contentMetadata: Omit<ContentMetadata, 'createdAt'>): Promise<void>;
+/**
+ * Gets content metadata by content ID
+ * @param connection - Database connection object
+ * @param contentId - Content ID to retrieve
+ * @returns Promise that resolves to ContentMetadata or null if not found
+ */
+export declare function getContentMetadata(connection: DatabaseConnection, contentId: string): Promise<ContentMetadata | null>;
+/**
+ * Gets content metadata by content hash (for deduplication)
+ * @param connection - Database connection object
+ * @param contentHash - Content hash to search for
+ * @returns Promise that resolves to ContentMetadata or null if not found
+ */
+export declare function getContentMetadataByHash(connection: DatabaseConnection, contentHash: string): Promise<ContentMetadata | null>;
+/**
+ * Gets all content metadata by storage type
+ * @param connection - Database connection object
+ * @param storageType - Storage type to filter by
+ * @returns Promise that resolves to array of ContentMetadata
+ */
+export declare function getContentMetadataByStorageType(connection: DatabaseConnection, storageType: 'filesystem' | 'content_dir'): Promise<ContentMetadata[]>;
+/**
+ * Deletes content metadata by content ID
+ * @param connection - Database connection object
+ * @param contentId - Content ID to delete
+ * @returns Promise that resolves to true if deleted, false if not found
+ */
+export declare function deleteContentMetadata(connection: DatabaseConnection, contentId: string): Promise<boolean>;
+/**
+ * Gets storage statistics from storage_stats table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to storage statistics
+ */
+export declare function getStorageStats(connection: DatabaseConnection): Promise<{
+    contentDirFiles: number;
+    contentDirSize: number;
+    filesystemRefs: number;
+    lastCleanup: Date | null;
+    updatedAt: Date;
+} | null>;
+/**
+ * Updates storage statistics in storage_stats table
+ * @param connection - Database connection object
+ * @param stats - Partial storage statistics to update
+ */
+export declare function updateStorageStats(connection: DatabaseConnection, stats: {
+    contentDirFiles?: number;
+    contentDirSize?: number;
+    filesystemRefs?: number;
+    lastCleanup?: Date;
+}): Promise<void>;
+//# sourceMappingURL=db.d.ts.map