rag-lite-ts 2.1.0 → 2.2.0

package/dist/esm/core/binary-index-format.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Binary Index Format Module
+ *
+ * Provides efficient binary serialization for HNSW vector indices.
+ *
+ * Format Specification:
+ * - Header: 24 bytes (6 × uint32)
+ * - Vectors: N × (4 + D × 4) bytes
+ * - Little-endian encoding for cross-platform compatibility
+ * - 4-byte alignment for Float32Array zero-copy views
+ *
+ * Performance:
+ * - 3.66x smaller than JSON format
+ * - 3.5x faster loading
+ * - Zero-copy Float32Array views
+ */
+export interface BinaryIndexData {
+    dimensions: number;
+    maxElements: number;
+    M: number;
+    efConstruction: number;
+    seed: number;
+    currentSize: number;
+    vectors: Array<{
+        id: number;
+        vector: Float32Array;
+    }>;
+    hasContentTypeGroups?: boolean;
+    textVectors?: Array<{
+        id: number;
+        vector: Float32Array;
+    }>;
+    imageVectors?: Array<{
+        id: number;
+        vector: Float32Array;
+    }>;
+}
+export declare class BinaryIndexFormat {
+    /**
+     * Save index data to binary format (original format for backward compatibility)
+     *
+     * File structure:
+     * - Header (24 bytes): dimensions, maxElements, M, efConstruction, seed, currentSize
+     * - Vectors: For each vector: id (4 bytes) + vector data (dimensions × 4 bytes)
+     *
+     * @param indexPath Path to save the binary index file
+     * @param data Index data to serialize
+     */
+    static save(indexPath: string, data: BinaryIndexData): Promise<void>;
+    /**
+     * Save index data to grouped binary format
+     *
+     * File structure:
+     * - Extended Header (40 bytes):
+     *   - Original 6 fields (24 bytes)
+     *   - hasGroups flag (4 bytes)
+     *   - textOffset (4 bytes)
+     *   - textCount (4 bytes)
+     *   - imageOffset (4 bytes)
+     *   - imageCount (4 bytes)
+     * - Data section: [text vectors...][image vectors...]
+     *
+     * @param indexPath Path to save the binary index file
+     * @param data Index data to serialize
+     */
+    static saveGrouped(indexPath: string, data: BinaryIndexData): Promise<void>;
+    /**
+     * Load index data from binary format (supports both original and grouped formats)
+     *
+     * Uses zero-copy Float32Array views for efficient loading.
+     * Copies the views to ensure data persistence after buffer lifecycle.
+     *
+     * @param indexPath Path to the binary index file
+     * @returns Deserialized index data
+     */
+    static load(indexPath: string): Promise<BinaryIndexData>;
+}
+//# sourceMappingURL=binary-index-format.d.ts.map

package/dist/esm/core/binary-index-format.js

@@ -0,0 +1,291 @@
+/**
+ * Binary Index Format Module
+ *
+ * Provides efficient binary serialization for HNSW vector indices.
+ *
+ * Format Specification:
+ * - Header: 24 bytes (6 × uint32)
+ * - Vectors: N × (4 + D × 4) bytes
+ * - Little-endian encoding for cross-platform compatibility
+ * - 4-byte alignment for Float32Array zero-copy views
+ *
+ * Performance:
+ * - 3.66x smaller than JSON format
+ * - 3.5x faster loading
+ * - Zero-copy Float32Array views
+ */
+import { readFileSync, writeFileSync } from 'fs';
+export class BinaryIndexFormat {
+    /**
+     * Save index data to binary format (original format for backward compatibility)
+     *
+     * File structure:
+     * - Header (24 bytes): dimensions, maxElements, M, efConstruction, seed, currentSize
+     * - Vectors: For each vector: id (4 bytes) + vector data (dimensions × 4 bytes)
+     *
+     * @param indexPath Path to save the binary index file
+     * @param data Index data to serialize
+     */
+    static async save(indexPath, data) {
+        // Calculate total size
+        const headerSize = 24; // 6 uint32 fields
+        const vectorSize = 4 + (data.dimensions * 4); // id + vector
+        const totalSize = headerSize + (data.currentSize * vectorSize);
+        const buffer = new ArrayBuffer(totalSize);
+        const view = new DataView(buffer);
+        let offset = 0;
+        // Write header (24 bytes, all little-endian)
+        view.setUint32(offset, data.dimensions, true);
+        offset += 4;
+        view.setUint32(offset, data.maxElements, true);
+        offset += 4;
+        view.setUint32(offset, data.M, true);
+        offset += 4;
+        view.setUint32(offset, data.efConstruction, true);
+        offset += 4;
+        view.setUint32(offset, data.seed, true);
+        offset += 4;
+        view.setUint32(offset, data.currentSize, true);
+        offset += 4;
+        // Write vectors
+        for (const item of data.vectors) {
+            // Ensure 4-byte alignment (should always be true with our format)
+            if (offset % 4 !== 0) {
+                throw new Error(`Offset ${offset} is not 4-byte aligned`);
+            }
+            // Write vector ID
+            view.setUint32(offset, item.id, true);
+            offset += 4;
+            // Write vector data
+            for (let i = 0; i < item.vector.length; i++) {
+                view.setFloat32(offset, item.vector[i], true);
+                offset += 4;
+            }
+        }
+        // Write to file
+        writeFileSync(indexPath, Buffer.from(buffer));
+    }
+    /**
+     * Save index data to grouped binary format
+     *
+     * File structure:
+     * - Extended Header (40 bytes):
+     *   - Original 6 fields (24 bytes)
+     *   - hasGroups flag (4 bytes)
+     *   - textOffset (4 bytes)
+     *   - textCount (4 bytes)
+     *   - imageOffset (4 bytes)
+     *   - imageCount (4 bytes)
+     * - Data section: [text vectors...][image vectors...]
+     *
+     * @param indexPath Path to save the binary index file
+     * @param data Index data to serialize
+     */
+    static async saveGrouped(indexPath, data) {
+        if (!data.hasContentTypeGroups || !data.textVectors || !data.imageVectors) {
+            // Fallback to original format
+            return this.save(indexPath, data);
+        }
+        const headerSize = 44; // Extended header: 24 + 20 bytes (hasGroups + textOffset + textCount + imageOffset + imageCount)
+        const vectorSize = 4 + (data.dimensions * 4); // id + vector
+        // Calculate offsets and total size
+        const textOffset = headerSize;
+        const imageOffset = textOffset + (data.textVectors.length * vectorSize);
+        const totalSize = imageOffset + (data.imageVectors.length * vectorSize);
+        const buffer = new ArrayBuffer(totalSize);
+        const view = new DataView(buffer);
+        let offset = 0;
+        // Write extended header (40 bytes, all little-endian)
+        if (offset + 40 > buffer.byteLength) {
+            throw new Error(`Header write would exceed buffer bounds: offset=${offset}, headerSize=40, bufferSize=${buffer.byteLength}`);
+        }
+        view.setUint32(offset, data.dimensions, true);
+        offset += 4;
+        view.setUint32(offset, data.maxElements, true);
+        offset += 4;
+        view.setUint32(offset, data.M, true);
+        offset += 4;
+        view.setUint32(offset, data.efConstruction, true);
+        offset += 4;
+        view.setUint32(offset, data.seed, true);
+        offset += 4;
+        view.setUint32(offset, data.currentSize, true);
+        offset += 4;
+        // Extended fields
+        view.setUint32(offset, 1, true);
+        offset += 4; // hasGroups = 1
+        view.setUint32(offset, textOffset, true);
+        offset += 4;
+        view.setUint32(offset, data.textVectors.length, true);
+        offset += 4;
+        view.setUint32(offset, imageOffset, true);
+        offset += 4;
+        view.setUint32(offset, data.imageVectors.length, true);
+        offset += 4;
+        // Write text vectors
+        for (const item of data.textVectors) {
+            // Ensure 4-byte alignment
+            if (offset % 4 !== 0) {
+                throw new Error(`Offset ${offset} is not 4-byte aligned`);
+            }
+            // Check bounds before writing
+            if (offset + 4 > buffer.byteLength) {
+                throw new Error(`ID write would exceed buffer bounds: offset=${offset}, bufferSize=${buffer.byteLength}`);
+            }
+            // Write vector ID
+            view.setUint32(offset, item.id, true);
+            offset += 4;
+            // Check bounds for vector data
+            const vectorDataSize = item.vector.length * 4;
+            if (offset + vectorDataSize > buffer.byteLength) {
+                throw new Error(`Vector data write would exceed buffer bounds: offset=${offset}, dataSize=${vectorDataSize}, bufferSize=${buffer.byteLength}`);
+            }
+            // Write vector data
+            for (let i = 0; i < item.vector.length; i++) {
+                view.setFloat32(offset, item.vector[i], true);
+                offset += 4;
+            }
+        }
+        // Write image vectors
+        for (const item of data.imageVectors) {
+            // Ensure 4-byte alignment
+            if (offset % 4 !== 0) {
+                throw new Error(`Offset ${offset} is not 4-byte aligned`);
+            }
+            // Check bounds before writing
+            if (offset + 4 > buffer.byteLength) {
+                throw new Error(`ID write would exceed buffer bounds: offset=${offset}, bufferSize=${buffer.byteLength}`);
+            }
+            // Write vector ID
+            view.setUint32(offset, item.id, true);
+            offset += 4;
+            // Check bounds for vector data
+            const vectorDataSize = item.vector.length * 4;
+            if (offset + vectorDataSize > buffer.byteLength) {
+                throw new Error(`Vector data write would exceed buffer bounds: offset=${offset}, dataSize=${vectorDataSize}, bufferSize=${buffer.byteLength}`);
+            }
+            // Write vector data
+            for (let i = 0; i < item.vector.length; i++) {
+                view.setFloat32(offset, item.vector[i], true);
+                offset += 4;
+            }
+        }
+        // Write to file
+        writeFileSync(indexPath, Buffer.from(buffer));
+    }
+    /**
+     * Load index data from binary format (supports both original and grouped formats)
+     *
+     * Uses zero-copy Float32Array views for efficient loading.
+     * Copies the views to ensure data persistence after buffer lifecycle.
+     *
+     * @param indexPath Path to the binary index file
+     * @returns Deserialized index data
+     */
+    static async load(indexPath) {
+        const buffer = readFileSync(indexPath);
+        const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+        let offset = 0;
+        // Read basic header (24 bytes, all little-endian)
+        const dimensions = view.getUint32(offset, true);
+        offset += 4;
+        const maxElements = view.getUint32(offset, true);
+        offset += 4;
+        const M = view.getUint32(offset, true);
+        offset += 4;
+        const efConstruction = view.getUint32(offset, true);
+        offset += 4;
+        const seed = view.getUint32(offset, true);
+        offset += 4;
+        const currentSize = view.getUint32(offset, true);
+        offset += 4;
+        // Check if this is the extended grouped format (40+ bytes header)
+        const hasGroups = buffer.byteLength >= 40 ? view.getUint32(offset, true) : 0;
+        if (hasGroups === 1 && buffer.byteLength >= 40) {
+            // Load grouped format
+            const textOffset = view.getUint32(offset + 4, true);
+            const textCount = view.getUint32(offset + 8, true);
+            const imageOffset = view.getUint32(offset + 12, true);
+            const imageCount = view.getUint32(offset + 16, true);
+            // Load text vectors
+            const textVectors = [];
+            offset = textOffset;
+            for (let i = 0; i < textCount; i++) {
+                // Ensure 4-byte alignment
+                if (offset % 4 !== 0) {
+                    throw new Error(`Offset ${offset} is not 4-byte aligned`);
+                }
+                // Read vector ID
+                const id = view.getUint32(offset, true);
+                offset += 4;
+                // Zero-copy Float32Array view
+                const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
+                // Copy to avoid buffer lifecycle issues
+                const vector = new Float32Array(vectorView);
+                offset += dimensions * 4;
+                textVectors.push({ id, vector });
+            }
+            // Load image vectors
+            const imageVectors = [];
+            offset = imageOffset;
+            for (let i = 0; i < imageCount; i++) {
+                // Ensure 4-byte alignment
+                if (offset % 4 !== 0) {
+                    throw new Error(`Offset ${offset} is not 4-byte aligned`);
+                }
+                // Read vector ID
+                const id = view.getUint32(offset, true);
+                offset += 4;
+                // Zero-copy Float32Array view
+                const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
+                // Copy to avoid buffer lifecycle issues
+                const vector = new Float32Array(vectorView);
+                offset += dimensions * 4;
+                imageVectors.push({ id, vector });
+            }
+            // Combine all vectors for backward compatibility
+            const allVectors = [...textVectors, ...imageVectors];
+            return {
+                dimensions,
+                maxElements,
+                M,
+                efConstruction,
+                seed,
+                currentSize,
+                vectors: allVectors,
+                hasContentTypeGroups: true,
+                textVectors,
+                imageVectors
+            };
+        }
+        else {
+            // Load original format
+            const vectors = [];
+            for (let i = 0; i < currentSize; i++) {
+                // Ensure 4-byte alignment (should always be true with our format)
+                if (offset % 4 !== 0) {
+                    throw new Error(`Offset ${offset} is not 4-byte aligned`);
+                }
+                // Read vector ID
+                const id = view.getUint32(offset, true);
+                offset += 4;
+                // Zero-copy Float32Array view (fast!)
+                const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
+                // Copy to avoid buffer lifecycle issues
+                const vector = new Float32Array(vectorView);
+                offset += dimensions * 4;
+                vectors.push({ id, vector });
+            }
+            return {
+                dimensions,
+                maxElements,
+                M,
+                efConstruction,
+                seed,
+                currentSize,
+                vectors
+            };
+        }
+    }
+}
+//# sourceMappingURL=binary-index-format.js.map

package/dist/esm/core/chunker.d.ts

@@ -0,0 +1,119 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Configuration for chunking behavior
+ */
+export interface ChunkConfig {
+    /** Target chunk size in tokens (200-300 recommended) */
+    chunkSize: number;
+    /** Overlap between chunks in tokens (50 recommended) */
+    chunkOverlap: number;
+}
+/**
+ * Generic document interface that can represent different content types
+ */
+export interface GenericDocument {
+    /** Source path or identifier */
+    source: string;
+    /** Document title */
+    title: string;
+    /** Content (text, image path, etc.) */
+    content: string;
+    /** Content type identifier (text, image, etc.) */
+    contentType: string;
+    /** Optional metadata for the document */
+    metadata?: Record<string, any>;
+}
+/**
+ * Generic chunk interface that can represent different content types
+ */
+export interface GenericChunk {
+    /** The content of the chunk (text, image path, etc.) */
+    content: string;
+    /** Content type identifier (text, image, etc.) */
+    contentType: string;
+    /** Index of this chunk within the document */
+    chunkIndex: number;
+    /** Optional metadata for the chunk */
+    metadata?: Record<string, any>;
+}
+/**
+ * Strategy interface for chunking different content types
+ */
+export interface ChunkingStrategy {
+    /**
+     * Check if this strategy applies to the given content type
+     */
+    appliesTo(contentType: string): boolean;
+    /**
+     * Chunk a document using this strategy
+     */
+    chunk(document: GenericDocument, config: ChunkConfig): Promise<GenericChunk[]>;
+}
+/**
+ * Registry for chunking strategies
+ */
+export declare class ChunkingStrategyRegistry {
+    private strategies;
+    /**
+     * Register a chunking strategy
+     */
+    register(strategy: ChunkingStrategy): void;
+    /**
+     * Find the appropriate strategy for a content type
+     */
+    findStrategy(contentType: string): ChunkingStrategy | undefined;
+    /**
+     * Get all registered strategies
+     */
+    getStrategies(): ChunkingStrategy[];
+}
+/**
+ * Default chunking configuration
+ */
+export declare const DEFAULT_CHUNK_CONFIG: ChunkConfig;
+/**
+ * Global chunking strategy registry
+ */
+export declare const chunkingRegistry: ChunkingStrategyRegistry;
+/**
+ * Generic chunking function that uses registered strategies
+ */
+export declare function chunkGenericDocument(document: GenericDocument, config?: ChunkConfig): Promise<GenericChunk[]>;
+/**
+ * Document interface for text chunking
+ */
+export interface Document {
+    /** Source path or identifier */
+    source: string;
+    /** Document title */
+    title: string;
+    /** Full text content */
+    content: string;
+    /** Optional metadata */
+    metadata?: Record<string, any>;
+}
+/**
+ * Chunk interface for text chunking results
+ */
+export interface Chunk {
+    /** The text content of the chunk */
+    text: string;
+    /** Index of this chunk within the document */
+    chunkIndex: number;
+    /** Number of tokens in this chunk */
+    tokenCount: number;
+}
+/**
+ * Text document chunking function
+ * Uses the text chunking strategy from the text implementation layer
+ */
+export declare function chunkDocument(document: Document, config?: ChunkConfig): Promise<Chunk[]>;
+/**
+ * Register the text chunking strategy with the global registry
+ * This should be called during application initialization
+ */
+export declare function registerTextChunkingStrategy(): Promise<void>;
+//# sourceMappingURL=chunker.d.ts.map

package/dist/esm/core/chunker.js

@@ -0,0 +1,73 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+/**
+ * Registry for chunking strategies
+ */
+export class ChunkingStrategyRegistry {
+    strategies = [];
+    /**
+     * Register a chunking strategy
+     */
+    register(strategy) {
+        this.strategies.push(strategy);
+    }
+    /**
+     * Find the appropriate strategy for a content type
+     */
+    findStrategy(contentType) {
+        return this.strategies.find(strategy => strategy.appliesTo(contentType));
+    }
+    /**
+     * Get all registered strategies
+     */
+    getStrategies() {
+        return [...this.strategies];
+    }
+}
+/**
+ * Default chunking configuration
+ */
+export const DEFAULT_CHUNK_CONFIG = {
+    chunkSize: 250, // Target 200-300 tokens
+    chunkOverlap: 50
+};
+/**
+ * Global chunking strategy registry
+ */
+export const chunkingRegistry = new ChunkingStrategyRegistry();
+/**
+ * Generic chunking function that uses registered strategies
+ */
+export async function chunkGenericDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    const strategy = chunkingRegistry.findStrategy(document.contentType);
+    if (!strategy) {
+        throw new Error(`No chunking strategy found for content type: ${document.contentType}`);
+    }
+    return strategy.chunk(document, config);
+}
+/**
+ * Text document chunking function
+ * Uses the text chunking strategy from the text implementation layer
+ */
+export async function chunkDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    // Import the text chunker implementation dynamically to avoid circular dependencies
+    const { chunkDocument: textChunkDocument } = await import('../text/chunker.js');
+    return textChunkDocument(document, config);
+}
+/**
+ * Register the text chunking strategy with the global registry
+ * This should be called during application initialization
+ */
+export async function registerTextChunkingStrategy() {
+    const { TextChunkingStrategy } = await import('../text/chunker.js');
+    const textStrategy = new TextChunkingStrategy();
+    chunkingRegistry.register(textStrategy);
+}
+// Auto-register the text strategy when this module is loaded
+// This ensures text chunking works out of the box
+registerTextChunkingStrategy().catch(error => {
+    console.warn('Failed to register text chunking strategy:', error);
+});
+//# sourceMappingURL=chunker.js.map

package/dist/esm/core/cli-database-utils.d.ts

@@ -0,0 +1,53 @@
+/**
+ * CLI Database Utilities - Database access helpers for CLI commands
+ * Provides database locking detection and retry mechanisms for CLI operations
+ * Prevents conflicts between CLI commands and long-running processes like MCP server
+ */
+/**
+ * CLI-specific database access options
+ */
+export interface CLIDatabaseOptions {
+    /** Maximum time to wait for database access (ms) */
+    maxWaitMs?: number;
+    /** Retry interval (ms) */
+    retryIntervalMs?: number;
+    /** Show progress messages to user */
+    showProgress?: boolean;
+    /** Command name for better error messages */
+    commandName?: string;
+}
+/**
+ * Wait for database to become available for CLI operations
+ * Provides user-friendly progress messages and error handling
+ */
+export declare function waitForCLIDatabaseAccess(dbPath: string, options?: CLIDatabaseOptions): Promise<void>;
+/**
+ * Execute a CLI operation with database access protection
+ * Automatically handles database locking and provides user feedback
+ */
+export declare function withCLIDatabaseAccess<T>(dbPath: string, operation: () => Promise<T>, options?: CLIDatabaseOptions): Promise<T>;
+/**
+ * Check if database is currently busy (non-blocking)
+ * Useful for showing warnings or status information
+ */
+export declare function isDatabaseBusy(dbPath: string): Promise<{
+    isBusy: boolean;
+    reason?: string;
+    suggestions?: string[];
+}>;
+/**
+ * Show database status information for debugging
+ * Useful for troubleshooting CLI issues
+ */
+export declare function showDatabaseStatus(dbPath: string): Promise<void>;
+/**
+ * Force cleanup of database connections (emergency use only)
+ * Use with caution - only for recovery from stuck states
+ */
+export declare function forceCleanupDatabase(dbPath: string): Promise<void>;
+/**
+ * Graceful shutdown helper for CLI commands
+ * Ensures proper cleanup when CLI commands are interrupted
+ */
+export declare function setupCLICleanup(dbPath?: string): void;
+//# sourceMappingURL=cli-database-utils.d.ts.map