rag-lite-ts 2.3.0 → 2.3.1

package/dist/cjs/cli/search.js

@@ -132,7 +132,7 @@ export async function runSearch(query, options = {}) {
                     process.exit(EXIT_CODES.MODEL_ERROR);
                 }
             }
-            // Prepare search options
+            // Prepare search options (with generation support)
             const searchOptions = {};
             if (options['top-k'] !== undefined) {
                 searchOptions.top_k = options['top-k'];
@@ -164,9 +164,60 @@ export async function runSearch(query, options = {}) {
             }
             // Track whether reranking will actually be used in this search
             const rerankingUsed = searchOptions.rerank === true;
+            // Handle generation options (experimental, text mode only)
+            const generateResponse = options.generate === true;
+            const generatorModel = options.generator;
+            const maxGenerationTokens = options['max-tokens'];
+            const generationTemperature = options.temperature;
+            const maxChunksForContext = options['max-chunks'];
+            // Generation only supported in text mode
+            if (generateResponse && isImage) {
+                console.warn('⚠️  [EXPERIMENTAL] Generation is only supported for text searches.');
+                console.warn('   Image search results will be returned without generation.');
+                console.warn('');
+            }
+            // Generation requires reranking - enable it automatically
+            let rerankingEnabledForGeneration = false;
+            if (generateResponse && !isImage && !searchOptions.rerank) {
+                searchOptions.rerank = true;
+                rerankingEnabledForGeneration = true;
+                console.log('📋 Reranking automatically enabled (required for generation)');
+            }
+            // Set up generator if generation is requested (text mode only)
+            let generateFn;
+            if (generateResponse && !isImage) {
+                try {
+                    console.log('🤖 [EXPERIMENTAL] Initializing response generator...');
+                    const { createGenerateFunctionFromModel, getDefaultGeneratorModel } = await import('../factories/generator-factory.js');
+                    const { getDefaultMaxChunksForContext } = await import('../core/generator-registry.js');
+                    const modelToUse = generatorModel || getDefaultGeneratorModel();
+                    const defaultChunks = getDefaultMaxChunksForContext(modelToUse) || 3;
+                    console.log(`   Model: ${modelToUse}`);
+                    console.log(`   Max chunks for context: ${maxChunksForContext || defaultChunks} (default: ${defaultChunks})`);
+                    generateFn = await createGenerateFunctionFromModel(modelToUse);
+                    searchEngine.setGenerateFunction(generateFn);
+                    console.log('✅ Generator initialized');
+                    console.log('');
+                }
+                catch (error) {
+                    console.error('❌ [EXPERIMENTAL] Failed to initialize generator:', error instanceof Error ? error.message : 'Unknown error');
+                    console.error('   Continuing without generation...');
+                    console.error('');
+                }
+            }
+            // Set generation options if generator is ready
+            if (generateFn && generateResponse && !isImage) {
+                searchOptions.generateResponse = true;
+                searchOptions.generationOptions = {
+                    maxTokens: maxGenerationTokens,
+                    temperature: generationTemperature,
+                    maxChunksForContext: maxChunksForContext
+                };
+            }
             // Perform search
             const startTime = Date.now();
             let results;
+            let generationResult;
             if (isImage && embedder) {
                 // Image-based search: embed the image and search with the vector
                 console.log('Embedding image...');
@@ -174,8 +225,14 @@ export async function runSearch(query, options = {}) {
                 console.log('Searching with image embedding...');
                 results = await searchEngine.searchWithVector(imageEmbedding.vector, searchOptions);
             }
+            else if (generateResponse && generateFn) {
+                // Text-based search with generation
+                const searchResult = await searchEngine.searchWithGeneration(query, searchOptions);
+                results = searchResult.results;
+                generationResult = searchResult.generation;
+            }
             else {
-                // Text-based search
+                // Standard text-based search
                 results = await searchEngine.search(query, searchOptions);
             }
             const searchTime = Date.now() - startTime;
@@ -216,6 +273,21 @@ export async function runSearch(query, options = {}) {
                     }
                     console.log('');
                 });
+                // Display generated response if available (experimental)
+                if (generationResult) {
+                    console.log('─'.repeat(50));
+                    console.log('🤖 Generated Response [EXPERIMENTAL]');
+                    console.log(`Model: ${generationResult.modelUsed}`);
+                    console.log('─'.repeat(50));
+                    console.log('');
+                    console.log(generationResult.response);
+                    console.log('');
+                    console.log('─'.repeat(50));
+                    console.log(`⏱️  Generation: ${(generationResult.generationTimeMs / 1000).toFixed(1)}s | ` +
+                        `📊 ${generationResult.tokensUsed} tokens | ` +
+                        `📄 ${generationResult.chunksUsedForContext} chunks used` +
+                        (generationResult.truncated ? ' (context truncated)' : ''));
+                }
                 // Show search statistics
                 const stats = await searchEngine.getStats();
                 console.log('─'.repeat(50));
@@ -233,6 +305,9 @@ export async function runSearch(query, options = {}) {
                 else {
                     console.log('Reranking: disabled');
                 }
+                if (generationResult) {
+                    console.log('Generation: enabled [EXPERIMENTAL]');
+                }
             }
         }
         finally {

package/dist/cjs/cli.js

@@ -54,6 +54,7 @@ Examples:
   raglite search "red car" --content-type image  # Search only image results
   raglite search ./photo.jpg       # Search with image (multimodal mode only)
   raglite search ./image.png --top-k 5  # Find similar images
+  raglite search "How does auth work?" --generate  # [EXPERIMENTAL] Generate AI response
   raglite ui                       # Launch web interface
 
   raglite rebuild                  # Rebuild the entire index
@@ -64,6 +65,13 @@ Options for search:
   --no-rerank           Disable reranking
   --content-type <type> Filter results by content type: 'text', 'image', or 'all' (default: all)
 
+  [EXPERIMENTAL] AI Response Generation (text mode only):
+  --generate            Generate an AI response from search results
+  --generator <model>   Generator model to use (default: SmolLM2-135M-Instruct)
+  --max-tokens <n>      Maximum tokens to generate (default: 512)
+  --temperature <n>     Sampling temperature 0-1 (default: 0.1)
+  --max-chunks <n>      Maximum chunks for context (default: 3 for 135M, 5 for 360M)
+
 Options for ingest:
   --model <name>       Use specific embedding model
   --mode <mode>        Processing mode: 'text' (default) or 'multimodal'
@@ -83,6 +91,12 @@ Available reranking strategies (multimodal mode):
   text-derived  Use image-to-text conversion + cross-encoder (default)
   disabled      No reranking, use vector similarity only
 
+[EXPERIMENTAL] Available generator models:
+  HuggingFaceTB/SmolLM2-135M-Instruct  (balanced, recommended default, uses top 3 chunks)
+  HuggingFaceTB/SmolLM2-360M-Instruct  (higher quality, slower, uses top 5 chunks)
+  
+  Note: Generation requires reranking (--rerank is automatically enabled with --generate)
+
 For more information, visit: https://github.com/your-repo/rag-lite-ts
 `);
 }
@@ -126,6 +140,10 @@ function parseArgs() {
             else if (optionName === 'force-rebuild') {
                 options.forceRebuild = true;
             }
+            else if (optionName === 'generate') {
+                // Handle --generate flag for experimental response generation
+                options.generate = true;
+            }
             else if (optionName === 'help') {
                 return { command: 'help', args: [], options: {} };
             }
@@ -136,7 +154,16 @@ function parseArgs() {
                 // Handle options with values
                 const nextArg = args[i + 1];
                 if (nextArg && !nextArg.startsWith('--')) {
-                    options[optionName] = nextArg;
+                    // Parse numeric values for specific options
+                    if (optionName === 'max-tokens' || optionName === 'top-k' || optionName === 'max-chunks') {
+                        options[optionName] = parseInt(nextArg, 10);
+                    }
+                    else if (optionName === 'temperature') {
+                        options[optionName] = parseFloat(nextArg);
+                    }
+                    else {
+                        options[optionName] = nextArg;
+                    }
                     i++; // Skip the next argument as it's the value
                 }
                 else {

package/dist/cjs/core/abstract-generator.d.ts

@@ -0,0 +1,97 @@
+/**
+ * CORE MODULE — Abstract Base Generator
+ *
+ * Provides model-agnostic base functionality for all generator implementations.
+ * This is an abstract base class, not a concrete implementation.
+ *
+ * ARCHITECTURAL NOTE:
+ * Similar to BaseUniversalEmbedder, this class provides shared infrastructure:
+ * - Model lifecycle management (loading, cleanup, disposal)
+ * - Token budget management
+ * - Error handling with helpful messages
+ * - Common utility methods
+ *
+ * IMPLEMENTATION LAYERS:
+ * - Text: InstructGenerator extends this class (SmolLM2-Instruct)
+ * - Text: CausalLMGenerator extends this class (DistilGPT2)
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { ResponseGenerator, GeneratorModelInfo, GeneratorModelType, GenerationRequest, GenerationResult, GeneratorCreationOptions } from './response-generator.js';
+import { GenerationError } from './response-generator.js';
+/**
+ * Abstract base class for response generators
+ * Provides common functionality and lifecycle management
+ */
+export declare abstract class BaseResponseGenerator implements ResponseGenerator {
+    readonly modelName: string;
+    protected _isLoaded: boolean;
+    protected _modelInfo: GeneratorModelInfo;
+    protected _options: GeneratorCreationOptions;
+    constructor(modelName: string, options?: GeneratorCreationOptions);
+    get modelType(): GeneratorModelType;
+    get maxContextLength(): number;
+    get maxOutputLength(): number;
+    isLoaded(): boolean;
+    getModelInfo(): GeneratorModelInfo;
+    /**
+     * Load the model - must be implemented by subclasses
+     */
+    abstract loadModel(): Promise<void>;
+    /**
+     * Generate text using the model - must be implemented by subclasses
+     * @param prompt - The formatted prompt string
+     * @param options - Generation options
+     * @returns Generated text
+     */
+    protected abstract generateText(prompt: string, options: {
+        maxTokens: number;
+        temperature: number;
+        topP: number;
+        topK: number;
+        repetitionPenalty: number;
+        stopSequences: string[];
+    }): Promise<{
+        text: string;
+        promptTokens: number;
+        completionTokens: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    }>;
+    /**
+     * Clean up resources - must be implemented by subclasses
+     */
+    abstract cleanup(): Promise<void>;
+    /**
+     * Generate a response based on query and retrieved chunks
+     * This method orchestrates the generation pipeline
+     */
+    generate(request: GenerationRequest): Promise<GenerationResult>;
+    /**
+     * Validate that the model is loaded before operations
+     */
+    protected ensureLoaded(): void;
+    /**
+     * Clean up response text by removing artifacts
+     */
+    protected cleanResponseText(text: string): string;
+    /**
+     * Log model loading progress
+     */
+    protected logModelLoading(stage: string, details?: string): void;
+    /**
+     * Handle model loading errors with helpful messages
+     */
+    protected handleLoadingError(error: Error): GenerationError;
+}
+/**
+ * Extended options for generator instances
+ */
+export interface GeneratorOptions extends GeneratorCreationOptions {
+    /** Log level for debugging */
+    logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'silent';
+}
+/**
+ * Create generator options with defaults
+ */
+export declare function createGeneratorOptions(options?: Partial<GeneratorOptions>): GeneratorOptions;
+//# sourceMappingURL=abstract-generator.d.ts.map

package/dist/cjs/core/abstract-generator.js

@@ -0,0 +1,222 @@
+/**
+ * CORE MODULE — Abstract Base Generator
+ *
+ * Provides model-agnostic base functionality for all generator implementations.
+ * This is an abstract base class, not a concrete implementation.
+ *
+ * ARCHITECTURAL NOTE:
+ * Similar to BaseUniversalEmbedder, this class provides shared infrastructure:
+ * - Model lifecycle management (loading, cleanup, disposal)
+ * - Token budget management
+ * - Error handling with helpful messages
+ * - Common utility methods
+ *
+ * IMPLEMENTATION LAYERS:
+ * - Text: InstructGenerator extends this class (SmolLM2-Instruct)
+ * - Text: CausalLMGenerator extends this class (DistilGPT2)
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import { GenerationError } from './response-generator.js';
+import { GeneratorRegistry } from './generator-registry.js';
+import { buildPrompt, getDefaultStopSequences } from './prompt-templates.js';
+// =============================================================================
+// BASE GENERATOR ABSTRACT CLASS
+// =============================================================================
+/**
+ * Abstract base class for response generators
+ * Provides common functionality and lifecycle management
+ */
+export class BaseResponseGenerator {
+    modelName;
+    _isLoaded = false;
+    _modelInfo;
+    _options;
+    constructor(modelName, options = {}) {
+        this.modelName = modelName;
+        const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
+        if (!modelInfo) {
+            throw new Error(`Generator model '${modelName}' is not supported. ` +
+                `Supported models: ${GeneratorRegistry.getSupportedGenerators().join(', ')}`);
+        }
+        this._modelInfo = modelInfo;
+        this._options = options;
+    }
+    // =============================================================================
+    // PUBLIC INTERFACE IMPLEMENTATION
+    // =============================================================================
+    get modelType() {
+        return this._modelInfo.type;
+    }
+    get maxContextLength() {
+        return this._modelInfo.capabilities.maxContextLength;
+    }
+    get maxOutputLength() {
+        return this._modelInfo.capabilities.defaultMaxOutputTokens;
+    }
+    isLoaded() {
+        return this._isLoaded;
+    }
+    getModelInfo() {
+        return { ...this._modelInfo }; // Return a copy to prevent mutation
+    }
+    // =============================================================================
+    // DEFAULT IMPLEMENTATION
+    // =============================================================================
+    /**
+     * Generate a response based on query and retrieved chunks
+     * This method orchestrates the generation pipeline
+     */
+    async generate(request) {
+        if (!this._isLoaded) {
+            await this.loadModel();
+        }
+        const startTime = Date.now();
+        try {
+            // Get generation parameters with defaults
+            const maxTokens = request.maxTokens ?? this._modelInfo.capabilities.defaultMaxOutputTokens;
+            const temperature = request.temperature ?? this._modelInfo.capabilities.recommendedTemperature;
+            const topP = request.topP ?? 0.9;
+            const topK = request.topK ?? 50;
+            const repetitionPenalty = request.repetitionPenalty ?? 1.1;
+            const stopSequences = request.stopSequences ?? getDefaultStopSequences(this.modelType);
+            // Get max chunks for context (configurable, with model-specific default)
+            const maxChunksForContext = request.maxChunksForContext ??
+                this._modelInfo.capabilities.defaultMaxChunksForContext;
+            // Limit chunks to maxChunksForContext (assumes chunks are already reranked)
+            const totalChunks = request.chunks.length;
+            const limitedChunks = request.chunks.slice(0, maxChunksForContext);
+            if (totalChunks > maxChunksForContext) {
+                console.log(`📊 Using top ${maxChunksForContext} of ${totalChunks} reranked chunks for generation`);
+            }
+            // Build the prompt with context
+            const builtPrompt = buildPrompt({
+                query: request.query,
+                chunks: limitedChunks,
+                modelType: this.modelType,
+                systemPrompt: request.systemPrompt,
+                maxContextLength: this.maxContextLength,
+                reservedOutputTokens: maxTokens,
+                includeSourceAttribution: request.includeSourceAttribution
+            });
+            // Log context info
+            if (builtPrompt.contextInfo.truncated) {
+                console.warn(`⚠️  Context truncated: Only ${builtPrompt.contextInfo.chunksIncluded} of ` +
+                    `${builtPrompt.contextInfo.totalChunks} chunks fit in context window`);
+            }
+            // Generate response
+            const result = await this.generateText(builtPrompt.prompt, {
+                maxTokens,
+                temperature,
+                topP,
+                topK,
+                repetitionPenalty,
+                stopSequences
+            });
+            const generationTimeMs = Date.now() - startTime;
+            // Clean up the response text
+            const cleanedResponse = this.cleanResponseText(result.text);
+            return {
+                response: cleanedResponse,
+                tokensUsed: result.promptTokens + result.completionTokens,
+                truncated: builtPrompt.contextInfo.truncated,
+                modelName: this.modelName,
+                generationTimeMs,
+                metadata: {
+                    promptTokens: result.promptTokens,
+                    completionTokens: result.completionTokens,
+                    chunksIncluded: builtPrompt.contextInfo.chunksIncluded,
+                    totalChunks: totalChunks, // Report original total, not limited
+                    finishReason: result.finishReason
+                }
+            };
+        }
+        catch (error) {
+            const generationTimeMs = Date.now() - startTime;
+            if (error instanceof GenerationError) {
+                throw error;
+            }
+            throw new GenerationError(this.modelName, 'generation', `Generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, error instanceof Error ? error : undefined);
+        }
+    }
+    // =============================================================================
+    // PROTECTED HELPER METHODS
+    // =============================================================================
+    /**
+     * Validate that the model is loaded before operations
+     */
+    ensureLoaded() {
+        if (!this._isLoaded) {
+            throw new GenerationError(this.modelName, 'generation', `Model '${this.modelName}' is not loaded. Call loadModel() first.`);
+        }
+    }
+    /**
+     * Clean up response text by removing artifacts
+     */
+    cleanResponseText(text) {
+        let cleaned = text.trim();
+        // Remove common artifacts
+        const artifactsToRemove = [
+            '<|im_end|>',
+            '<|im_start|>',
+            '<|endoftext|>',
+            '<|assistant|>',
+            '<|user|>',
+            '<|system|>'
+        ];
+        for (const artifact of artifactsToRemove) {
+            cleaned = cleaned.split(artifact)[0];
+        }
+        // Remove trailing incomplete sentences (if cut off at max tokens)
+        if (cleaned.length > 0 && !cleaned.match(/[.!?]$/)) {
+            const lastSentenceEnd = Math.max(cleaned.lastIndexOf('.'), cleaned.lastIndexOf('!'), cleaned.lastIndexOf('?'));
+            if (lastSentenceEnd > cleaned.length * 0.5) {
+                cleaned = cleaned.substring(0, lastSentenceEnd + 1);
+            }
+        }
+        return cleaned.trim();
+    }
+    /**
+     * Log model loading progress
+     */
+    logModelLoading(stage, details) {
+        const message = `[${this.modelName}] ${stage}`;
+        if (details) {
+            console.log(`${message}: ${details}`);
+        }
+        else {
+            console.log(message);
+        }
+    }
+    /**
+     * Handle model loading errors with helpful messages
+     */
+    handleLoadingError(error) {
+        const baseMessage = `Failed to load generator model '${this.modelName}': ${error.message}`;
+        // Provide specific guidance based on error type
+        if (error.message.includes('network') || error.message.includes('fetch')) {
+            return new GenerationError(this.modelName, 'loading', `${baseMessage}\n` +
+                `This appears to be a network error. Please check your internet connection ` +
+                `and ensure the model repository is accessible.`, error);
+        }
+        if (error.message.includes('memory') || error.message.includes('OOM')) {
+            return new GenerationError(this.modelName, 'loading', `${baseMessage}\n` +
+                `This appears to be a memory error. The model requires ` +
+                `${this._modelInfo.requirements.minimumMemory}MB. Try closing other applications ` +
+                `or using a smaller model like 'Xenova/distilgpt2'.`, error);
+        }
+        return new GenerationError(this.modelName, 'loading', baseMessage, error);
+    }
+}
+/**
+ * Create generator options with defaults
+ */
+export function createGeneratorOptions(options = {}) {
+    return {
+        timeout: 60000, // 60 seconds
+        enableGPU: false,
+        logLevel: 'info',
+        ...options
+    };
+}
+//# sourceMappingURL=abstract-generator.js.map

package/dist/cjs/core/binary-index-format.js

@@ -190,6 +190,9 @@ export class BinaryIndexFormat {
         const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
         let offset = 0;
         // Read basic header (24 bytes, all little-endian)
+        if (buffer.byteLength < 24) {
+            throw new Error(`Index file too small: expected at least 24 bytes, got ${buffer.byteLength}`);
+        }
         const dimensions = view.getUint32(offset, true);
         offset += 4;
         const maxElements = view.getUint32(offset, true);
@@ -202,10 +205,20 @@ export class BinaryIndexFormat {
         offset += 4;
         const currentSize = view.getUint32(offset, true);
         offset += 4;
-        // Check if this is the extended grouped format (40+ bytes header)
-        const hasGroups = buffer.byteLength >= 40 ? view.getUint32(offset, true) : 0;
-        if (hasGroups === 1 && buffer.byteLength >= 40) {
-            // Load grouped format
+        // Calculate expected size for original format
+        const vectorSize = 4 + (dimensions * 4); // id + vector
+        const expectedOriginalSize = 24 + (currentSize * vectorSize);
+        // Check if this is the extended grouped format (44 bytes header)
+        // Extended header has: 24 bytes basic + 4 bytes hasGroups + 16 bytes for offsets/counts = 44 bytes
+        // Only check for grouped format if file is larger than expected original format size
+        const hasGroups = buffer.byteLength > expectedOriginalSize && buffer.byteLength >= 44 && offset + 4 <= buffer.byteLength
+            ? view.getUint32(offset, true)
+            : 0;
+        if (hasGroups === 1 && buffer.byteLength >= 44) {
+            // Load grouped format - ensure we have enough bytes for extended header
+            if (offset + 20 > buffer.byteLength) {
+                throw new Error(`Index file too small for grouped format: expected at least ${offset + 20} bytes, got ${buffer.byteLength}`);
+            }
             const textOffset = view.getUint32(offset + 4, true);
             const textCount = view.getUint32(offset + 8, true);
             const imageOffset = view.getUint32(offset + 12, true);
@@ -218,14 +231,23 @@ export class BinaryIndexFormat {
                 if (offset % 4 !== 0) {
                     throw new Error(`Offset ${offset} is not 4-byte aligned`);
                 }
+                // Check bounds before reading vector ID
+                if (offset + 4 > buffer.byteLength) {
+                    throw new Error(`Text vector ID at offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
+                }
                 // Read vector ID
                 const id = view.getUint32(offset, true);
                 offset += 4;
+                // Check bounds before reading vector data
+                const vectorDataSize = dimensions * 4;
+                if (offset + vectorDataSize > buffer.byteLength) {
+                    throw new Error(`Text vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
+                }
                 // Zero-copy Float32Array view
                 const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
                 // Copy to avoid buffer lifecycle issues
                 const vector = new Float32Array(vectorView);
-                offset += dimensions * 4;
+                offset += vectorDataSize;
                 textVectors.push({ id, vector });
             }
             // Load image vectors
@@ -236,14 +258,23 @@ export class BinaryIndexFormat {
                 if (offset % 4 !== 0) {
                     throw new Error(`Offset ${offset} is not 4-byte aligned`);
                 }
+                // Check bounds before reading vector ID
+                if (offset + 4 > buffer.byteLength) {
+                    throw new Error(`Image vector ID at offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
+                }
                 // Read vector ID
                 const id = view.getUint32(offset, true);
                 offset += 4;
+                // Check bounds before reading vector data
+                const vectorDataSize = dimensions * 4;
+                if (offset + vectorDataSize > buffer.byteLength) {
+                    throw new Error(`Image vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
+                }
                 // Zero-copy Float32Array view
                 const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
                 // Copy to avoid buffer lifecycle issues
                 const vector = new Float32Array(vectorView);
-                offset += dimensions * 4;
+                offset += vectorDataSize;
                 imageVectors.push({ id, vector });
             }
             // Combine all vectors for backward compatibility
@@ -269,14 +300,23 @@ export class BinaryIndexFormat {
                 if (offset % 4 !== 0) {
                     throw new Error(`Offset ${offset} is not 4-byte aligned`);
                 }
+                // Check bounds before reading vector ID
+                if (offset + 4 > buffer.byteLength) {
+                    throw new Error(`Offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
+                }
                 // Read vector ID
                 const id = view.getUint32(offset, true);
                 offset += 4;
+                // Check bounds before reading vector data
+                const vectorDataSize = dimensions * 4;
+                if (offset + vectorDataSize > buffer.byteLength) {
+                    throw new Error(`Vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
+                }
                 // Zero-copy Float32Array view (fast!)
                 const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
                 // Copy to avoid buffer lifecycle issues
                 const vector = new Float32Array(vectorView);
-                offset += dimensions * 4;
+                offset += vectorDataSize;
                 vectors.push({ id, vector });
             }
             return {