rag-lite-ts 2.2.0 → 2.3.1

package/dist/cjs/text/generators/causal-lm-generator.js

@@ -0,0 +1,197 @@
+/**
+ * TEXT IMPLEMENTATION — Causal LM Generator for DistilGPT2
+ *
+ * Implements ResponseGenerator interface for causal language models.
+ * Supports Xenova/distilgpt2 for fast, basic text generation.
+ *
+ * Features:
+ * - Simple prompt formatting (no chat template)
+ * - Fast generation with smaller model
+ * - Streaming generation support
+ * - Resource management via ResourceManager
+ *
+ * Note: Causal LM models don't support system prompts, so responses
+ * may be less focused than instruction-tuned models.
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import '../../dom-polyfills.js';
+import { BaseResponseGenerator } from '../../core/abstract-generator.js';
+import { GenerationError } from '../../core/response-generator.js';
+import { getResourceManager } from '../../core/resource-manager.js';
+import { config } from '../../core/config.js';
+// =============================================================================
+// CAUSAL LM GENERATOR IMPLEMENTATION
+// =============================================================================
+/**
+ * Causal LM generator implementation for DistilGPT2
+ *
+ * Uses causal language models that generate text based on simple prompts.
+ * Faster but may produce less focused responses than instruct models.
+ */
+export class CausalLMGenerator extends BaseResponseGenerator {
+    pipeline = null;
+    tokenizer = null;
+    resourceManager = getResourceManager();
+    resourceId;
+    constructor(modelName, options = {}) {
+        super(modelName, options);
+        // Validate model is a causal-lm model
+        if (this.modelType !== 'causal-lm') {
+            throw new Error(`CausalLMGenerator requires a causal-lm model, but '${modelName}' is type '${this.modelType}'`);
+        }
+    }
+    // =============================================================================
+    // MODEL LIFECYCLE
+    // =============================================================================
+    /**
+     * Load the causal LM model using transformers.js
+     */
+    async loadModel() {
+        if (this._isLoaded && this.pipeline) {
+            return;
+        }
+        try {
+            this.logModelLoading('Loading causal LM generator model');
+            // Ensure DOM polyfills
+            if (typeof globalThis.self === 'undefined') {
+                globalThis.self = globalThis;
+            }
+            // Dynamic import transformers.js
+            const { pipeline, AutoTokenizer } = await import('@huggingface/transformers');
+            // Load tokenizer first for token counting
+            this.logModelLoading('Loading tokenizer');
+            this.tokenizer = await AutoTokenizer.from_pretrained(this.modelName, {
+                cache_dir: this._options.cachePath || config.model_cache_path
+            });
+            // Load text generation pipeline
+            this.logModelLoading('Loading text generation pipeline');
+            this.pipeline = await pipeline('text-generation', this.modelName, {
+                cache_dir: this._options.cachePath || config.model_cache_path,
+                dtype: 'fp32'
+            });
+            // Register with resource manager
+            this.resourceId = this.resourceManager.registerModel(this.pipeline, this.modelName, 'generator');
+            this._isLoaded = true;
+            this.logModelLoading('Model loaded successfully');
+        }
+        catch (error) {
+            this._isLoaded = false;
+            throw this.handleLoadingError(error);
+        }
+    }
+    /**
+     * Clean up model resources
+     */
+    async cleanup() {
+        try {
+            if (this.resourceId) {
+                await this.resourceManager.cleanupResource(this.resourceId);
+                this.resourceId = undefined;
+            }
+            // Clear references
+            this.pipeline = null;
+            this.tokenizer = null;
+            this._isLoaded = false;
+            // Force GC if available
+            if (global.gc) {
+                global.gc();
+            }
+            this.logModelLoading('Resources cleaned up');
+        }
+        catch (error) {
+            console.warn(`Cleanup error: ${error instanceof Error ? error.message : 'Unknown'}`);
+            this.pipeline = null;
+            this.tokenizer = null;
+            this._isLoaded = false;
+        }
+    }
+    // =============================================================================
+    // GENERATION IMPLEMENTATION
+    // =============================================================================
+    /**
+     * Generate text using the causal LM model
+     */
+    async generateText(prompt, options) {
+        this.ensureLoaded();
+        try {
+            // Count prompt tokens
+            const promptTokens = await this.countTokens(prompt);
+            // Generate - GPT2 uses return_full_text differently
+            const result = await this.pipeline(prompt, {
+                max_new_tokens: options.maxTokens,
+                temperature: Math.max(0.1, options.temperature), // GPT2 needs temp > 0
+                top_p: options.topP,
+                top_k: options.topK,
+                repetition_penalty: options.repetitionPenalty,
+                do_sample: true,
+                return_full_text: true, // GPT2 needs full text
+                pad_token_id: this.tokenizer?.eos_token_id // GPT2 uses eos as pad
+            });
+            // Extract generated text (remove prompt)
+            let generatedText = result[0]?.generated_text || '';
+            if (generatedText.startsWith(prompt)) {
+                generatedText = generatedText.substring(prompt.length);
+            }
+            // Process stop sequences
+            let finalText = generatedText;
+            let finishReason = 'complete';
+            for (const stopSeq of options.stopSequences) {
+                const stopIndex = finalText.indexOf(stopSeq);
+                if (stopIndex !== -1) {
+                    finalText = finalText.substring(0, stopIndex);
+                    finishReason = 'stop_sequence';
+                    break;
+                }
+            }
+            // Count completion tokens
+            const completionTokens = await this.countTokens(finalText);
+            // Check if we hit max tokens
+            if (completionTokens >= options.maxTokens - 5) {
+                finishReason = 'length';
+            }
+            return {
+                text: finalText,
+                promptTokens,
+                completionTokens,
+                finishReason
+            };
+        }
+        catch (error) {
+            throw new GenerationError(this.modelName, 'generation', `Text generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, error instanceof Error ? error : undefined);
+        }
+    }
+    /**
+     * Generate text with streaming output
+     */
+    async *generateStream(request) {
+        // For now, fall back to non-streaming and yield the full response
+        // TODO: Implement true streaming when transformers.js supports it better
+        const result = await this.generate(request);
+        yield result.response;
+    }
+    // =============================================================================
+    // HELPER METHODS
+    // =============================================================================
+    /**
+     * Count tokens in a text string
+     */
+    async countTokens(text) {
+        if (!this.tokenizer) {
+            // Fallback to estimation
+            return Math.ceil(text.length / 4);
+        }
+        try {
+            const encoded = await this.tokenizer(text, {
+                return_tensors: false,
+                padding: false,
+                truncation: false
+            });
+            return encoded.input_ids?.length || Math.ceil(text.length / 4);
+        }
+        catch {
+            return Math.ceil(text.length / 4);
+        }
+    }
+}
+//# sourceMappingURL=causal-lm-generator.js.map

package/dist/cjs/text/generators/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * TEXT GENERATORS — Export Module
+ *
+ * Provides text generation capabilities for RAG response synthesis.
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export { InstructGenerator } from './instruct-generator.js';
+export { CausalLMGenerator } from './causal-lm-generator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/text/generators/index.js

@@ -0,0 +1,10 @@
+/**
+ * TEXT GENERATORS — Export Module
+ *
+ * Provides text generation capabilities for RAG response synthesis.
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export { InstructGenerator } from './instruct-generator.js';
+export { CausalLMGenerator } from './causal-lm-generator.js';
+//# sourceMappingURL=index.js.map

package/dist/cjs/text/generators/instruct-generator.d.ts

@@ -0,0 +1,62 @@
+/**
+ * TEXT IMPLEMENTATION — Instruct Generator for SmolLM2-Instruct Models
+ *
+ * Implements ResponseGenerator interface for instruction-tuned models.
+ * Supports SmolLM2-135M-Instruct and SmolLM2-360M-Instruct.
+ *
+ * Features:
+ * - Chat template formatting with system/user/assistant roles
+ * - Streaming generation support
+ * - Proper stop sequence handling
+ * - Resource management via ResourceManager
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import '../../dom-polyfills.js';
+import { BaseResponseGenerator, type GeneratorOptions } from '../../core/abstract-generator.js';
+/**
+ * Instruct generator implementation for SmolLM2-Instruct models
+ *
+ * Uses instruction-tuned models that understand chat templates with
+ * system, user, and assistant roles for better response quality.
+ */
+export declare class InstructGenerator extends BaseResponseGenerator {
+    private pipeline;
+    private tokenizer;
+    private resourceManager;
+    private resourceId?;
+    constructor(modelName: string, options?: GeneratorOptions);
+    /**
+     * Load the instruct model using transformers.js
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Generate text using the instruct model
+     */
+    protected generateText(prompt: string, options: {
+        maxTokens: number;
+        temperature: number;
+        topP: number;
+        topK: number;
+        repetitionPenalty: number;
+        stopSequences: string[];
+    }): Promise<{
+        text: string;
+        promptTokens: number;
+        completionTokens: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    }>;
+    /**
+     * Generate text with streaming output
+     */
+    generateStream(request: import('../../core/response-generator.js').GenerationRequest): AsyncIterable<string>;
+    /**
+     * Count tokens in a text string
+     */
+    private countTokens;
+}
+//# sourceMappingURL=instruct-generator.d.ts.map

package/dist/cjs/text/generators/instruct-generator.js

@@ -0,0 +1,192 @@
+/**
+ * TEXT IMPLEMENTATION — Instruct Generator for SmolLM2-Instruct Models
+ *
+ * Implements ResponseGenerator interface for instruction-tuned models.
+ * Supports SmolLM2-135M-Instruct and SmolLM2-360M-Instruct.
+ *
+ * Features:
+ * - Chat template formatting with system/user/assistant roles
+ * - Streaming generation support
+ * - Proper stop sequence handling
+ * - Resource management via ResourceManager
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import '../../dom-polyfills.js';
+import { BaseResponseGenerator } from '../../core/abstract-generator.js';
+import { GenerationError } from '../../core/response-generator.js';
+import { getResourceManager } from '../../core/resource-manager.js';
+import { config } from '../../core/config.js';
+// =============================================================================
+// INSTRUCT GENERATOR IMPLEMENTATION
+// =============================================================================
+/**
+ * Instruct generator implementation for SmolLM2-Instruct models
+ *
+ * Uses instruction-tuned models that understand chat templates with
+ * system, user, and assistant roles for better response quality.
+ */
+export class InstructGenerator extends BaseResponseGenerator {
+    pipeline = null;
+    tokenizer = null;
+    resourceManager = getResourceManager();
+    resourceId;
+    constructor(modelName, options = {}) {
+        super(modelName, options);
+        // Validate model is an instruct model
+        if (this.modelType !== 'instruct') {
+            throw new Error(`InstructGenerator requires an instruct model, but '${modelName}' is type '${this.modelType}'`);
+        }
+    }
+    // =============================================================================
+    // MODEL LIFECYCLE
+    // =============================================================================
+    /**
+     * Load the instruct model using transformers.js
+     */
+    async loadModel() {
+        if (this._isLoaded && this.pipeline) {
+            return;
+        }
+        try {
+            this.logModelLoading('Loading instruct generator model');
+            // Ensure DOM polyfills
+            if (typeof globalThis.self === 'undefined') {
+                globalThis.self = globalThis;
+            }
+            // Dynamic import transformers.js
+            const { pipeline, AutoTokenizer } = await import('@huggingface/transformers');
+            // Load tokenizer first for token counting
+            this.logModelLoading('Loading tokenizer');
+            this.tokenizer = await AutoTokenizer.from_pretrained(this.modelName, {
+                cache_dir: this._options.cachePath || config.model_cache_path
+            });
+            // Load text generation pipeline
+            this.logModelLoading('Loading text generation pipeline');
+            this.pipeline = await pipeline('text-generation', this.modelName, {
+                cache_dir: this._options.cachePath || config.model_cache_path,
+                dtype: 'fp32'
+            });
+            // Register with resource manager
+            this.resourceId = this.resourceManager.registerModel(this.pipeline, this.modelName, 'generator');
+            this._isLoaded = true;
+            this.logModelLoading('Model loaded successfully');
+        }
+        catch (error) {
+            this._isLoaded = false;
+            throw this.handleLoadingError(error);
+        }
+    }
+    /**
+     * Clean up model resources
+     */
+    async cleanup() {
+        try {
+            if (this.resourceId) {
+                await this.resourceManager.cleanupResource(this.resourceId);
+                this.resourceId = undefined;
+            }
+            // Clear references
+            this.pipeline = null;
+            this.tokenizer = null;
+            this._isLoaded = false;
+            // Force GC if available
+            if (global.gc) {
+                global.gc();
+            }
+            this.logModelLoading('Resources cleaned up');
+        }
+        catch (error) {
+            console.warn(`Cleanup error: ${error instanceof Error ? error.message : 'Unknown'}`);
+            this.pipeline = null;
+            this.tokenizer = null;
+            this._isLoaded = false;
+        }
+    }
+    // =============================================================================
+    // GENERATION IMPLEMENTATION
+    // =============================================================================
+    /**
+     * Generate text using the instruct model
+     */
+    async generateText(prompt, options) {
+        this.ensureLoaded();
+        try {
+            // Count prompt tokens
+            const promptTokens = await this.countTokens(prompt);
+            // Generate
+            const result = await this.pipeline(prompt, {
+                max_new_tokens: options.maxTokens,
+                temperature: options.temperature,
+                top_p: options.topP,
+                top_k: options.topK,
+                repetition_penalty: options.repetitionPenalty,
+                do_sample: options.temperature > 0,
+                return_full_text: false,
+                pad_token_id: this.tokenizer?.pad_token_id,
+                eos_token_id: this.tokenizer?.eos_token_id
+            });
+            // Extract generated text
+            const generatedText = result[0]?.generated_text || '';
+            // Process stop sequences
+            let finalText = generatedText;
+            let finishReason = 'complete';
+            for (const stopSeq of options.stopSequences) {
+                const stopIndex = finalText.indexOf(stopSeq);
+                if (stopIndex !== -1) {
+                    finalText = finalText.substring(0, stopIndex);
+                    finishReason = 'stop_sequence';
+                    break;
+                }
+            }
+            // Count completion tokens
+            const completionTokens = await this.countTokens(finalText);
+            // Check if we hit max tokens
+            if (completionTokens >= options.maxTokens - 5) {
+                finishReason = 'length';
+            }
+            return {
+                text: finalText,
+                promptTokens,
+                completionTokens,
+                finishReason
+            };
+        }
+        catch (error) {
+            throw new GenerationError(this.modelName, 'generation', `Text generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, error instanceof Error ? error : undefined);
+        }
+    }
+    /**
+     * Generate text with streaming output
+     */
+    async *generateStream(request) {
+        // For now, fall back to non-streaming and yield the full response
+        // TODO: Implement true streaming when transformers.js supports it better
+        const result = await this.generate(request);
+        yield result.response;
+    }
+    // =============================================================================
+    // HELPER METHODS
+    // =============================================================================
+    /**
+     * Count tokens in a text string
+     */
+    async countTokens(text) {
+        if (!this.tokenizer) {
+            // Fallback to estimation
+            return Math.ceil(text.length / 4);
+        }
+        try {
+            const encoded = await this.tokenizer(text, {
+                return_tensors: false,
+                padding: false,
+                truncation: false
+            });
+            return encoded.input_ids?.length || Math.ceil(text.length / 4);
+        }
+        catch {
+            return Math.ceil(text.length / 4);
+        }
+    }
+}
+//# sourceMappingURL=instruct-generator.js.map

package/dist/esm/cli/indexer.js

@@ -80,7 +80,7 @@ async function validateModeConfiguration(options) {
  */
 export async function runIngest(path, options = {}) {
     try {
-        // Handle --rebuild-if-needed flag immediately to prevent dimension mismatch error
+        // Handle --force-rebuild flag immediately to prevent dimension mismatch errors
         // Validate path exists
         const resolvedPath = resolve(path);
         if (!existsSync(resolvedPath)) {
@@ -159,26 +159,52 @@ export async function runIngest(path, options = {}) {
             factoryOptions.mode = options.mode;
             console.log(`Using processing mode: ${options.mode}`);
         }
-        if (options.rebuildIfNeeded) {
+        if (options.forceRebuild) {
             factoryOptions.forceRebuild = true;
-            console.log('Force rebuild enabled due to rebuildIfNeeded option');
-            // Delete old index file immediately to prevent dimension mismatch errors
-            const indexPath = process.env.RAG_INDEX_FILE || './vector-index.bin';
-            const { existsSync, unlinkSync } = await import('fs');
-            if (existsSync(indexPath)) {
-                try {
-                    unlinkSync(indexPath);
-                    console.log('🗑️ Removed old index file to prevent dimension mismatch');
-                }
-                catch (error) {
-                    console.warn(`⚠️ Could not remove old index file: ${error}`);
-                }
-            }
+            console.log('Force rebuild enabled (--force-rebuild)');
         }
         // Validate mode-specific model and strategy combinations
         await validateModeConfiguration(factoryOptions);
         const dbPath = process.env.RAG_DB_FILE || './db.sqlite';
         const indexPath = process.env.RAG_INDEX_FILE || './vector-index.bin';
+        // --force-rebuild: Always delete DB (and sidecars) and index to guarantee a clean rebuild.
+        if (options.forceRebuild) {
+            try {
+                const { existsSync: fsExistsSync, unlinkSync } = await import('fs');
+                console.log('🗑️ Deleting existing database and index to perform a clean rebuild...');
+                // Remove WAL/SHM if present (common on SQLite with WAL journaling).
+                const sidecars = [`${dbPath}-wal`, `${dbPath}-shm`];
+                for (const p of sidecars) {
+                    if (fsExistsSync(p)) {
+                        try {
+                            unlinkSync(p);
+                        }
+                        catch (e) {
+                            console.warn(`⚠️ Could not remove SQLite sidecar file (${p}):`, e);
+                        }
+                    }
+                }
+                if (fsExistsSync(dbPath)) {
+                    try {
+                        unlinkSync(dbPath);
+                    }
+                    catch (e) {
+                        console.warn(`⚠️ Could not remove database file (${dbPath}):`, e);
+                    }
+                }
+                if (fsExistsSync(indexPath)) {
+                    try {
+                        unlinkSync(indexPath);
+                    }
+                    catch (e) {
+                        console.warn(`⚠️ Could not remove index file (${indexPath}):`, e);
+                    }
+                }
+            }
+            catch (error) {
+                console.warn('⚠️ Could not delete existing database/index for clean rebuild:', error instanceof Error ? error.message : String(error));
+            }
+        }
         // Setup graceful cleanup
         setupCLICleanup(dbPath);
         // Check if database is busy before starting
@@ -218,6 +244,22 @@ export async function runIngest(path, options = {}) {
                 console.log(`Processing rate: ${chunksPerSecond} chunks/second`);
             }
             console.log('\nIngestion completed successfully!');
+            // Run VACUUM to compact the SQLite database after ingestion
+            try {
+                const { openDatabase } = await import('../core/db.js');
+                const vacuumDb = await openDatabase(dbPath);
+                try {
+                    console.log('Running VACUUM to optimize database size...');
+                    await vacuumDb.run('VACUUM');
+                    console.log('VACUUM completed successfully.');
+                }
+                finally {
+                    await vacuumDb.close();
+                }
+            }
+            catch (vacuumError) {
+                console.warn('⚠️  VACUUM operation failed or was skipped:', vacuumError instanceof Error ? vacuumError.message : String(vacuumError));
+            }
             // Display mode-specific information
             const mode = options.mode || 'text';
             if (mode === 'multimodal') {
@@ -397,6 +439,22 @@ export async function runRebuild() {
                 console.log('All embeddings have been regenerated with the current model.');
                 console.log('');
                 console.log('You can now search your documents using: raglite search "your query"');
+                // Run VACUUM to compact the SQLite database after rebuild
+                try {
+                    const { openDatabase } = await import('../core/db.js');
+                    const vacuumDb = await openDatabase(dbPath);
+                    try {
+                        console.log('Running VACUUM to optimize database size after rebuild...');
+                        await vacuumDb.run('VACUUM');
+                        console.log('VACUUM completed successfully.');
+                    }
+                    finally {
+                        await vacuumDb.close();
+                    }
+                }
+                catch (vacuumError) {
+                    console.warn('⚠️  VACUUM operation failed or was skipped:', vacuumError instanceof Error ? vacuumError.message : String(vacuumError));
+                }
             }
             finally {
                 await db.close();