rag-lite-ts 2.3.0 → 2.3.1

package/dist/esm/core/vector-index.js

@@ -7,7 +7,7 @@
 import { Worker } from 'worker_threads';
 import { existsSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { dirname, join } from 'path';
+import { dirname, join, resolve } from 'path';
 import { handleError, ErrorCategory, ErrorSeverity, createError } from './error-handler.js';
 import { createMissingFileError, createDimensionMismatchError } from './actionable-error-messages.js';
 export class VectorIndex {
@@ -34,20 +34,40 @@ export class VectorIndex {
         const currentFile = fileURLToPath(import.meta.url);
         const currentDir = dirname(currentFile);
         // Always prefer .js (compiled output)
-        const jsPath = join(currentDir, 'vector-index-worker.js');
+        const jsPath = resolve(join(currentDir, 'vector-index-worker.js'));
         // Check if .js exists in current directory (compiled)
         if (existsSync(jsPath)) {
             return jsPath;
         }
-        // If running from src/ (development), try dist/ paths
-        if (currentDir.includes('src')) {
-            // Find project root (go up from src/core)
-            const projectRoot = currentDir.replace(/[\\/]src[\\/]core.*$/, '');
-            const distEsmPath = join(projectRoot, 'dist', 'esm', 'core', 'vector-index-worker.js');
-            const distCjsPath = join(projectRoot, 'dist', 'cjs', 'core', 'vector-index-worker.js');
+        // Helper function to find project root
+        const findProjectRoot = () => {
+            let dir = currentDir;
+            // Look for dist/ in the path
+            const distMatch = dir.match(/^(.+?)[\\/]dist[\\/]/);
+            if (distMatch) {
+                return distMatch[1];
+            }
+            // If no dist/, try going up from src/core
+            const srcMatch = dir.match(/^(.+?)[\\/]src[\\/]core/);
+            if (srcMatch) {
+                return srcMatch[1];
+            }
+            // If in node_modules, extract package root
+            const nodeModulesMatch = dir.match(/^(.+?)[\\/]node_modules/);
+            if (nodeModulesMatch) {
+                return nodeModulesMatch[1];
+            }
+            return null;
+        };
+        const projectRoot = findProjectRoot();
+        if (projectRoot) {
+            // Try ESM first (preferred for ES modules)
+            const distEsmPath = resolve(join(projectRoot, 'dist', 'esm', 'core', 'vector-index-worker.js'));
             if (existsSync(distEsmPath)) {
                 return distEsmPath;
             }
+            // Try CJS as fallback
+            const distCjsPath = resolve(join(projectRoot, 'dist', 'cjs', 'core', 'vector-index-worker.js'));
             if (existsSync(distCjsPath)) {
                 return distCjsPath;
             }
@@ -55,8 +75,8 @@ export class VectorIndex {
         // If running from node_modules (installed package), try dist paths
         if (currentDir.includes('node_modules')) {
             const packageRoot = currentDir.split('node_modules')[0];
-            const distEsmPath = join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'esm', 'core', 'vector-index-worker.js');
-            const distCjsPath = join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'cjs', 'core', 'vector-index-worker.js');
+            const distEsmPath = resolve(join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'esm', 'core', 'vector-index-worker.js'));
+            const distCjsPath = resolve(join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'cjs', 'core', 'vector-index-worker.js'));
             if (existsSync(distEsmPath)) {
                 return distEsmPath;
             }
@@ -68,7 +88,8 @@ export class VectorIndex {
         throw new Error(`Worker file not found. Expected: ${jsPath}\n` +
             'Please run "npm run build" to compile the vector-index-worker.ts file.\n' +
             `Current directory: ${currentDir}\n` +
-            `Checked paths: ${jsPath}, ${currentDir.includes('src') ? join(currentDir.replace(/[\\/]src[\\/]core.*$/, ''), 'dist', 'esm', 'core', 'vector-index-worker.js') : 'N/A'}`);
+            `Project root: ${projectRoot || 'not found'}\n` +
+            `Checked paths: ${jsPath}${projectRoot ? `, ${join(projectRoot, 'dist', 'esm', 'core', 'vector-index-worker.js')}, ${join(projectRoot, 'dist', 'cjs', 'core', 'vector-index-worker.js')}` : ''}`);
     }
     /**
      * Ensure worker is created and ready
@@ -326,18 +347,47 @@ export class VectorIndex {
      */
     async cleanup() {
         if (this.worker) {
+            const workerToTerminate = this.worker;
+            // Clear state first to prevent new operations
+            this.worker = null;
+            this.isInitialized = false;
             try {
-                // Send cleanup message (worker will acknowledge)
-                await this.sendMessage('cleanup');
+                // Send cleanup message (worker will acknowledge) with timeout
+                // Use the worker directly since we've cleared this.worker
+                const cleanupPromise = new Promise((resolve, reject) => {
+                    const id = this.messageId++;
+                    const timeout = setTimeout(() => {
+                        this.messageQueue.delete(id);
+                        reject(new Error('Cleanup timeout'));
+                    }, 1000);
+                    this.messageQueue.set(id, {
+                        resolve: () => {
+                            clearTimeout(timeout);
+                            resolve();
+                        },
+                        reject: (error) => {
+                            clearTimeout(timeout);
+                            reject(error);
+                        }
+                    });
+                    workerToTerminate.postMessage({ id, type: 'cleanup', payload: undefined });
+                });
+                await cleanupPromise;
             }
             catch (error) {
-                // Ignore errors during cleanup
+                // Ignore errors during cleanup - worker might already be terminating
+            }
+            finally {
+                // Clear message queue
+                this.messageQueue.clear();
+                // Terminate worker - this frees ALL WebAssembly memory
+                try {
+                    await workerToTerminate.terminate();
+                }
+                catch (error) {
+                    // Ignore termination errors
+                }
             }
-            // Terminate worker - this frees ALL WebAssembly memory
-            await this.worker.terminate();
-            this.worker = null;
-            this.isInitialized = false;
-            this.messageQueue.clear();
         }
     }
 }

package/dist/esm/factories/generator-factory.d.ts

@@ -0,0 +1,88 @@
+/**
+ * FACTORY MODULE — Generator Factory for RAG Response Generation
+ *
+ * Factory functions for creating response generator instances.
+ * Handles model validation, lazy loading, and proper initialization.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, DEFAULT, uses top 3 chunks)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, uses top 5 chunks)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { ResponseGenerator, GeneratorCreationOptions, GenerateFunction } from '../core/response-generator.js';
+/**
+ * Create a response generator for the specified model
+ *
+ * Uses lazy loading to defer model initialization until first use.
+ * Validates model compatibility before creation.
+ *
+ * @param modelName - Name of the generator model (default: SmolLM2-135M-Instruct)
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a ResponseGenerator instance
+ * @throws {GeneratorValidationError} If model is not supported
+ *
+ * @example
+ * ```typescript
+ * // Create default generator (recommended)
+ * const generator = await createResponseGenerator();
+ *
+ * // Create higher quality generator
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct');
+ *
+ * // Create with options
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct', {
+ *   cachePath: './models'
+ * });
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export declare function createResponseGenerator(modelName?: string, options?: GeneratorCreationOptions): Promise<ResponseGenerator>;
+/**
+ * Create a GenerateFunction from a model name
+ *
+ * This is a convenience function that creates a generator and wraps it
+ * in a function suitable for dependency injection into SearchEngine.
+ *
+ * @param modelName - Name of the generator model
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a GenerateFunction
+ *
+ * @example
+ * ```typescript
+ * const generateFn = await createGenerateFunctionFromModel();
+ * const result = await generateFn(query, chunks);
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export declare function createGenerateFunctionFromModel(modelName?: string, options?: GeneratorCreationOptions): Promise<GenerateFunction>;
+/**
+ * Get the default generator model name
+ */
+export declare function getDefaultGeneratorModel(): string;
+/**
+ * List available generator models
+ */
+export declare function listGeneratorModels(): string[];
+/**
+ * Check if a model name is a valid generator
+ */
+export declare function isValidGeneratorModel(modelName: string): boolean;
+/**
+ * Get recommended generator for specific use case
+ *
+ * @param preferSpeed - Prefer faster generation over quality
+ * @param preferQuality - Prefer higher quality over speed
+ * @returns Recommended model name
+ */
+export declare function getRecommendedGenerator(options?: {
+    preferSpeed?: boolean;
+    preferQuality?: boolean;
+}): string;
+export { GeneratorRegistry, DEFAULT_GENERATOR_MODEL } from '../core/generator-registry.js';
+//# sourceMappingURL=generator-factory.d.ts.map

package/dist/esm/factories/generator-factory.js

@@ -0,0 +1,151 @@
+/**
+ * FACTORY MODULE — Generator Factory for RAG Response Generation
+ *
+ * Factory functions for creating response generator instances.
+ * Handles model validation, lazy loading, and proper initialization.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, DEFAULT, uses top 3 chunks)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, uses top 5 chunks)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import { createGenerateFunction, GeneratorValidationError } from '../core/response-generator.js';
+import { GeneratorRegistry, DEFAULT_GENERATOR_MODEL, getGeneratorType } from '../core/generator-registry.js';
+import { LazyGeneratorLoader } from '../core/lazy-dependency-loader.js';
+// =============================================================================
+// GENERATOR FACTORY
+// =============================================================================
+/**
+ * Create a response generator for the specified model
+ *
+ * Uses lazy loading to defer model initialization until first use.
+ * Validates model compatibility before creation.
+ *
+ * @param modelName - Name of the generator model (default: SmolLM2-135M-Instruct)
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a ResponseGenerator instance
+ * @throws {GeneratorValidationError} If model is not supported
+ *
+ * @example
+ * ```typescript
+ * // Create default generator (recommended)
+ * const generator = await createResponseGenerator();
+ *
+ * // Create higher quality generator
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct');
+ *
+ * // Create with options
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct', {
+ *   cachePath: './models'
+ * });
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export async function createResponseGenerator(modelName = DEFAULT_GENERATOR_MODEL, options = {}) {
+    console.log(`🏭 [EXPERIMENTAL] Creating response generator: ${modelName}`);
+    // Step 1: Validate model
+    const validation = GeneratorRegistry.validateGenerator(modelName);
+    if (!validation.isValid) {
+        throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), validation.errors.join('; '));
+    }
+    // Log warnings
+    if (validation.warnings.length > 0) {
+        console.warn(`⚠️  Warnings for generator '${modelName}':`);
+        validation.warnings.forEach(w => console.warn(`  • ${w}`));
+    }
+    // Log suggestions
+    if (validation.suggestions.length > 0) {
+        console.info(`💡 Suggestions for generator '${modelName}':`);
+        validation.suggestions.forEach(s => console.info(`  • ${s}`));
+    }
+    // Step 2: Get model type and create appropriate generator
+    const modelType = getGeneratorType(modelName);
+    if (!modelType) {
+        throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), `Could not determine model type for '${modelName}'`);
+    }
+    // Step 3: Use lazy loading to create the generator
+    let generator;
+    switch (modelType) {
+        case 'instruct':
+            generator = await LazyGeneratorLoader.loadInstructGenerator(modelName, options);
+            break;
+        case 'causal-lm':
+            generator = await LazyGeneratorLoader.loadCausalLMGenerator(modelName, options);
+            break;
+        default:
+            throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), `Unsupported generator type: ${modelType}`);
+    }
+    console.log(`✅ [EXPERIMENTAL] Response generator created: ${modelName}`);
+    return generator;
+}
+/**
+ * Create a GenerateFunction from a model name
+ *
+ * This is a convenience function that creates a generator and wraps it
+ * in a function suitable for dependency injection into SearchEngine.
+ *
+ * @param modelName - Name of the generator model
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a GenerateFunction
+ *
+ * @example
+ * ```typescript
+ * const generateFn = await createGenerateFunctionFromModel();
+ * const result = await generateFn(query, chunks);
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export async function createGenerateFunctionFromModel(modelName = DEFAULT_GENERATOR_MODEL, options = {}) {
+    const generator = await createResponseGenerator(modelName, options);
+    return createGenerateFunction(generator);
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+/**
+ * Get the default generator model name
+ */
+export function getDefaultGeneratorModel() {
+    return DEFAULT_GENERATOR_MODEL;
+}
+/**
+ * List available generator models
+ */
+export function listGeneratorModels() {
+    return GeneratorRegistry.getSupportedGenerators();
+}
+/**
+ * Check if a model name is a valid generator
+ */
+export function isValidGeneratorModel(modelName) {
+    return GeneratorRegistry.getGeneratorInfo(modelName) !== null;
+}
+/**
+ * Get recommended generator for specific use case
+ *
+ * @param preferSpeed - Prefer faster generation over quality
+ * @param preferQuality - Prefer higher quality over speed
+ * @returns Recommended model name
+ */
+export function getRecommendedGenerator(options = {}) {
+    const { preferSpeed, preferQuality } = options;
+    if (preferSpeed) {
+        return 'Xenova/distilgpt2';
+    }
+    if (preferQuality) {
+        return 'HuggingFaceTB/SmolLM2-360M-Instruct';
+    }
+    // Default: balanced option
+    return DEFAULT_GENERATOR_MODEL;
+}
+// =============================================================================
+// EXPORTS
+// =============================================================================
+export { GeneratorRegistry, DEFAULT_GENERATOR_MODEL } from '../core/generator-registry.js';
+//# sourceMappingURL=generator-factory.js.map

package/dist/esm/factories/index.d.ts

@@ -24,4 +24,5 @@
 export { IngestionFactory } from './ingestion-factory.js';
 export { SearchFactory } from './search-factory.js';
 export type { IngestionFactoryOptions, ContentSystemConfig } from './ingestion-factory.js';
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './generator-factory.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/factories/index.js

@@ -26,4 +26,9 @@ export { IngestionFactory } from './ingestion-factory.js';
 // Polymorphic search factory (recommended for automatic mode detection)
 // Re-exported from core for convenience
 export { SearchFactory } from './search-factory.js';
+// =============================================================================
+// EXPERIMENTAL: Generator Factory
+// =============================================================================
+// Generator factory for creating response generators (experimental)
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './generator-factory.js';
 //# sourceMappingURL=index.js.map

package/dist/esm/index.d.ts

@@ -73,4 +73,13 @@ export type { Chunk, Preprocessor, PreprocessorOptions, PreprocessingConfig } fr
 export type { IngestionOptions, IngestionResult } from './core/ingestion.js';
 export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError, type ErrorContext } from './core/error-handler.js';
 export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './factories/generator-factory.js';
+export type { ResponseGenerator, GeneratorModelType, GenerationRequest, GenerationResult, GeneratorCapabilities, GeneratorRequirements, GeneratorModelInfo, GeneratorValidationResult, GeneratorCreationOptions, GenerateFunction, CreateGeneratorFunction } from './core/response-generator.js';
+export { GeneratorValidationError, GenerationError, ContextWindowError } from './core/response-generator.js';
+export { supportsStreaming, isInstructModel, createGenerateFunction } from './core/response-generator.js';
+export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './core/generator-registry.js';
+export { DEFAULT_SYSTEM_PROMPT, formatContextChunks, buildPrompt, estimateTokenCount, type ContextFormattingOptions, type FormattedContext, type PromptBuildOptions, type BuiltPrompt } from './core/prompt-templates.js';
+export { InstructGenerator } from './text/generators/instruct-generator.js';
+export { CausalLMGenerator } from './text/generators/causal-lm-generator.js';
+export type { ExtendedSearchOptions, SearchResultWithGeneration } from './core/search.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/index.js

@@ -109,4 +109,20 @@ export { resolveRagLitePaths, ensureRagLiteStructure, migrateToRagLiteStructure,
 export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError } from './core/error-handler.js';
 // API-specific errors
 export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+// =============================================================================
+// EXPERIMENTAL: RESPONSE GENERATION
+// =============================================================================
+// Generator factory functions
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './factories/generator-factory.js';
+// Generator errors
+export { GeneratorValidationError, GenerationError, ContextWindowError } from './core/response-generator.js';
+// Generator utilities
+export { supportsStreaming, isInstructModel, createGenerateFunction } from './core/response-generator.js';
+// Generator registry
+export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './core/generator-registry.js';
+// Prompt templates
+export { DEFAULT_SYSTEM_PROMPT, formatContextChunks, buildPrompt, estimateTokenCount } from './core/prompt-templates.js';
+// Generator implementations
+export { InstructGenerator } from './text/generators/instruct-generator.js';
+export { CausalLMGenerator } from './text/generators/causal-lm-generator.js';
 //# sourceMappingURL=index.js.map

package/dist/esm/text/generators/causal-lm-generator.d.ts

@@ -0,0 +1,65 @@
+/**
+ * TEXT IMPLEMENTATION — Causal LM Generator for DistilGPT2
+ *
+ * Implements ResponseGenerator interface for causal language models.
+ * Supports Xenova/distilgpt2 for fast, basic text generation.
+ *
+ * Features:
+ * - Simple prompt formatting (no chat template)
+ * - Fast generation with smaller model
+ * - Streaming generation support
+ * - Resource management via ResourceManager
+ *
+ * Note: Causal LM models don't support system prompts, so responses
+ * may be less focused than instruction-tuned models.
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import '../../dom-polyfills.js';
+import { BaseResponseGenerator, type GeneratorOptions } from '../../core/abstract-generator.js';
+/**
+ * Causal LM generator implementation for DistilGPT2
+ *
+ * Uses causal language models that generate text based on simple prompts.
+ * Faster but may produce less focused responses than instruct models.
+ */
+export declare class CausalLMGenerator extends BaseResponseGenerator {
+    private pipeline;
+    private tokenizer;
+    private resourceManager;
+    private resourceId?;
+    constructor(modelName: string, options?: GeneratorOptions);
+    /**
+     * Load the causal LM model using transformers.js
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Generate text using the causal LM model
+     */
+    protected generateText(prompt: string, options: {
+        maxTokens: number;
+        temperature: number;
+        topP: number;
+        topK: number;
+        repetitionPenalty: number;
+        stopSequences: string[];
+    }): Promise<{
+        text: string;
+        promptTokens: number;
+        completionTokens: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    }>;
+    /**
+     * Generate text with streaming output
+     */
+    generateStream(request: import('../../core/response-generator.js').GenerationRequest): AsyncIterable<string>;
+    /**
+     * Count tokens in a text string
+     */
+    private countTokens;
+}
+//# sourceMappingURL=causal-lm-generator.d.ts.map