rag-lite-ts 2.3.0 → 2.3.1

package/dist/esm/core/prompt-templates.js

@@ -0,0 +1,225 @@
+/**
+ * CORE MODULE — Prompt Templates for RAG Response Generation
+ *
+ * Provides prompt engineering utilities for different generator model types.
+ * Handles context formatting, token budget management, and system prompts.
+ *
+ * PROMPT STRATEGIES:
+ * - Instruct models: Use chat template with system/user/assistant roles
+ * - Causal LM models: Use simple document + question format
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+// =============================================================================
+// DEFAULT PROMPTS
+// =============================================================================
+/**
+ * Default system prompt for instruct models
+ * Emphasizes grounded responses using only provided context
+ */
+export const DEFAULT_SYSTEM_PROMPT = `You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:
+
+1. Answer ONLY using information found in the context documents
+2. If the answer cannot be found in the context, say "I cannot find this information in the provided documents"
+3. Do not make up information or use external knowledge
+4. Be concise and direct in your response
+5. If the context is incomplete or unclear, acknowledge this limitation`;
+/**
+ * Default system prompt for RAG with source attribution
+ */
+export const DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION = `You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:
+
+1. Answer ONLY using information found in the context documents
+2. When possible, mention which document the information comes from
+3. If the answer cannot be found in the context, say "I cannot find this information in the provided documents"
+4. Do not make up information or use external knowledge
+5. Be concise and direct in your response`;
+// =============================================================================
+// CHAT TEMPLATES
+// =============================================================================
+/**
+ * SmolLM2 chat template format
+ * Uses <|im_start|> and <|im_end|> tokens
+ */
+export const SMOLLM2_CHAT_TEMPLATE = {
+    systemStart: '<|im_start|>system\n',
+    systemEnd: '<|im_end|>\n',
+    userStart: '<|im_start|>user\n',
+    userEnd: '<|im_end|>\n',
+    assistantStart: '<|im_start|>assistant\n',
+    assistantEnd: '<|im_end|>',
+    endOfText: '<|endoftext|>'
+};
+/**
+ * Format search result chunks into context string for the prompt
+ *
+ * @param chunks - Search result chunks to format
+ * @param options - Formatting options
+ * @returns Formatted context with metadata
+ */
+export function formatContextChunks(chunks, options) {
+    const { maxContextTokens, includeDocumentInfo = true, includeScores = false, chunkSeparator = '\n---\n', tokenEstimationRatio = 4 // ~4 chars per token for English
+     } = options;
+    const maxChars = maxContextTokens * tokenEstimationRatio;
+    let currentChars = 0;
+    const includedChunks = [];
+    let truncated = false;
+    for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        // Format this chunk
+        let chunkText = '';
+        if (includeDocumentInfo) {
+            chunkText += `[Document ${i + 1}: ${chunk.document.title}]`;
+            if (includeScores) {
+                chunkText += ` (relevance: ${(chunk.score * 100).toFixed(1)}%)`;
+            }
+            chunkText += '\n';
+        }
+        chunkText += chunk.content;
+        // Check if adding this chunk would exceed budget
+        const chunkChars = chunkText.length + (includedChunks.length > 0 ? chunkSeparator.length : 0);
+        if (currentChars + chunkChars > maxChars) {
+            // Check if we can fit a truncated version of this chunk
+            const remainingChars = maxChars - currentChars - (includedChunks.length > 0 ? chunkSeparator.length : 0);
+            if (remainingChars > 100 && includedChunks.length === 0) {
+                // Truncate the first chunk if it's the only option
+                chunkText = chunkText.substring(0, remainingChars - 20) + '\n[Content truncated...]';
+                includedChunks.push(chunkText);
+                currentChars += chunkText.length;
+            }
+            truncated = true;
+            break;
+        }
+        includedChunks.push(chunkText);
+        currentChars += chunkChars;
+    }
+    const text = includedChunks.join(chunkSeparator);
+    const estimatedTokens = Math.ceil(text.length / tokenEstimationRatio);
+    return {
+        text,
+        estimatedTokens,
+        chunksIncluded: includedChunks.length,
+        totalChunks: chunks.length,
+        truncated
+    };
+}
+/**
+ * Build a complete prompt for the generator model
+ *
+ * @param options - Prompt building options
+ * @returns Built prompt with metadata
+ */
+export function buildPrompt(options) {
+    const { query, chunks, modelType, systemPrompt, maxContextLength, reservedOutputTokens, includeSourceAttribution = false } = options;
+    // Calculate available tokens for context
+    const promptOverhead = modelType === 'instruct' ? 150 : 50; // Tokens for formatting
+    const queryTokens = Math.ceil(query.length / 4);
+    const availableContextTokens = maxContextLength - reservedOutputTokens - promptOverhead - queryTokens;
+    // Format context chunks
+    const contextInfo = formatContextChunks(chunks, {
+        maxContextTokens: availableContextTokens,
+        includeDocumentInfo: true,
+        includeScores: false
+    });
+    // Build prompt based on model type
+    let prompt;
+    let systemPromptUsed;
+    if (modelType === 'instruct') {
+        prompt = buildInstructPrompt(query, contextInfo.text, systemPrompt, includeSourceAttribution);
+        systemPromptUsed = systemPrompt || (includeSourceAttribution ? DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION : DEFAULT_SYSTEM_PROMPT);
+    }
+    else {
+        prompt = buildCausalLMPrompt(query, contextInfo.text);
+    }
+    const estimatedTokens = Math.ceil(prompt.length / 4);
+    return {
+        prompt,
+        estimatedTokens,
+        contextInfo,
+        systemPromptUsed
+    };
+}
+/**
+ * Build prompt for instruct models (SmolLM2-Instruct)
+ * Uses chat template format with system/user/assistant roles
+ */
+function buildInstructPrompt(query, context, customSystemPrompt, includeSourceAttribution = false) {
+    const systemPrompt = customSystemPrompt ||
+        (includeSourceAttribution ? DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION : DEFAULT_SYSTEM_PROMPT);
+    const template = SMOLLM2_CHAT_TEMPLATE;
+    const userMessage = `Context:
+${context}
+
+Question: ${query}
+
+Answer based only on the context above:`;
+    return `${template.systemStart}${systemPrompt}${template.systemEnd}${template.userStart}${userMessage}${template.userEnd}${template.assistantStart}`;
+}
+/**
+ * Build prompt for causal LM models (DistilGPT2)
+ * Uses simple document + question format without roles
+ */
+function buildCausalLMPrompt(query, context) {
+    return `The following documents contain information to answer the question.
+
+Documents:
+${context}
+
+Based on the documents above, answer this question: ${query}
+
+Answer:`;
+}
+// =============================================================================
+// TOKEN ESTIMATION
+// =============================================================================
+/**
+ * Estimate token count for a string
+ * Uses a simple character-based heuristic (~4 chars per token for English)
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export function estimateTokenCount(text) {
+    // Simple heuristic: ~4 characters per token for English text
+    // This is a rough approximation; actual tokenization varies by model
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Calculate available context budget
+ *
+ * @param maxContextLength - Maximum context window size
+ * @param reservedOutputTokens - Tokens reserved for generation
+ * @param promptOverhead - Tokens used by prompt formatting
+ * @returns Available tokens for context chunks
+ */
+export function calculateContextBudget(maxContextLength, reservedOutputTokens, promptOverhead = 100) {
+    return Math.max(0, maxContextLength - reservedOutputTokens - promptOverhead);
+}
+// =============================================================================
+// STOP SEQUENCES
+// =============================================================================
+/**
+ * Get default stop sequences for a model type
+ *
+ * @param modelType - Generator model type
+ * @returns Array of stop sequences
+ */
+export function getDefaultStopSequences(modelType) {
+    if (modelType === 'instruct') {
+        return [
+            SMOLLM2_CHAT_TEMPLATE.assistantEnd,
+            SMOLLM2_CHAT_TEMPLATE.endOfText,
+            '<|im_start|>',
+            '\n\nQuestion:',
+            '\n\nContext:'
+        ];
+    }
+    // Causal LM stop sequences
+    return [
+        '\n\nQuestion:',
+        '\n\nDocuments:',
+        '\n\n---',
+        '<|endoftext|>'
+    ];
+}
+//# sourceMappingURL=prompt-templates.js.map

package/dist/esm/core/response-generator.d.ts

@@ -0,0 +1,132 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { SearchResult } from './types.js';
+/** Supported generator model types */
+export type GeneratorModelType = 'causal-lm' | 'instruct';
+/** Generation request containing query and retrieved context */
+export interface GenerationRequest {
+    query: string;
+    chunks: SearchResult[];
+    systemPrompt?: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    topK?: number;
+    repetitionPenalty?: number;
+    stopSequences?: string[];
+    includeSourceAttribution?: boolean;
+    /** Maximum number of chunks to include in context (overrides model default) */
+    maxChunksForContext?: number;
+}
+/** Result of text generation */
+export interface GenerationResult {
+    response: string;
+    tokensUsed: number;
+    truncated: boolean;
+    modelName: string;
+    generationTimeMs: number;
+    metadata: {
+        promptTokens: number;
+        completionTokens: number;
+        chunksIncluded: number;
+        totalChunks: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    };
+}
+/** Generator model capabilities */
+export interface GeneratorCapabilities {
+    supportsStreaming: boolean;
+    supportsSystemPrompt: boolean;
+    instructionTuned: boolean;
+    maxContextLength: number;
+    defaultMaxOutputTokens: number;
+    recommendedTemperature: number;
+    /** Maximum number of chunks to use for context (default varies by model) */
+    defaultMaxChunksForContext: number;
+}
+/** Generator model requirements */
+export interface GeneratorRequirements {
+    transformersJsVersion: string;
+    minimumMemory: number;
+    requiredFeatures: readonly string[];
+    platformSupport: readonly string[];
+}
+/** Complete generator model information */
+export interface GeneratorModelInfo {
+    name: string;
+    type: GeneratorModelType;
+    version: string;
+    capabilities: GeneratorCapabilities;
+    requirements: GeneratorRequirements;
+    isDefault?: boolean;
+    description?: string;
+}
+/** Generator validation result */
+export interface GeneratorValidationResult {
+    isValid: boolean;
+    errors: string[];
+    warnings: string[];
+    suggestions: string[];
+}
+/** Options for creating generator instances */
+export interface GeneratorCreationOptions {
+    cachePath?: string;
+    timeout?: number;
+    enableGPU?: boolean;
+    defaultGenerationOptions?: Partial<GenerationRequest>;
+    customConfig?: Record<string, any>;
+}
+/**
+ * Universal response generator interface
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export interface ResponseGenerator {
+    readonly modelName: string;
+    readonly modelType: GeneratorModelType;
+    readonly maxContextLength: number;
+    readonly maxOutputLength: number;
+    generate(request: GenerationRequest): Promise<GenerationResult>;
+    generateStream?(request: GenerationRequest): AsyncIterable<string>;
+    loadModel(): Promise<void>;
+    isLoaded(): boolean;
+    getModelInfo(): GeneratorModelInfo;
+    cleanup(): Promise<void>;
+}
+export type GenerateFunction = (query: string, chunks: SearchResult[], options?: Partial<GenerationRequest>) => Promise<GenerationResult>;
+export type CreateGeneratorFunction = (modelName: string, options?: GeneratorCreationOptions) => Promise<ResponseGenerator>;
+export declare class GeneratorValidationError extends Error {
+    readonly modelName: string;
+    readonly availableModels: readonly string[];
+    constructor(modelName: string, availableModels: readonly string[], message: string);
+}
+export declare class GenerationError extends Error {
+    readonly modelName: string;
+    readonly stage: 'loading' | 'tokenization' | 'generation' | 'decoding';
+    readonly cause?: Error | undefined;
+    constructor(modelName: string, stage: 'loading' | 'tokenization' | 'generation' | 'decoding', message: string, cause?: Error | undefined);
+}
+export declare class ContextWindowError extends Error {
+    readonly requiredTokens: number;
+    readonly availableTokens: number;
+    constructor(requiredTokens: number, availableTokens: number, message: string);
+}
+export declare function supportsStreaming(generator: ResponseGenerator): generator is ResponseGenerator & {
+    generateStream(request: GenerationRequest): AsyncIterable<string>;
+};
+export declare function isInstructModel(generator: ResponseGenerator): boolean;
+export declare function createGenerateFunction(generator: ResponseGenerator): GenerateFunction;
+//# sourceMappingURL=response-generator.d.ts.map

package/dist/esm/core/response-generator.js

@@ -0,0 +1,69 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+// =============================================================================
+// ERROR CLASSES
+// =============================================================================
+export class GeneratorValidationError extends Error {
+    modelName;
+    availableModels;
+    constructor(modelName, availableModels, message) {
+        super(message);
+        this.modelName = modelName;
+        this.availableModels = availableModels;
+        this.name = 'GeneratorValidationError';
+    }
+}
+export class GenerationError extends Error {
+    modelName;
+    stage;
+    cause;
+    constructor(modelName, stage, message, cause) {
+        super(message);
+        this.modelName = modelName;
+        this.stage = stage;
+        this.cause = cause;
+        this.name = 'GenerationError';
+    }
+}
+export class ContextWindowError extends Error {
+    requiredTokens;
+    availableTokens;
+    constructor(requiredTokens, availableTokens, message) {
+        super(message);
+        this.requiredTokens = requiredTokens;
+        this.availableTokens = availableTokens;
+        this.name = 'ContextWindowError';
+    }
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+export function supportsStreaming(generator) {
+    return typeof generator.generateStream === 'function';
+}
+export function isInstructModel(generator) {
+    return generator.modelType === 'instruct';
+}
+export function createGenerateFunction(generator) {
+    return async (query, chunks, options) => {
+        if (!generator.isLoaded()) {
+            await generator.loadModel();
+        }
+        return generator.generate({ query, chunks, ...options });
+    };
+}
+//# sourceMappingURL=response-generator.js.map

package/dist/esm/core/search.d.ts

@@ -6,17 +6,54 @@ import { IndexManager } from '../index-manager.js';
 import { DatabaseConnection } from './db.js';
 import type { SearchResult, SearchOptions } from './types.js';
 import type { EmbedFunction, RerankFunction } from './interfaces.js';
+import type { GenerateFunction } from './response-generator.js';
 /**
  * Search engine that provides semantic search capabilities
  * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
  * Uses explicit dependency injection for clean architecture
  */
+/**
+ * Extended search options with generation support
+ * @experimental Generation features are experimental
+ */
+export interface ExtendedSearchOptions extends SearchOptions {
+    /** Enable AI response generation from search results */
+    generateResponse?: boolean;
+    /** Generator model to use (default: SmolLM2-135M-Instruct) */
+    generatorModel?: string;
+    /** Generation options */
+    generationOptions?: {
+        maxTokens?: number;
+        temperature?: number;
+        systemPrompt?: string;
+        /** Maximum chunks to use for context (overrides model default) */
+        maxChunksForContext?: number;
+    };
+}
+/**
+ * Search result with optional generated response
+ * @experimental Generation features are experimental
+ */
+export interface SearchResultWithGeneration {
+    /** Search results (chunks) */
+    results: SearchResult[];
+    /** Generated response (if generation was enabled) */
+    generation?: {
+        response: string;
+        modelUsed: string;
+        tokensUsed: number;
+        truncated: boolean;
+        chunksUsedForContext: number;
+        generationTimeMs: number;
+    };
+}
 export declare class SearchEngine {
     private embedFn;
     private indexManager;
     private db;
     private rerankFn?;
     private contentResolver?;
+    private generateFn?;
     /**
      * Creates a new SearchEngine with explicit dependency injection
      *
@@ -71,7 +108,41 @@ export declare class SearchEngine {
      * const search = new SearchEngine(customEmbedFn, indexManager, db);
      * ```
      */
-    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver);
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver, generateFn?: GenerateFunction);
+    /**
+     * Set or update the generate function
+     * @experimental This method is experimental and may change
+     */
+    setGenerateFunction(generateFn: GenerateFunction | undefined): void;
+    /**
+     * Check if generation is available
+     * @experimental This method is experimental and may change
+     */
+    hasGenerationCapability(): boolean;
+    /**
+     * Perform semantic search with optional AI response generation
+     *
+     * This method extends the standard search with optional response generation.
+     * When generation is enabled, the retrieved chunks are used as context for
+     * an AI model to generate a synthesized response.
+     *
+     * @param query - Search query string
+     * @param options - Extended search options including generation settings
+     * @returns Promise resolving to search results with optional generated response
+     *
+     * @example
+     * ```typescript
+     * // Search with generation
+     * const result = await search.searchWithGeneration('How does auth work?', {
+     *   top_k: 5,
+     *   generateResponse: true
+     * });
+     * console.log(result.generation?.response);
+     * ```
+     *
+     * @experimental This method is experimental and may change in future versions.
+     */
+    searchWithGeneration(query: string, options?: ExtendedSearchOptions): Promise<SearchResultWithGeneration>;
     /**
      * Perform semantic search on the indexed documents
      * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking

package/dist/esm/core/search.js

@@ -5,17 +5,13 @@
 import { getChunksByEmbeddingIds } from './db.js';
 import { config } from './config.js';
 import { createMissingDependencyError } from './actionable-error-messages.js';
-/**
- * Search engine that provides semantic search capabilities
- * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
- * Uses explicit dependency injection for clean architecture
- */
 export class SearchEngine {
     embedFn;
     indexManager;
     db;
     rerankFn;
     contentResolver;
+    generateFn;
     /**
      * Creates a new SearchEngine with explicit dependency injection
      *
@@ -70,7 +66,7 @@ export class SearchEngine {
      * const search = new SearchEngine(customEmbedFn, indexManager, db);
      * ```
      */
-    constructor(embedFn, indexManager, db, rerankFn, contentResolver) {
+    constructor(embedFn, indexManager, db, rerankFn, contentResolver, generateFn) {
         this.embedFn = embedFn;
         this.indexManager = indexManager;
         this.db = db;
@@ -93,6 +89,83 @@ export class SearchEngine {
         }
         // Initialize ContentResolver if provided, or create lazily when needed
         this.contentResolver = contentResolver;
+        // Initialize GenerateFunction if provided (experimental)
+        this.generateFn = generateFn;
+    }
+    /**
+     * Set or update the generate function
+     * @experimental This method is experimental and may change
+     */
+    setGenerateFunction(generateFn) {
+        this.generateFn = generateFn;
+    }
+    /**
+     * Check if generation is available
+     * @experimental This method is experimental and may change
+     */
+    hasGenerationCapability() {
+        return this.generateFn !== undefined;
+    }
+    /**
+     * Perform semantic search with optional AI response generation
+     *
+     * This method extends the standard search with optional response generation.
+     * When generation is enabled, the retrieved chunks are used as context for
+     * an AI model to generate a synthesized response.
+     *
+     * @param query - Search query string
+     * @param options - Extended search options including generation settings
+     * @returns Promise resolving to search results with optional generated response
+     *
+     * @example
+     * ```typescript
+     * // Search with generation
+     * const result = await search.searchWithGeneration('How does auth work?', {
+     *   top_k: 5,
+     *   generateResponse: true
+     * });
+     * console.log(result.generation?.response);
+     * ```
+     *
+     * @experimental This method is experimental and may change in future versions.
+     */
+    async searchWithGeneration(query, options = {}) {
+        // Perform standard search
+        const results = await this.search(query, options);
+        // If generation not requested or no results, return without generation
+        if (!options.generateResponse || results.length === 0) {
+            return { results };
+        }
+        // Check if generation is available
+        if (!this.generateFn) {
+            console.warn('⚠️ [EXPERIMENTAL] Generation requested but no generator configured');
+            return { results };
+        }
+        try {
+            console.log('🤖 [EXPERIMENTAL] Generating response from search results...');
+            const generationResult = await this.generateFn(query, results, {
+                maxTokens: options.generationOptions?.maxTokens,
+                temperature: options.generationOptions?.temperature,
+                systemPrompt: options.generationOptions?.systemPrompt,
+                maxChunksForContext: options.generationOptions?.maxChunksForContext
+            });
+            return {
+                results,
+                generation: {
+                    response: generationResult.response,
+                    modelUsed: generationResult.modelName,
+                    tokensUsed: generationResult.tokensUsed,
+                    truncated: generationResult.truncated,
+                    chunksUsedForContext: generationResult.metadata.chunksIncluded,
+                    generationTimeMs: generationResult.generationTimeMs
+                }
+            };
+        }
+        catch (error) {
+            console.error('❌ [EXPERIMENTAL] Generation failed:', error instanceof Error ? error.message : 'Unknown error');
+            // Return results without generation on error
+            return { results };
+        }
     }
     /**
      * Perform semantic search on the indexed documents

package/dist/esm/core/types.d.ts

@@ -63,4 +63,5 @@ export interface Document {
 }
 export type { DatabaseConnection } from './db.js';
 export type { ContentChunk as ChunkResult } from './db.js';
+export type { GenerationRequest, GenerationResult, GenerateFunction, ResponseGenerator, GeneratorModelInfo } from './response-generator.js';
 //# sourceMappingURL=types.d.ts.map

package/dist/esm/core/vector-index-worker.js

@@ -242,6 +242,16 @@ function handleSetEf(payload) {
 function handleIndexExists(payload) {
     return existsSync(payload.indexPath);
 }
+// Handle unhandled promise rejections to prevent worker crashes
+process.on('unhandledRejection', (reason, promise) => {
+    console.error('Worker unhandled rejection:', reason);
+    // Don't exit - just log the error
+});
+// Handle uncaught exceptions
+process.on('uncaughtException', (error) => {
+    console.error('Worker uncaught exception:', error);
+    // Don't exit - just log the error
+});
 // Main message handler
 parentPort.on('message', async (request) => {
     try {