rag-lite-ts 2.2.0 → 2.3.1

package/dist/esm/core/response-generator.d.ts

@@ -0,0 +1,132 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { SearchResult } from './types.js';
+/** Supported generator model types */
+export type GeneratorModelType = 'causal-lm' | 'instruct';
+/** Generation request containing query and retrieved context */
+export interface GenerationRequest {
+    query: string;
+    chunks: SearchResult[];
+    systemPrompt?: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    topK?: number;
+    repetitionPenalty?: number;
+    stopSequences?: string[];
+    includeSourceAttribution?: boolean;
+    /** Maximum number of chunks to include in context (overrides model default) */
+    maxChunksForContext?: number;
+}
+/** Result of text generation */
+export interface GenerationResult {
+    response: string;
+    tokensUsed: number;
+    truncated: boolean;
+    modelName: string;
+    generationTimeMs: number;
+    metadata: {
+        promptTokens: number;
+        completionTokens: number;
+        chunksIncluded: number;
+        totalChunks: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    };
+}
+/** Generator model capabilities */
+export interface GeneratorCapabilities {
+    supportsStreaming: boolean;
+    supportsSystemPrompt: boolean;
+    instructionTuned: boolean;
+    maxContextLength: number;
+    defaultMaxOutputTokens: number;
+    recommendedTemperature: number;
+    /** Maximum number of chunks to use for context (default varies by model) */
+    defaultMaxChunksForContext: number;
+}
+/** Generator model requirements */
+export interface GeneratorRequirements {
+    transformersJsVersion: string;
+    minimumMemory: number;
+    requiredFeatures: readonly string[];
+    platformSupport: readonly string[];
+}
+/** Complete generator model information */
+export interface GeneratorModelInfo {
+    name: string;
+    type: GeneratorModelType;
+    version: string;
+    capabilities: GeneratorCapabilities;
+    requirements: GeneratorRequirements;
+    isDefault?: boolean;
+    description?: string;
+}
+/** Generator validation result */
+export interface GeneratorValidationResult {
+    isValid: boolean;
+    errors: string[];
+    warnings: string[];
+    suggestions: string[];
+}
+/** Options for creating generator instances */
+export interface GeneratorCreationOptions {
+    cachePath?: string;
+    timeout?: number;
+    enableGPU?: boolean;
+    defaultGenerationOptions?: Partial<GenerationRequest>;
+    customConfig?: Record<string, any>;
+}
+/**
+ * Universal response generator interface
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export interface ResponseGenerator {
+    readonly modelName: string;
+    readonly modelType: GeneratorModelType;
+    readonly maxContextLength: number;
+    readonly maxOutputLength: number;
+    generate(request: GenerationRequest): Promise<GenerationResult>;
+    generateStream?(request: GenerationRequest): AsyncIterable<string>;
+    loadModel(): Promise<void>;
+    isLoaded(): boolean;
+    getModelInfo(): GeneratorModelInfo;
+    cleanup(): Promise<void>;
+}
+export type GenerateFunction = (query: string, chunks: SearchResult[], options?: Partial<GenerationRequest>) => Promise<GenerationResult>;
+export type CreateGeneratorFunction = (modelName: string, options?: GeneratorCreationOptions) => Promise<ResponseGenerator>;
+export declare class GeneratorValidationError extends Error {
+    readonly modelName: string;
+    readonly availableModels: readonly string[];
+    constructor(modelName: string, availableModels: readonly string[], message: string);
+}
+export declare class GenerationError extends Error {
+    readonly modelName: string;
+    readonly stage: 'loading' | 'tokenization' | 'generation' | 'decoding';
+    readonly cause?: Error | undefined;
+    constructor(modelName: string, stage: 'loading' | 'tokenization' | 'generation' | 'decoding', message: string, cause?: Error | undefined);
+}
+export declare class ContextWindowError extends Error {
+    readonly requiredTokens: number;
+    readonly availableTokens: number;
+    constructor(requiredTokens: number, availableTokens: number, message: string);
+}
+export declare function supportsStreaming(generator: ResponseGenerator): generator is ResponseGenerator & {
+    generateStream(request: GenerationRequest): AsyncIterable<string>;
+};
+export declare function isInstructModel(generator: ResponseGenerator): boolean;
+export declare function createGenerateFunction(generator: ResponseGenerator): GenerateFunction;
+//# sourceMappingURL=response-generator.d.ts.map

package/dist/esm/core/response-generator.js

@@ -0,0 +1,69 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+// =============================================================================
+// ERROR CLASSES
+// =============================================================================
+export class GeneratorValidationError extends Error {
+    modelName;
+    availableModels;
+    constructor(modelName, availableModels, message) {
+        super(message);
+        this.modelName = modelName;
+        this.availableModels = availableModels;
+        this.name = 'GeneratorValidationError';
+    }
+}
+export class GenerationError extends Error {
+    modelName;
+    stage;
+    cause;
+    constructor(modelName, stage, message, cause) {
+        super(message);
+        this.modelName = modelName;
+        this.stage = stage;
+        this.cause = cause;
+        this.name = 'GenerationError';
+    }
+}
+export class ContextWindowError extends Error {
+    requiredTokens;
+    availableTokens;
+    constructor(requiredTokens, availableTokens, message) {
+        super(message);
+        this.requiredTokens = requiredTokens;
+        this.availableTokens = availableTokens;
+        this.name = 'ContextWindowError';
+    }
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+export function supportsStreaming(generator) {
+    return typeof generator.generateStream === 'function';
+}
+export function isInstructModel(generator) {
+    return generator.modelType === 'instruct';
+}
+export function createGenerateFunction(generator) {
+    return async (query, chunks, options) => {
+        if (!generator.isLoaded()) {
+            await generator.loadModel();
+        }
+        return generator.generate({ query, chunks, ...options });
+    };
+}
+//# sourceMappingURL=response-generator.js.map

package/dist/esm/core/search-pipeline.js

@@ -123,7 +123,7 @@ export class SearchPipelineCoordinator {
             throw new Error('Index manager not set. Set index manager before executing pipeline.');
         }
         try {
-            return this.indexManager.search(queryVector, topK);
+            return await this.indexManager.search(queryVector, topK);
         }
         catch (error) {
             if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {

package/dist/esm/core/search.d.ts

@@ -6,17 +6,54 @@ import { IndexManager } from '../index-manager.js';
 import { DatabaseConnection } from './db.js';
 import type { SearchResult, SearchOptions } from './types.js';
 import type { EmbedFunction, RerankFunction } from './interfaces.js';
+import type { GenerateFunction } from './response-generator.js';
 /**
  * Search engine that provides semantic search capabilities
  * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
  * Uses explicit dependency injection for clean architecture
  */
+/**
+ * Extended search options with generation support
+ * @experimental Generation features are experimental
+ */
+export interface ExtendedSearchOptions extends SearchOptions {
+    /** Enable AI response generation from search results */
+    generateResponse?: boolean;
+    /** Generator model to use (default: SmolLM2-135M-Instruct) */
+    generatorModel?: string;
+    /** Generation options */
+    generationOptions?: {
+        maxTokens?: number;
+        temperature?: number;
+        systemPrompt?: string;
+        /** Maximum chunks to use for context (overrides model default) */
+        maxChunksForContext?: number;
+    };
+}
+/**
+ * Search result with optional generated response
+ * @experimental Generation features are experimental
+ */
+export interface SearchResultWithGeneration {
+    /** Search results (chunks) */
+    results: SearchResult[];
+    /** Generated response (if generation was enabled) */
+    generation?: {
+        response: string;
+        modelUsed: string;
+        tokensUsed: number;
+        truncated: boolean;
+        chunksUsedForContext: number;
+        generationTimeMs: number;
+    };
+}
 export declare class SearchEngine {
     private embedFn;
     private indexManager;
     private db;
     private rerankFn?;
     private contentResolver?;
+    private generateFn?;
     /**
      * Creates a new SearchEngine with explicit dependency injection
      *
@@ -71,7 +108,41 @@ export declare class SearchEngine {
      * const search = new SearchEngine(customEmbedFn, indexManager, db);
      * ```
      */
-    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver);
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver, generateFn?: GenerateFunction);
+    /**
+     * Set or update the generate function
+     * @experimental This method is experimental and may change
+     */
+    setGenerateFunction(generateFn: GenerateFunction | undefined): void;
+    /**
+     * Check if generation is available
+     * @experimental This method is experimental and may change
+     */
+    hasGenerationCapability(): boolean;
+    /**
+     * Perform semantic search with optional AI response generation
+     *
+     * This method extends the standard search with optional response generation.
+     * When generation is enabled, the retrieved chunks are used as context for
+     * an AI model to generate a synthesized response.
+     *
+     * @param query - Search query string
+     * @param options - Extended search options including generation settings
+     * @returns Promise resolving to search results with optional generated response
+     *
+     * @example
+     * ```typescript
+     * // Search with generation
+     * const result = await search.searchWithGeneration('How does auth work?', {
+     *   top_k: 5,
+     *   generateResponse: true
+     * });
+     * console.log(result.generation?.response);
+     * ```
+     *
+     * @experimental This method is experimental and may change in future versions.
+     */
+    searchWithGeneration(query: string, options?: ExtendedSearchOptions): Promise<SearchResultWithGeneration>;
     /**
      * Perform semantic search on the indexed documents
      * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking

package/dist/esm/core/search.js

@@ -5,17 +5,13 @@
 import { getChunksByEmbeddingIds } from './db.js';
 import { config } from './config.js';
 import { createMissingDependencyError } from './actionable-error-messages.js';
-/**
- * Search engine that provides semantic search capabilities
- * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
- * Uses explicit dependency injection for clean architecture
- */
 export class SearchEngine {
     embedFn;
     indexManager;
     db;
     rerankFn;
     contentResolver;
+    generateFn;
     /**
      * Creates a new SearchEngine with explicit dependency injection
      *
@@ -70,7 +66,7 @@ export class SearchEngine {
      * const search = new SearchEngine(customEmbedFn, indexManager, db);
      * ```
      */
-    constructor(embedFn, indexManager, db, rerankFn, contentResolver) {
+    constructor(embedFn, indexManager, db, rerankFn, contentResolver, generateFn) {
         this.embedFn = embedFn;
         this.indexManager = indexManager;
         this.db = db;
@@ -93,6 +89,83 @@ export class SearchEngine {
         }
         // Initialize ContentResolver if provided, or create lazily when needed
         this.contentResolver = contentResolver;
+        // Initialize GenerateFunction if provided (experimental)
+        this.generateFn = generateFn;
+    }
+    /**
+     * Set or update the generate function
+     * @experimental This method is experimental and may change
+     */
+    setGenerateFunction(generateFn) {
+        this.generateFn = generateFn;
+    }
+    /**
+     * Check if generation is available
+     * @experimental This method is experimental and may change
+     */
+    hasGenerationCapability() {
+        return this.generateFn !== undefined;
+    }
+    /**
+     * Perform semantic search with optional AI response generation
+     *
+     * This method extends the standard search with optional response generation.
+     * When generation is enabled, the retrieved chunks are used as context for
+     * an AI model to generate a synthesized response.
+     *
+     * @param query - Search query string
+     * @param options - Extended search options including generation settings
+     * @returns Promise resolving to search results with optional generated response
+     *
+     * @example
+     * ```typescript
+     * // Search with generation
+     * const result = await search.searchWithGeneration('How does auth work?', {
+     *   top_k: 5,
+     *   generateResponse: true
+     * });
+     * console.log(result.generation?.response);
+     * ```
+     *
+     * @experimental This method is experimental and may change in future versions.
+     */
+    async searchWithGeneration(query, options = {}) {
+        // Perform standard search
+        const results = await this.search(query, options);
+        // If generation not requested or no results, return without generation
+        if (!options.generateResponse || results.length === 0) {
+            return { results };
+        }
+        // Check if generation is available
+        if (!this.generateFn) {
+            console.warn('⚠️ [EXPERIMENTAL] Generation requested but no generator configured');
+            return { results };
+        }
+        try {
+            console.log('🤖 [EXPERIMENTAL] Generating response from search results...');
+            const generationResult = await this.generateFn(query, results, {
+                maxTokens: options.generationOptions?.maxTokens,
+                temperature: options.generationOptions?.temperature,
+                systemPrompt: options.generationOptions?.systemPrompt,
+                maxChunksForContext: options.generationOptions?.maxChunksForContext
+            });
+            return {
+                results,
+                generation: {
+                    response: generationResult.response,
+                    modelUsed: generationResult.modelName,
+                    tokensUsed: generationResult.tokensUsed,
+                    truncated: generationResult.truncated,
+                    chunksUsedForContext: generationResult.metadata.chunksIncluded,
+                    generationTimeMs: generationResult.generationTimeMs
+                }
+            };
+        }
+        catch (error) {
+            console.error('❌ [EXPERIMENTAL] Generation failed:', error instanceof Error ? error.message : 'Unknown error');
+            // Return results without generation on error
+            return { results };
+        }
     }
     /**
      * Perform semantic search on the indexed documents
@@ -140,7 +213,7 @@ export class SearchEngine {
             let searchResult;
             try {
                 const contentType = options.contentType;
-                searchResult = this.indexManager.search(queryVector, topK, contentType);
+                searchResult = await this.indexManager.search(queryVector, topK, contentType);
             }
             catch (error) {
                 if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {

package/dist/esm/core/types.d.ts

@@ -63,4 +63,5 @@ export interface Document {
 }
 export type { DatabaseConnection } from './db.js';
 export type { ContentChunk as ChunkResult } from './db.js';
+export type { GenerationRequest, GenerationResult, GenerateFunction, ResponseGenerator, GeneratorModelInfo } from './response-generator.js';
 //# sourceMappingURL=types.d.ts.map

package/dist/esm/core/vector-index-messages.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export interface VectorIndexRequest {
+    id: number;
+    type: 'init' | 'loadIndex' | 'saveIndex' | 'addVector' | 'addVectors' | 'search' | 'getCurrentCount' | 'resizeIndex' | 'reset' | 'cleanup' | 'indexExists' | 'setEf';
+    payload?: any;
+}
+export interface VectorIndexResponse {
+    id: number;
+    type: 'success' | 'error';
+    payload?: any;
+    error?: string;
+}
+export interface InitPayload {
+    dimensions: number;
+    maxElements: number;
+    M?: number;
+    efConstruction?: number;
+    seed?: number;
+    indexPath: string;
+}
+export interface LoadIndexPayload {
+    indexPath: string;
+}
+export interface AddVectorPayload {
+    id: number;
+    vector: ArrayBuffer;
+    dimensions: number;
+}
+export interface AddVectorsPayload {
+    vectors: Array<{
+        id: number;
+        vector: ArrayBuffer;
+        dimensions: number;
+    }>;
+}
+export interface SearchPayload {
+    queryVector: ArrayBuffer;
+    dimensions: number;
+    k: number;
+}
+export interface ResizeIndexPayload {
+    newMaxElements: number;
+}
+export interface SetEfPayload {
+    ef: number;
+}
+export interface IndexExistsPayload {
+    indexPath: string;
+}
+//# sourceMappingURL=vector-index-messages.d.ts.map

package/dist/esm/core/vector-index-messages.js

@@ -0,0 +1,5 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export {};
+//# sourceMappingURL=vector-index-messages.js.map

package/dist/esm/core/vector-index-worker.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Worker thread for VectorIndex operations
+ * Isolates hnswlib-wasm WebAssembly memory to prevent accumulation
+ */
+export {};
+//# sourceMappingURL=vector-index-worker.d.ts.map