rag-lite-ts 2.3.0 → 2.3.1

package/dist/cjs/core/prompt-templates.d.ts

@@ -0,0 +1,138 @@
+/**
+ * CORE MODULE — Prompt Templates for RAG Response Generation
+ *
+ * Provides prompt engineering utilities for different generator model types.
+ * Handles context formatting, token budget management, and system prompts.
+ *
+ * PROMPT STRATEGIES:
+ * - Instruct models: Use chat template with system/user/assistant roles
+ * - Causal LM models: Use simple document + question format
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { SearchResult } from './types.js';
+import type { GeneratorModelType } from './response-generator.js';
+/**
+ * Default system prompt for instruct models
+ * Emphasizes grounded responses using only provided context
+ */
+export declare const DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:\n\n1. Answer ONLY using information found in the context documents\n2. If the answer cannot be found in the context, say \"I cannot find this information in the provided documents\"\n3. Do not make up information or use external knowledge\n4. Be concise and direct in your response\n5. If the context is incomplete or unclear, acknowledge this limitation";
+/**
+ * Default system prompt for RAG with source attribution
+ */
+export declare const DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION = "You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:\n\n1. Answer ONLY using information found in the context documents\n2. When possible, mention which document the information comes from\n3. If the answer cannot be found in the context, say \"I cannot find this information in the provided documents\"\n4. Do not make up information or use external knowledge\n5. Be concise and direct in your response";
+/**
+ * SmolLM2 chat template format
+ * Uses <|im_start|> and <|im_end|> tokens
+ */
+export declare const SMOLLM2_CHAT_TEMPLATE: {
+    systemStart: string;
+    systemEnd: string;
+    userStart: string;
+    userEnd: string;
+    assistantStart: string;
+    assistantEnd: string;
+    endOfText: string;
+};
+/**
+ * Options for formatting context chunks
+ */
+export interface ContextFormattingOptions {
+    /** Maximum tokens available for context */
+    maxContextTokens: number;
+    /** Include document titles/sources */
+    includeDocumentInfo?: boolean;
+    /** Include relevance scores */
+    includeScores?: boolean;
+    /** Separator between chunks */
+    chunkSeparator?: string;
+    /** Token estimation function (chars to tokens ratio) */
+    tokenEstimationRatio?: number;
+}
+/**
+ * Result of context formatting
+ */
+export interface FormattedContext {
+    /** Formatted context string */
+    text: string;
+    /** Estimated token count */
+    estimatedTokens: number;
+    /** Number of chunks included */
+    chunksIncluded: number;
+    /** Total chunks available */
+    totalChunks: number;
+    /** Whether context was truncated */
+    truncated: boolean;
+}
+/**
+ * Format search result chunks into context string for the prompt
+ *
+ * @param chunks - Search result chunks to format
+ * @param options - Formatting options
+ * @returns Formatted context with metadata
+ */
+export declare function formatContextChunks(chunks: SearchResult[], options: ContextFormattingOptions): FormattedContext;
+/**
+ * Options for building the complete prompt
+ */
+export interface PromptBuildOptions {
+    /** User's query */
+    query: string;
+    /** Search result chunks */
+    chunks: SearchResult[];
+    /** Generator model type */
+    modelType: GeneratorModelType;
+    /** Custom system prompt (optional) */
+    systemPrompt?: string;
+    /** Maximum context window tokens */
+    maxContextLength: number;
+    /** Tokens reserved for output */
+    reservedOutputTokens: number;
+    /** Include source attribution hint */
+    includeSourceAttribution?: boolean;
+}
+/**
+ * Result of prompt building
+ */
+export interface BuiltPrompt {
+    /** Complete prompt string */
+    prompt: string;
+    /** Estimated total tokens */
+    estimatedTokens: number;
+    /** Context metadata */
+    contextInfo: FormattedContext;
+    /** System prompt used (if any) */
+    systemPromptUsed?: string;
+}
+/**
+ * Build a complete prompt for the generator model
+ *
+ * @param options - Prompt building options
+ * @returns Built prompt with metadata
+ */
+export declare function buildPrompt(options: PromptBuildOptions): BuiltPrompt;
+/**
+ * Estimate token count for a string
+ * Uses a simple character-based heuristic (~4 chars per token for English)
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export declare function estimateTokenCount(text: string): number;
+/**
+ * Calculate available context budget
+ *
+ * @param maxContextLength - Maximum context window size
+ * @param reservedOutputTokens - Tokens reserved for generation
+ * @param promptOverhead - Tokens used by prompt formatting
+ * @returns Available tokens for context chunks
+ */
+export declare function calculateContextBudget(maxContextLength: number, reservedOutputTokens: number, promptOverhead?: number): number;
+/**
+ * Get default stop sequences for a model type
+ *
+ * @param modelType - Generator model type
+ * @returns Array of stop sequences
+ */
+export declare function getDefaultStopSequences(modelType: GeneratorModelType): string[];
+//# sourceMappingURL=prompt-templates.d.ts.map

package/dist/cjs/core/prompt-templates.js

@@ -0,0 +1,225 @@
+/**
+ * CORE MODULE — Prompt Templates for RAG Response Generation
+ *
+ * Provides prompt engineering utilities for different generator model types.
+ * Handles context formatting, token budget management, and system prompts.
+ *
+ * PROMPT STRATEGIES:
+ * - Instruct models: Use chat template with system/user/assistant roles
+ * - Causal LM models: Use simple document + question format
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+// =============================================================================
+// DEFAULT PROMPTS
+// =============================================================================
+/**
+ * Default system prompt for instruct models
+ * Emphasizes grounded responses using only provided context
+ */
+export const DEFAULT_SYSTEM_PROMPT = `You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:
+
+1. Answer ONLY using information found in the context documents
+2. If the answer cannot be found in the context, say "I cannot find this information in the provided documents"
+3. Do not make up information or use external knowledge
+4. Be concise and direct in your response
+5. If the context is incomplete or unclear, acknowledge this limitation`;
+/**
+ * Default system prompt for RAG with source attribution
+ */
+export const DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION = `You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:
+
+1. Answer ONLY using information found in the context documents
+2. When possible, mention which document the information comes from
+3. If the answer cannot be found in the context, say "I cannot find this information in the provided documents"
+4. Do not make up information or use external knowledge
+5. Be concise and direct in your response`;
+// =============================================================================
+// CHAT TEMPLATES
+// =============================================================================
+/**
+ * SmolLM2 chat template format
+ * Uses <|im_start|> and <|im_end|> tokens
+ */
+export const SMOLLM2_CHAT_TEMPLATE = {
+    systemStart: '<|im_start|>system\n',
+    systemEnd: '<|im_end|>\n',
+    userStart: '<|im_start|>user\n',
+    userEnd: '<|im_end|>\n',
+    assistantStart: '<|im_start|>assistant\n',
+    assistantEnd: '<|im_end|>',
+    endOfText: '<|endoftext|>'
+};
+/**
+ * Format search result chunks into context string for the prompt
+ *
+ * @param chunks - Search result chunks to format
+ * @param options - Formatting options
+ * @returns Formatted context with metadata
+ */
+export function formatContextChunks(chunks, options) {
+    const { maxContextTokens, includeDocumentInfo = true, includeScores = false, chunkSeparator = '\n---\n', tokenEstimationRatio = 4 // ~4 chars per token for English
+     } = options;
+    const maxChars = maxContextTokens * tokenEstimationRatio;
+    let currentChars = 0;
+    const includedChunks = [];
+    let truncated = false;
+    for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        // Format this chunk
+        let chunkText = '';
+        if (includeDocumentInfo) {
+            chunkText += `[Document ${i + 1}: ${chunk.document.title}]`;
+            if (includeScores) {
+                chunkText += ` (relevance: ${(chunk.score * 100).toFixed(1)}%)`;
+            }
+            chunkText += '\n';
+        }
+        chunkText += chunk.content;
+        // Check if adding this chunk would exceed budget
+        const chunkChars = chunkText.length + (includedChunks.length > 0 ? chunkSeparator.length : 0);
+        if (currentChars + chunkChars > maxChars) {
+            // Check if we can fit a truncated version of this chunk
+            const remainingChars = maxChars - currentChars - (includedChunks.length > 0 ? chunkSeparator.length : 0);
+            if (remainingChars > 100 && includedChunks.length === 0) {
+                // Truncate the first chunk if it's the only option
+                chunkText = chunkText.substring(0, remainingChars - 20) + '\n[Content truncated...]';
+                includedChunks.push(chunkText);
+                currentChars += chunkText.length;
+            }
+            truncated = true;
+            break;
+        }
+        includedChunks.push(chunkText);
+        currentChars += chunkChars;
+    }
+    const text = includedChunks.join(chunkSeparator);
+    const estimatedTokens = Math.ceil(text.length / tokenEstimationRatio);
+    return {
+        text,
+        estimatedTokens,
+        chunksIncluded: includedChunks.length,
+        totalChunks: chunks.length,
+        truncated
+    };
+}
+/**
+ * Build a complete prompt for the generator model
+ *
+ * @param options - Prompt building options
+ * @returns Built prompt with metadata
+ */
+export function buildPrompt(options) {
+    const { query, chunks, modelType, systemPrompt, maxContextLength, reservedOutputTokens, includeSourceAttribution = false } = options;
+    // Calculate available tokens for context
+    const promptOverhead = modelType === 'instruct' ? 150 : 50; // Tokens for formatting
+    const queryTokens = Math.ceil(query.length / 4);
+    const availableContextTokens = maxContextLength - reservedOutputTokens - promptOverhead - queryTokens;
+    // Format context chunks
+    const contextInfo = formatContextChunks(chunks, {
+        maxContextTokens: availableContextTokens,
+        includeDocumentInfo: true,
+        includeScores: false
+    });
+    // Build prompt based on model type
+    let prompt;
+    let systemPromptUsed;
+    if (modelType === 'instruct') {
+        prompt = buildInstructPrompt(query, contextInfo.text, systemPrompt, includeSourceAttribution);
+        systemPromptUsed = systemPrompt || (includeSourceAttribution ? DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION : DEFAULT_SYSTEM_PROMPT);
+    }
+    else {
+        prompt = buildCausalLMPrompt(query, contextInfo.text);
+    }
+    const estimatedTokens = Math.ceil(prompt.length / 4);
+    return {
+        prompt,
+        estimatedTokens,
+        contextInfo,
+        systemPromptUsed
+    };
+}
+/**
+ * Build prompt for instruct models (SmolLM2-Instruct)
+ * Uses chat template format with system/user/assistant roles
+ */
+function buildInstructPrompt(query, context, customSystemPrompt, includeSourceAttribution = false) {
+    const systemPrompt = customSystemPrompt ||
+        (includeSourceAttribution ? DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION : DEFAULT_SYSTEM_PROMPT);
+    const template = SMOLLM2_CHAT_TEMPLATE;
+    const userMessage = `Context:
+${context}
+
+Question: ${query}
+
+Answer based only on the context above:`;
+    return `${template.systemStart}${systemPrompt}${template.systemEnd}${template.userStart}${userMessage}${template.userEnd}${template.assistantStart}`;
+}
+/**
+ * Build prompt for causal LM models (DistilGPT2)
+ * Uses simple document + question format without roles
+ */
+function buildCausalLMPrompt(query, context) {
+    return `The following documents contain information to answer the question.
+
+Documents:
+${context}
+
+Based on the documents above, answer this question: ${query}
+
+Answer:`;
+}
+// =============================================================================
+// TOKEN ESTIMATION
+// =============================================================================
+/**
+ * Estimate token count for a string
+ * Uses a simple character-based heuristic (~4 chars per token for English)
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export function estimateTokenCount(text) {
+    // Simple heuristic: ~4 characters per token for English text
+    // This is a rough approximation; actual tokenization varies by model
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Calculate available context budget
+ *
+ * @param maxContextLength - Maximum context window size
+ * @param reservedOutputTokens - Tokens reserved for generation
+ * @param promptOverhead - Tokens used by prompt formatting
+ * @returns Available tokens for context chunks
+ */
+export function calculateContextBudget(maxContextLength, reservedOutputTokens, promptOverhead = 100) {
+    return Math.max(0, maxContextLength - reservedOutputTokens - promptOverhead);
+}
+// =============================================================================
+// STOP SEQUENCES
+// =============================================================================
+/**
+ * Get default stop sequences for a model type
+ *
+ * @param modelType - Generator model type
+ * @returns Array of stop sequences
+ */
+export function getDefaultStopSequences(modelType) {
+    if (modelType === 'instruct') {
+        return [
+            SMOLLM2_CHAT_TEMPLATE.assistantEnd,
+            SMOLLM2_CHAT_TEMPLATE.endOfText,
+            '<|im_start|>',
+            '\n\nQuestion:',
+            '\n\nContext:'
+        ];
+    }
+    // Causal LM stop sequences
+    return [
+        '\n\nQuestion:',
+        '\n\nDocuments:',
+        '\n\n---',
+        '<|endoftext|>'
+    ];
+}
+//# sourceMappingURL=prompt-templates.js.map

package/dist/cjs/core/response-generator.d.ts

@@ -0,0 +1,132 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { SearchResult } from './types.js';
+/** Supported generator model types */
+export type GeneratorModelType = 'causal-lm' | 'instruct';
+/** Generation request containing query and retrieved context */
+export interface GenerationRequest {
+    query: string;
+    chunks: SearchResult[];
+    systemPrompt?: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    topK?: number;
+    repetitionPenalty?: number;
+    stopSequences?: string[];
+    includeSourceAttribution?: boolean;
+    /** Maximum number of chunks to include in context (overrides model default) */
+    maxChunksForContext?: number;
+}
+/** Result of text generation */
+export interface GenerationResult {
+    response: string;
+    tokensUsed: number;
+    truncated: boolean;
+    modelName: string;
+    generationTimeMs: number;
+    metadata: {
+        promptTokens: number;
+        completionTokens: number;
+        chunksIncluded: number;
+        totalChunks: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    };
+}
+/** Generator model capabilities */
+export interface GeneratorCapabilities {
+    supportsStreaming: boolean;
+    supportsSystemPrompt: boolean;
+    instructionTuned: boolean;
+    maxContextLength: number;
+    defaultMaxOutputTokens: number;
+    recommendedTemperature: number;
+    /** Maximum number of chunks to use for context (default varies by model) */
+    defaultMaxChunksForContext: number;
+}
+/** Generator model requirements */
+export interface GeneratorRequirements {
+    transformersJsVersion: string;
+    minimumMemory: number;
+    requiredFeatures: readonly string[];
+    platformSupport: readonly string[];
+}
+/** Complete generator model information */
+export interface GeneratorModelInfo {
+    name: string;
+    type: GeneratorModelType;
+    version: string;
+    capabilities: GeneratorCapabilities;
+    requirements: GeneratorRequirements;
+    isDefault?: boolean;
+    description?: string;
+}
+/** Generator validation result */
+export interface GeneratorValidationResult {
+    isValid: boolean;
+    errors: string[];
+    warnings: string[];
+    suggestions: string[];
+}
+/** Options for creating generator instances */
+export interface GeneratorCreationOptions {
+    cachePath?: string;
+    timeout?: number;
+    enableGPU?: boolean;
+    defaultGenerationOptions?: Partial<GenerationRequest>;
+    customConfig?: Record<string, any>;
+}
+/**
+ * Universal response generator interface
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export interface ResponseGenerator {
+    readonly modelName: string;
+    readonly modelType: GeneratorModelType;
+    readonly maxContextLength: number;
+    readonly maxOutputLength: number;
+    generate(request: GenerationRequest): Promise<GenerationResult>;
+    generateStream?(request: GenerationRequest): AsyncIterable<string>;
+    loadModel(): Promise<void>;
+    isLoaded(): boolean;
+    getModelInfo(): GeneratorModelInfo;
+    cleanup(): Promise<void>;
+}
+export type GenerateFunction = (query: string, chunks: SearchResult[], options?: Partial<GenerationRequest>) => Promise<GenerationResult>;
+export type CreateGeneratorFunction = (modelName: string, options?: GeneratorCreationOptions) => Promise<ResponseGenerator>;
+export declare class GeneratorValidationError extends Error {
+    readonly modelName: string;
+    readonly availableModels: readonly string[];
+    constructor(modelName: string, availableModels: readonly string[], message: string);
+}
+export declare class GenerationError extends Error {
+    readonly modelName: string;
+    readonly stage: 'loading' | 'tokenization' | 'generation' | 'decoding';
+    readonly cause?: Error | undefined;
+    constructor(modelName: string, stage: 'loading' | 'tokenization' | 'generation' | 'decoding', message: string, cause?: Error | undefined);
+}
+export declare class ContextWindowError extends Error {
+    readonly requiredTokens: number;
+    readonly availableTokens: number;
+    constructor(requiredTokens: number, availableTokens: number, message: string);
+}
+export declare function supportsStreaming(generator: ResponseGenerator): generator is ResponseGenerator & {
+    generateStream(request: GenerationRequest): AsyncIterable<string>;
+};
+export declare function isInstructModel(generator: ResponseGenerator): boolean;
+export declare function createGenerateFunction(generator: ResponseGenerator): GenerateFunction;
+//# sourceMappingURL=response-generator.d.ts.map

package/dist/cjs/core/response-generator.js

@@ -0,0 +1,69 @@
+/**
+ * CORE MODULE — Response Generator Interface for RAG Response Generation
+ *
+ * Model-agnostic interfaces supporting text generation from retrieved context.
+ * Designed for runtime polymorphism and extensibility, following the same
+ * patterns established by the UniversalEmbedder interface.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, recommended, 3 chunks default)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, 5 chunks default)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+// =============================================================================
+// ERROR CLASSES
+// =============================================================================
+export class GeneratorValidationError extends Error {
+    modelName;
+    availableModels;
+    constructor(modelName, availableModels, message) {
+        super(message);
+        this.modelName = modelName;
+        this.availableModels = availableModels;
+        this.name = 'GeneratorValidationError';
+    }
+}
+export class GenerationError extends Error {
+    modelName;
+    stage;
+    cause;
+    constructor(modelName, stage, message, cause) {
+        super(message);
+        this.modelName = modelName;
+        this.stage = stage;
+        this.cause = cause;
+        this.name = 'GenerationError';
+    }
+}
+export class ContextWindowError extends Error {
+    requiredTokens;
+    availableTokens;
+    constructor(requiredTokens, availableTokens, message) {
+        super(message);
+        this.requiredTokens = requiredTokens;
+        this.availableTokens = availableTokens;
+        this.name = 'ContextWindowError';
+    }
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+export function supportsStreaming(generator) {
+    return typeof generator.generateStream === 'function';
+}
+export function isInstructModel(generator) {
+    return generator.modelType === 'instruct';
+}
+export function createGenerateFunction(generator) {
+    return async (query, chunks, options) => {
+        if (!generator.isLoaded()) {
+            await generator.loadModel();
+        }
+        return generator.generate({ query, chunks, ...options });
+    };
+}
+//# sourceMappingURL=response-generator.js.map

package/dist/cjs/core/search.d.ts

@@ -6,17 +6,54 @@ import { IndexManager } from '../index-manager.js';
 import { DatabaseConnection } from './db.js';
 import type { SearchResult, SearchOptions } from './types.js';
 import type { EmbedFunction, RerankFunction } from './interfaces.js';
+import type { GenerateFunction } from './response-generator.js';
 /**
  * Search engine that provides semantic search capabilities
  * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking
  * Uses explicit dependency injection for clean architecture
  */
+/**
+ * Extended search options with generation support
+ * @experimental Generation features are experimental
+ */
+export interface ExtendedSearchOptions extends SearchOptions {
+    /** Enable AI response generation from search results */
+    generateResponse?: boolean;
+    /** Generator model to use (default: SmolLM2-135M-Instruct) */
+    generatorModel?: string;
+    /** Generation options */
+    generationOptions?: {
+        maxTokens?: number;
+        temperature?: number;
+        systemPrompt?: string;
+        /** Maximum chunks to use for context (overrides model default) */
+        maxChunksForContext?: number;
+    };
+}
+/**
+ * Search result with optional generated response
+ * @experimental Generation features are experimental
+ */
+export interface SearchResultWithGeneration {
+    /** Search results (chunks) */
+    results: SearchResult[];
+    /** Generated response (if generation was enabled) */
+    generation?: {
+        response: string;
+        modelUsed: string;
+        tokensUsed: number;
+        truncated: boolean;
+        chunksUsedForContext: number;
+        generationTimeMs: number;
+    };
+}
 export declare class SearchEngine {
     private embedFn;
     private indexManager;
     private db;
     private rerankFn?;
     private contentResolver?;
+    private generateFn?;
     /**
      * Creates a new SearchEngine with explicit dependency injection
      *
@@ -71,7 +108,41 @@ export declare class SearchEngine {
      * const search = new SearchEngine(customEmbedFn, indexManager, db);
      * ```
      */
-    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver);
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, rerankFn?: RerankFunction | undefined, contentResolver?: import('./content-resolver.js').ContentResolver, generateFn?: GenerateFunction);
+    /**
+     * Set or update the generate function
+     * @experimental This method is experimental and may change
+     */
+    setGenerateFunction(generateFn: GenerateFunction | undefined): void;
+    /**
+     * Check if generation is available
+     * @experimental This method is experimental and may change
+     */
+    hasGenerationCapability(): boolean;
+    /**
+     * Perform semantic search with optional AI response generation
+     *
+     * This method extends the standard search with optional response generation.
+     * When generation is enabled, the retrieved chunks are used as context for
+     * an AI model to generate a synthesized response.
+     *
+     * @param query - Search query string
+     * @param options - Extended search options including generation settings
+     * @returns Promise resolving to search results with optional generated response
+     *
+     * @example
+     * ```typescript
+     * // Search with generation
+     * const result = await search.searchWithGeneration('How does auth work?', {
+     *   top_k: 5,
+     *   generateResponse: true
+     * });
+     * console.log(result.generation?.response);
+     * ```
+     *
+     * @experimental This method is experimental and may change in future versions.
+     */
+    searchWithGeneration(query: string, options?: ExtendedSearchOptions): Promise<SearchResultWithGeneration>;
     /**
      * Perform semantic search on the indexed documents
      * Implements the core search pipeline: query embedding → vector search → metadata retrieval → optional reranking