rag-lite-ts 2.1.0 → 2.2.0

package/dist/esm/search-standalone.js

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+/**
+ * Standalone search script for direct node execution
+ * Usage: node search.js <query> [--top-k <number>] [--rerank|--no-rerank]
+ */
+import { runSearch } from './cli/search.js';
+import { EXIT_CODES, ConfigurationError } from './core/config.js';
+function parseArgs() {
+    const args = process.argv.slice(2);
+    if (args.length === 0) {
+        console.error('RAG-lite TS Document Search');
+        console.error('');
+        console.error('Usage: node search.js <query> [options]');
+        console.error('');
+        console.error('Arguments:');
+        console.error('  <query>    Search query (wrap in quotes if it contains spaces)');
+        console.error('');
+        console.error('Options:');
+        console.error('  --top-k <number>    Number of results to return (default: 10, max: 100)');
+        console.error('  --rerank           Enable reranking for better results');
+        console.error('  --no-rerank        Disable reranking');
+        console.error('');
+        console.error('Examples:');
+        console.error('  node search.js "machine learning"');
+        console.error('  node search.js "API documentation" --top-k 10');
+        console.error('  node search.js "tutorial" --rerank');
+        console.error('  node search.js "how to install" --top-k 20 --rerank');
+        console.error('');
+        console.error('Note: Make sure you have ingested documents first using:');
+        console.error('  node indexer.js <path>');
+        process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+    }
+    const queryParts = [];
+    const options = {};
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === '--top-k') {
+            const nextArg = args[i + 1];
+            if (!nextArg) {
+                console.error('Error: --top-k requires a numeric value');
+                console.error('');
+                console.error('Examples:');
+                console.error('  --top-k 5     # Return 5 results');
+                console.error('  --top-k 20    # Return 20 results');
+                process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+            }
+            const topK = parseInt(nextArg, 10);
+            if (isNaN(topK) || topK <= 0) {
+                console.error('Error: --top-k must be a positive number');
+                console.error('');
+                console.error('Examples:');
+                console.error('  --top-k 5     # Return 5 results');
+                console.error('  --top-k 20    # Return 20 results');
+                process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+            }
+            if (topK > 100) {
+                console.warn(`Warning: Large --top-k value (${topK}) may impact performance. Consider using a smaller value.`);
+            }
+            options['top-k'] = topK;
+            i++; // Skip next argument
+        }
+        else if (arg === '--rerank') {
+            options.rerank = true;
+        }
+        else if (arg === '--no-rerank') {
+            options.rerank = false;
+        }
+        else if (arg === '--help' || arg === '-h') {
+            // Re-show help and exit
+            parseArgs();
+        }
+        else if (arg.startsWith('--')) {
+            console.error(`Error: Unknown option '${arg}'`);
+            console.error('');
+            console.error('Available options:');
+            console.error('  --top-k <number>    Number of results to return');
+            console.error('  --rerank           Enable reranking');
+            console.error('  --no-rerank        Disable reranking');
+            console.error('  --help, -h         Show this help');
+            process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+        }
+        else {
+            queryParts.push(arg);
+        }
+    }
+    const query = queryParts.join(' ');
+    if (!query.trim()) {
+        console.error('Error: Query cannot be empty');
+        console.error('');
+        console.error('Please provide a search query:');
+        console.error('  node search.js "your search terms"');
+        process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+    }
+    if (query.trim().length > 500) {
+        console.error('Error: Query is too long (maximum 500 characters)');
+        console.error('');
+        console.error('Please use a shorter, more specific query.');
+        process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+    }
+    return { query, options };
+}
+async function main() {
+    const { query, options } = parseArgs();
+    await runSearch(query, options);
+}
+main().catch((error) => {
+    if (error instanceof ConfigurationError) {
+        console.error('Configuration Error:');
+        console.error(error.message);
+        process.exit(error.exitCode);
+    }
+    else {
+        console.error('Fatal Error:', error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.GENERAL_ERROR);
+    }
+});
+//# sourceMappingURL=search-standalone.js.map

package/dist/esm/search.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Public API SearchEngine - Simple constructor with Chameleon Architecture
+ *
+ * This class provides a clean, simple API that automatically adapts to the mode
+ * (text or multimodal) stored in the database during ingestion. The system detects
+ * the mode and creates the appropriate embedder and reranker without user intervention.
+ *
+ * Chameleon Architecture Features:
+ * - Automatic mode detection from database configuration
+ * - Seamless switching between text and multimodal modes
+ * - Appropriate embedder selection (sentence-transformer or CLIP)
+ * - Mode-specific reranking strategies
+ *
+ * @example
+ * ```typescript
+ * // Simple usage - mode automatically detected from database
+ * const search = new SearchEngine('./index.bin', './db.sqlite');
+ * const results = await search.search('query');
+ *
+ * // Works for both text and multimodal databases
+ * // Text mode: uses sentence-transformer embeddings
+ * // Multimodal mode: uses CLIP embeddings for cross-modal search
+ *
+ * // With options (advanced)
+ * const search = new SearchEngine('./index.bin', './db.sqlite', {
+ *   enableReranking: true
+ * });
+ * ```
+ */
+import type { SearchResult, SearchOptions, EmbedFunction, RerankFunction } from './core/types.js';
+export interface SearchEngineOptions {
+    /** Embedding model name override */
+    embeddingModel?: string;
+    /** Embedding batch size override */
+    batchSize?: number;
+    /** Reranking model name override */
+    rerankingModel?: string;
+    /** Whether to enable reranking (default: true) */
+    enableReranking?: boolean;
+    /** Top-k results to return (default: from config) */
+    topK?: number;
+    /** Custom embedding function (advanced usage) */
+    embedFn?: EmbedFunction;
+    /** Custom reranking function (advanced usage) */
+    rerankFn?: RerankFunction;
+}
+export declare class SearchEngine {
+    private indexPath;
+    private dbPath;
+    private options;
+    private coreEngine;
+    private initPromise;
+    constructor(indexPath: string, dbPath: string, options?: SearchEngineOptions);
+    /**
+     * Initialize the search engine using polymorphic factory or direct injection
+     *
+     * Chameleon Architecture Implementation:
+     * - Automatically detects mode from database (text or multimodal)
+     * - Creates appropriate embedder based on detected mode
+     * - Applies mode-specific reranking strategies
+     * - Provides seamless polymorphic behavior
+     */
+    private initialize;
+    /**
+     * Perform semantic search
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    getContent(contentId: string, format?: 'file' | 'base64'): Promise<string>;
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    getContentBatch(contentIds: string[], format?: 'file' | 'base64'): Promise<string[]>;
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    getContentMetadata(contentId: string): Promise<import('./core/content-resolver.js').ContentMetadata>;
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    verifyContentExists(contentId: string): Promise<boolean>;
+    /**
+     * Clean up resources
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=search.d.ts.map

package/dist/esm/search.js

@@ -0,0 +1,177 @@
+/**
+ * Public API SearchEngine - Simple constructor with Chameleon Architecture
+ *
+ * This class provides a clean, simple API that automatically adapts to the mode
+ * (text or multimodal) stored in the database during ingestion. The system detects
+ * the mode and creates the appropriate embedder and reranker without user intervention.
+ *
+ * Chameleon Architecture Features:
+ * - Automatic mode detection from database configuration
+ * - Seamless switching between text and multimodal modes
+ * - Appropriate embedder selection (sentence-transformer or CLIP)
+ * - Mode-specific reranking strategies
+ *
+ * @example
+ * ```typescript
+ * // Simple usage - mode automatically detected from database
+ * const search = new SearchEngine('./index.bin', './db.sqlite');
+ * const results = await search.search('query');
+ *
+ * // Works for both text and multimodal databases
+ * // Text mode: uses sentence-transformer embeddings
+ * // Multimodal mode: uses CLIP embeddings for cross-modal search
+ *
+ * // With options (advanced)
+ * const search = new SearchEngine('./index.bin', './db.sqlite', {
+ *   enableReranking: true
+ * });
+ * ```
+ */
+import { SearchEngine as CoreSearchEngine } from './core/search.js';
+export class SearchEngine {
+    indexPath;
+    dbPath;
+    options;
+    coreEngine = null;
+    initPromise = null;
+    constructor(indexPath, dbPath, options = {}) {
+        this.indexPath = indexPath;
+        this.dbPath = dbPath;
+        this.options = options;
+        // Validate required parameters
+        if (!indexPath || typeof indexPath !== 'string' || indexPath.trim() === '') {
+            throw new Error('Both indexPath and dbPath are required.\n' +
+                'Example: const search = new SearchEngine("./index.bin", "./db.sqlite");\n' +
+                'Or use: const search = await SearchFactory.create("./index.bin", "./db.sqlite");');
+        }
+        if (!dbPath || typeof dbPath !== 'string' || dbPath.trim() === '') {
+            throw new Error('Both indexPath and dbPath are required.\n' +
+                'Example: const search = new SearchEngine("./index.bin", "./db.sqlite");\n' +
+                'Or use: const search = await SearchFactory.create("./index.bin", "./db.sqlite");');
+        }
+    }
+    /**
+     * Initialize the search engine using polymorphic factory or direct injection
+     *
+     * Chameleon Architecture Implementation:
+     * - Automatically detects mode from database (text or multimodal)
+     * - Creates appropriate embedder based on detected mode
+     * - Applies mode-specific reranking strategies
+     * - Provides seamless polymorphic behavior
+     */
+    async initialize() {
+        if (this.coreEngine) {
+            return; // Already initialized
+        }
+        if (this.initPromise) {
+            return this.initPromise; // Initialization in progress
+        }
+        this.initPromise = (async () => {
+            // If custom functions are provided, use direct dependency injection
+            if (this.options.embedFn || this.options.rerankFn) {
+                const { IndexManager } = await import('./index-manager.js');
+                const { openDatabase } = await import('./core/db.js');
+                const { createTextEmbedFunction } = await import('./text/embedder.js');
+                const { existsSync } = await import('fs');
+                // Validate files exist
+                if (!existsSync(this.indexPath)) {
+                    throw new Error(`Vector index not found at: ${this.indexPath}`);
+                }
+                if (!existsSync(this.dbPath)) {
+                    throw new Error(`Database not found at: ${this.dbPath}`);
+                }
+                // Use custom embedFn or create default
+                const embedFn = this.options.embedFn || createTextEmbedFunction(this.options.embeddingModel);
+                // Get model defaults for dimensions
+                const { getModelDefaults, config } = await import('./core/config.js');
+                const modelDefaults = getModelDefaults(this.options.embeddingModel || config.embedding_model);
+                // Initialize dependencies
+                const db = await openDatabase(this.dbPath);
+                const indexManager = new IndexManager(this.indexPath, this.dbPath, modelDefaults.dimensions, this.options.embeddingModel);
+                await indexManager.initialize();
+                // Create ContentResolver for unified content system
+                const { ContentResolver } = await import('./core/content-resolver.js');
+                const contentResolver = new ContentResolver(db);
+                // Create core engine with dependency injection
+                this.coreEngine = new CoreSearchEngine(embedFn, indexManager, db, this.options.rerankFn, contentResolver);
+            }
+            else {
+                // Use core polymorphic factory for automatic mode detection (Chameleon Architecture)
+                // This enables SearchEngine to automatically adapt to text or multimodal mode
+                // based on the configuration stored in the database during ingestion
+                const { SearchFactory } = await import('./factories/search-factory.js');
+                this.coreEngine = await SearchFactory.create(this.indexPath, this.dbPath);
+            }
+        })();
+        return this.initPromise;
+    }
+    /**
+     * Perform semantic search
+     */
+    async search(query, options) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.search(query, options);
+    }
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    async getContent(contentId, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContent(contentId, format);
+    }
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    async getContentBatch(contentIds, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentBatch(contentIds, format);
+    }
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    async getContentMetadata(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentMetadata(contentId);
+    }
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    async verifyContentExists(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.verifyContentExists(contentId);
+    }
+    /**
+     * Clean up resources
+     */
+    async cleanup() {
+        if (this.coreEngine) {
+            await this.coreEngine.cleanup();
+        }
+    }
+}
+//# sourceMappingURL=search.js.map

package/dist/esm/test-utils.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Test utilities for multi-model support
+ * Provides common configurations and helpers for testing with different embedding models
+ */
+export interface TestModel {
+    name: string;
+    dimensions: number;
+    chunkSize: number;
+    batchSize: number;
+}
+export declare const TEST_MODELS: TestModel[];
+/**
+ * Retrieve model configuration by name
+ * @param modelName - The name of the model to retrieve
+ * @returns Model configuration object or undefined if not found
+ */
+export declare function getTestModel(modelName: string): TestModel | undefined;
+//# sourceMappingURL=test-utils.d.ts.map

package/dist/esm/test-utils.js

@@ -0,0 +1,27 @@
+/**
+ * Test utilities for multi-model support
+ * Provides common configurations and helpers for testing with different embedding models
+ */
+export const TEST_MODELS = [
+    {
+        name: 'sentence-transformers/all-MiniLM-L6-v2',
+        dimensions: 384,
+        chunkSize: 250,
+        batchSize: 16
+    },
+    {
+        name: 'Xenova/all-mpnet-base-v2',
+        dimensions: 768,
+        chunkSize: 400,
+        batchSize: 8
+    }
+];
+/**
+ * Retrieve model configuration by name
+ * @param modelName - The name of the model to retrieve
+ * @returns Model configuration object or undefined if not found
+ */
+export function getTestModel(modelName) {
+    return TEST_MODELS.find(m => m.name === modelName);
+}
+//# sourceMappingURL=test-utils.js.map

package/dist/esm/text/chunker.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Text-specific chunking implementation
+ * Implements the ChunkingStrategy interface for text content
+ */
+import '../dom-polyfills.js';
+import { ChunkingStrategy, GenericDocument, GenericChunk, ChunkConfig } from '../core/chunker.js';
+/**
+ * Document interface for text chunking
+ */
+export interface Document {
+    source: string;
+    title: string;
+    content: string;
+    metadata?: Record<string, any>;
+}
+export interface Chunk {
+    text: string;
+    chunkIndex: number;
+    tokenCount: number;
+}
+/**
+ * Text chunking strategy implementation
+ */
+export declare class TextChunkingStrategy implements ChunkingStrategy {
+    appliesTo(contentType: string): boolean;
+    chunk(document: GenericDocument, config: ChunkConfig): Promise<GenericChunk[]>;
+}
+/**
+ * Text document chunking function
+ * Converts between text-specific and generic interfaces
+ */
+export declare function chunkDocument(document: Document, config?: ChunkConfig): Promise<Chunk[]>;
+//# sourceMappingURL=chunker.d.ts.map