rag-lite-ts 1.0.1

package/dist/reranker.d.ts

@@ -0,0 +1,40 @@
+import type { SearchResult } from './types.js';
+/**
+ * Cross-encoder reranker for improving search result quality
+ * Uses a cross-encoder model to rerank initial vector search results
+ */
+export declare class CrossEncoderReranker {
+    private model;
+    private tokenizer;
+    private modelName;
+    private static readonly FALLBACK_MODELS;
+    /**
+     * Load the cross-encoder model with graceful fallback
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Try to load a specific model
+     */
+    private tryLoadModel;
+    /**
+     * Rerank search results using cross-encoder scoring
+     * @param query - Original search query
+     * @param results - Initial search results from vector search
+     * @returns Promise resolving to reranked results
+     */
+    rerank(query: string, results: SearchResult[]): Promise<SearchResult[]>;
+    /**
+     * Check if the model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Get the model name being used
+     */
+    getModelName(): string;
+    /**
+     * Simple text-based reranking using keyword matching and text similarity
+     * This is a fallback when cross-encoder models don't work well for the content
+     */
+    private simpleTextReranking;
+}
+//# sourceMappingURL=reranker.d.ts.map

package/dist/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG/C;;;GAGG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,SAAS,CAA2C;IAE5D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAKrC;IAEF;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBhC;;OAEG;YACW,YAAY;IAsB1B;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqE7E;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;CAqF5B"}

package/dist/reranker.js

@@ -0,0 +1,212 @@
+import { AutoTokenizer, AutoModelForSequenceClassification } from '@huggingface/transformers';
+import { config } from './config.js';
+/**
+ * Cross-encoder reranker for improving search result quality
+ * Uses a cross-encoder model to rerank initial vector search results
+ */
+export class CrossEncoderReranker {
+    model = null; // Use any to avoid complex transformers.js typing issues
+    tokenizer = null;
+    modelName = 'Xenova/ms-marco-MiniLM-L-6-v2'; // Use working model as default
+    // Alternative models in case the primary fails
+    static FALLBACK_MODELS = [
+        'Xenova/ms-marco-MiniLM-L-6-v2', // Primary - optimized for transformers.js
+        'cross-encoder/ms-marco-MiniLM-L-6-v2', // Original (likely to fail)
+        'cross-encoder/ms-marco-MiniLM-L-2-v2', // Smaller original (likely to fail)
+        // Note: sentence-transformers/all-MiniLM-L6-v2 is a bi-encoder, not cross-encoder, so removed
+    ];
+    /**
+     * Load the cross-encoder model with graceful fallback
+     */
+    async loadModel() {
+        // Try primary model first (should work since it's Xenova)
+        if (await this.tryLoadModel(this.modelName)) {
+            return;
+        }
+        // Try fallback models if primary fails
+        console.warn(`Primary model ${this.modelName} failed, trying fallbacks...`);
+        for (const fallbackModel of CrossEncoderReranker.FALLBACK_MODELS) {
+            if (fallbackModel === this.modelName)
+                continue; // Skip already tried model
+            console.warn(`Trying fallback model: ${fallbackModel}`);
+            if (await this.tryLoadModel(fallbackModel)) {
+                this.modelName = fallbackModel;
+                return;
+            }
+        }
+        console.warn('All cross-encoder models failed to load. Reranking will be disabled.');
+        this.model = null;
+        this.tokenizer = null;
+    }
+    /**
+     * Try to load a specific model
+     */
+    async tryLoadModel(modelName) {
+        try {
+            console.log(`Loading cross-encoder model: ${modelName}`);
+            // Load model and tokenizer separately for proper cross-encoder usage
+            this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
+                cache_dir: config.model_cache_path,
+                dtype: 'fp32' // Suppress dtype warnings
+            });
+            this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
+                cache_dir: config.model_cache_path
+            });
+            console.log(`Cross-encoder model loaded successfully: ${modelName}`);
+            return true;
+        }
+        catch (error) {
+            console.warn(`Failed to load model ${modelName}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            return false;
+        }
+    }
+    /**
+     * Rerank search results using cross-encoder scoring
+     * @param query - Original search query
+     * @param results - Initial search results from vector search
+     * @returns Promise resolving to reranked results
+     */
+    async rerank(query, results) {
+        if (!this.model || !this.tokenizer) {
+            throw new Error('Cross-encoder model not loaded. Call loadModel() first.');
+        }
+        if (results.length === 0) {
+            return results;
+        }
+        try {
+            // Prepare queries and documents for proper cross-encoder format
+            const queries = results.map(() => query);
+            const documents = results.map(result => result.text);
+            // Tokenize using the proper cross-encoder format with text_pair
+            const features = this.tokenizer(queries, {
+                text_pair: documents,
+                padding: true,
+                truncation: true,
+                return_tensors: 'pt'
+            });
+            // Get model predictions
+            const output = await this.model(features);
+            // Extract logits - these are the raw relevance scores
+            const logits = output.logits;
+            // Convert logits to scores and pair with results
+            const scores = results.map((result, i) => {
+                // For cross-encoders, we typically use the raw logit as the relevance score
+                // Higher logits = more relevant
+                const rawScore = Array.isArray(logits) ? logits[i] : logits.data[i];
+                // Apply sigmoid to convert logit to probability-like score
+                const score = 1 / (1 + Math.exp(-rawScore));
+                return {
+                    score: score,
+                    result: result
+                };
+            });
+            // Check if scores look reasonable
+            const maxScore = Math.max(...scores.map(s => s.score));
+            const minScore = Math.min(...scores.map(s => s.score));
+            const scoreRange = maxScore - minScore;
+            // If there's very little variation in scores, fall back to simple reranking
+            if (scoreRange < 0.1) {
+                console.log(`Cross-encoder scores have low variation (range: ${scoreRange.toFixed(3)}), using simple text reranking`);
+                return this.simpleTextReranking(query, results);
+            }
+            // Sort by cross-encoder scores (descending)
+            scores.sort((a, b) => b.score - a.score);
+            // Update results with new scores and return reranked results
+            return scores.map(item => ({
+                ...item.result,
+                score: item.score
+            }));
+        }
+        catch (error) {
+            console.warn(`Reranking failed, falling back to simple text reranking: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            return this.simpleTextReranking(query, results);
+        }
+    }
+    /**
+     * Check if the model is loaded
+     */
+    isLoaded() {
+        return this.model !== null && this.tokenizer !== null;
+    }
+    /**
+     * Get the model name being used
+     */
+    getModelName() {
+        return this.modelName;
+    }
+    /**
+     * Simple text-based reranking using keyword matching and text similarity
+     * This is a fallback when cross-encoder models don't work well for the content
+     */
+    simpleTextReranking(query, results) {
+        const queryLower = query.toLowerCase();
+        const queryWords = queryLower.split(/\s+/).filter(word => word.length > 2);
+        const rerankedResults = results.map(result => {
+            const textLower = result.text.toLowerCase();
+            const titleLower = result.document.title?.toLowerCase() || '';
+            let score = result.score; // Start with vector search score
+            let bonus = 0;
+            // Bonus for exact query matches
+            if (textLower.includes(queryLower)) {
+                bonus += 0.15;
+            }
+            // Bonus for title matches
+            if (titleLower.includes(queryLower)) {
+                bonus += 0.1;
+            }
+            // Special handling for "What is X?" queries
+            if (queryLower.startsWith('what is')) {
+                const subject = queryLower.replace('what is', '').trim().replace(/\?$/, '');
+                // Major bonus for definitional patterns like "X is a/an..."
+                const definitionPatterns = [
+                    `${subject} is a`,
+                    `${subject} is an`,
+                    `${subject} is the`,
+                    `${subject} provides`,
+                    `${subject} helps`
+                ];
+                for (const pattern of definitionPatterns) {
+                    if (textLower.includes(pattern)) {
+                        bonus += 0.3; // Large bonus for definitional content
+                        break;
+                    }
+                }
+            }
+            // Bonus for individual word matches
+            let wordMatches = 0;
+            for (const word of queryWords) {
+                if (textLower.includes(word)) {
+                    wordMatches++;
+                }
+                if (titleLower.includes(word)) {
+                    wordMatches += 0.5; // Title matches are worth more
+                }
+            }
+            // Normalize word match bonus
+            if (queryWords.length > 0) {
+                bonus += (wordMatches / queryWords.length) * 0.1;
+            }
+            // Bonus for content that appears to be introductory/definitional
+            const introKeywords = ['introduction', 'what is', 'overview', 'about', 'definition', '# introduction'];
+            for (const keyword of introKeywords) {
+                if (textLower.includes(keyword) || titleLower.includes(keyword)) {
+                    bonus += 0.08;
+                }
+            }
+            // Extra bonus for files with "intro" in the path
+            if (result.document.source?.toLowerCase().includes('intro')) {
+                bonus += 0.1;
+            }
+            // Apply bonus but cap the total score at 1.0
+            const newScore = Math.min(1.0, score + bonus);
+            return {
+                ...result,
+                score: newScore
+            };
+        });
+        // Sort by new scores (descending)
+        rerankedResults.sort((a, b) => b.score - a.score);
+        return rerankedResults;
+    }
+}
+//# sourceMappingURL=reranker.js.map

package/dist/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,kCAAkC,EAAE,MAAM,2BAA2B,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IACvB,KAAK,GAAQ,IAAI,CAAC,CAAC,yDAAyD;IAC5E,SAAS,GAAQ,IAAI,CAAC;IACtB,SAAS,GAAW,+BAA+B,CAAC,CAAC,+BAA+B;IAC5F,+CAA+C;IACvC,MAAM,CAAU,eAAe,GAAG;QACxC,+BAA+B,EAAE,0CAA0C;QAC3E,sCAAsC,EAAE,4BAA4B;QACpE,sCAAsC,EAAE,oCAAoC;QAC5E,8FAA8F;KAC/F,CAAC;IAEF;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,0DAA0D;QAC1D,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,OAAO,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,SAAS,8BAA8B,CAAC,CAAC;QAC5E,KAAK,MAAM,aAAa,IAAI,oBAAoB,CAAC,eAAe,EAAE,CAAC;YACjE,IAAI,aAAa,KAAK,IAAI,CAAC,SAAS;gBAAE,SAAS,CAAC,2BAA2B;YAE3E,OAAO,CAAC,IAAI,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC;YACxD,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC;gBAC/B,OAAO;YACT,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACrF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,SAAiB;QAC1C,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;YAEzD,qEAAqE;YACrE,IAAI,CAAC,KAAK,GAAG,MAAM,kCAAkC,CAAC,eAAe,CAAC,SAAS,EAAE;gBAC/E,SAAS,EAAE,MAAM,CAAC,gBAAgB;gBAClC,KAAK,EAAE,MAAM,CAAC,0BAA0B;aACzC,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,SAAS,EAAE;gBAC9D,SAAS,EAAE,MAAM,CAAC,gBAAgB;aACnC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,wBAAwB,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YAC/G,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAuB;QACjD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAErD,gEAAgE;YAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;gBACvC,SAAS,EAAE,SAAS;gBACpB,OAAO,EAAE,IAAI;gBACb,UAAU,EAAE,IAAI;gBAChB,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,wBAAwB;YACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAE1C,sDAAsD;YACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAE7B,iDAAiD;YACjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,4EAA4E;gBAC5E,gCAAgC;gBAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAEpE,2DAA2D;gBAC3D,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAE5C,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,MAAM;iBACf,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,kCAAkC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,QAAQ,GAAG,QAAQ,CAAC;YAEvC,4EAA4E;YAC5E,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,mDAAmD,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC;gBACtH,OAAO,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;YAED,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAEzC,6DAA6D;YAC7D,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACzB,GAAG,IAAI,CAAC,MAAM;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,4DAA4D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YACrI,OAAO,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;OAGG;IACK,mBAAmB,CAAC,KAAa,EAAE,OAAuB;QAChE,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3E,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAE9D,IAAI,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,iCAAiC;YAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,gCAAgC;YAChC,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACnC,KAAK,IAAI,IAAI,CAAC;YAChB,CAAC;YAED,0BAA0B;YAC1B,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,KAAK,IAAI,GAAG,CAAC;YACf,CAAC;YAED,4CAA4C;YAC5C,IAAI,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAE5E,4DAA4D;gBAC5D,MAAM,kBAAkB,GAAG;oBACzB,GAAG,OAAO,OAAO;oBACjB,GAAG,OAAO,QAAQ;oBAClB,GAAG,OAAO,SAAS;oBACnB,GAAG,OAAO,WAAW;oBACrB,GAAG,OAAO,QAAQ;iBACnB,CAAC;gBAEF,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;oBACzC,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;wBAChC,KAAK,IAAI,GAAG,CAAC,CAAC,uCAAuC;wBACrD,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,oCAAoC;YACpC,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,WAAW,EAAE,CAAC;gBAChB,CAAC;gBACD,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9B,WAAW,IAAI,GAAG,CAAC,CAAC,+BAA+B;gBACrD,CAAC;YACH,CAAC;YAED,6BAA6B;YAC7B,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;YACnD,CAAC;YAED,iEAAiE;YACjE,MAAM,aAAa,GAAG,CAAC,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;YACvG,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;gBACpC,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChE,KAAK,IAAI,IAAI,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,iDAAiD;YACjD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5D,KAAK,IAAI,GAAG,CAAC;YACf,CAAC;YAED,6CAA6C;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC;YAE9C,OAAO;gBACL,GAAG,MAAM;gBACT,KAAK,EAAE,QAAQ;aAChB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAElD,OAAO,eAAe,CAAC;IACzB,CAAC"}

package/dist/resource-manager-demo.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Demo script showing ResourceManager usage
+ * This demonstrates the internal resource management system
+ */
+declare function demonstrateResourceManager(): Promise<void>;
+export { demonstrateResourceManager };
+//# sourceMappingURL=resource-manager-demo.d.ts.map

package/dist/resource-manager-demo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource-manager-demo.d.ts","sourceRoot":"","sources":["../src/resource-manager-demo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,iBAAe,0BAA0B,kBA4CxC;AAOD,OAAO,EAAE,0BAA0B,EAAE,CAAC"}

package/dist/resource-manager-demo.js

@@ -0,0 +1,52 @@
+/**
+ * Demo script showing ResourceManager usage
+ * This demonstrates the internal resource management system
+ */
+import { ResourceManager } from './resource-manager.js';
+import { CommonErrors } from './api-errors.js';
+async function demonstrateResourceManager() {
+    console.log('=== ResourceManager Demo ===\n');
+    // 1. Create ResourceManager instances
+    console.log('1. Creating ResourceManager instances...');
+    const manager1 = ResourceManager.getInstance({ basePath: './demo-data' });
+    const manager2 = ResourceManager.getInstance({ basePath: './demo-data' }); // Same config
+    const manager3 = ResourceManager.getInstance({ basePath: './other-data' }); // Different config
+    console.log(`   - manager1 === manager2: ${manager1 === manager2} (singleton pattern)`);
+    console.log(`   - manager1 === manager3: ${manager1 === manager3} (different configs)`);
+    console.log(`   - Active instances: ${ResourceManager.getActiveInstanceCount()}`);
+    // 2. Check if instances exist
+    console.log('\n2. Checking instance existence...');
+    console.log(`   - Has instance for './demo-data': ${ResourceManager.hasInstance({ basePath: './demo-data' })}`);
+    console.log(`   - Has instance for './nonexistent': ${ResourceManager.hasInstance({ basePath: './nonexistent' })}`);
+    // 3. Validate search files (will fail since files don't exist)
+    console.log('\n3. Validating search files...');
+    try {
+        await manager1.validateSearchFiles();
+        console.log('   - Validation passed');
+    }
+    catch (error) {
+        console.log(`   - Validation failed (expected): ${error instanceof Error ? error.message.split('\n')[0] : 'Unknown error'}`);
+    }
+    // 4. Demonstrate error handling
+    console.log('\n4. Demonstrating error handling...');
+    try {
+        throw CommonErrors.NO_DOCUMENTS_INGESTED;
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            console.log(`   - Error caught: ${error.message}`);
+        }
+    }
+    // 5. Cleanup resources
+    console.log('\n5. Cleaning up resources...');
+    console.log(`   - Before cleanup: ${ResourceManager.getActiveInstanceCount()} instances`);
+    await ResourceManager.cleanupAll();
+    console.log(`   - After cleanup: ${ResourceManager.getActiveInstanceCount()} instances`);
+    console.log('\n=== Demo Complete ===');
+}
+// Run the demo if this file is executed directly
+if (process.argv[1] && process.argv[1].endsWith('resource-manager-demo.js')) {
+    demonstrateResourceManager().catch(console.error);
+}
+export { demonstrateResourceManager };
+//# sourceMappingURL=resource-manager-demo.js.map

package/dist/resource-manager-demo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource-manager-demo.js","sourceRoot":"","sources":["../src/resource-manager-demo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,KAAK,UAAU,0BAA0B;IACvC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAE9C,sCAAsC;IACtC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,cAAc;IACzF,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,mBAAmB;IAE/F,OAAO,CAAC,GAAG,CAAC,+BAA+B,QAAQ,KAAK,QAAQ,sBAAsB,CAAC,CAAC;IACxF,OAAO,CAAC,GAAG,CAAC,+BAA+B,QAAQ,KAAK,QAAQ,sBAAsB,CAAC,CAAC;IACxF,OAAO,CAAC,GAAG,CAAC,0BAA0B,eAAe,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;IAElF,8BAA8B;IAC9B,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,wCAAwC,eAAe,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;IAChH,OAAO,CAAC,GAAG,CAAC,0CAA0C,eAAe,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;IAEpH,+DAA+D;IAC/D,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/H,CAAC;IAED,gCAAgC;IAChC,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IACpD,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,qBAAqB,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,wBAAwB,eAAe,CAAC,sBAAsB,EAAE,YAAY,CAAC,CAAC;IAC1F,MAAM,eAAe,CAAC,UAAU,EAAE,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,uBAAuB,eAAe,CAAC,sBAAsB,EAAE,YAAY,CAAC,CAAC;IAEzF,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;AACzC,CAAC;AAED,iDAAiD;AACjD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;IAC5E,0BAA0B,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC;AAED,OAAO,EAAE,0BAA0B,EAAE,CAAC"}

package/dist/resource-manager.d.ts

@@ -0,0 +1,129 @@
+import { DatabaseConnection } from './db.js';
+import { IndexManager } from './index-manager.js';
+import { EmbeddingEngine } from './embedder.js';
+/**
+ * Configuration for resource initialization
+ */
+export interface ResourceConfig {
+    basePath?: string;
+    dbPath?: string;
+    indexPath?: string;
+    modelName?: string;
+    batchSize?: number;
+    skipModelCheck?: boolean;
+}
+/**
+ * Managed resources container
+ */
+export interface ManagedResources {
+    database: DatabaseConnection;
+    indexManager: IndexManager;
+    embedder: EmbeddingEngine;
+}
+/**
+ * Internal ResourceManager for automatic initialization and cleanup
+ * Implements singleton pattern for resource sharing across instances
+ * Requirements: 5.1, 5.4 - Automatic resource management and lifecycle handling
+ */
+export declare class ResourceManager {
+    private static instances;
+    private static cleanupHandlersSet;
+    private database;
+    private indexManager;
+    private embedder;
+    private isInitialized;
+    private initializationPromise;
+    private readonly resourceKey;
+    private readonly config;
+    /**
+     * Get or create ResourceManager instance for the given configuration
+     * Implements singleton pattern for resource sharing
+     * @param config - Resource configuration
+     * @returns ResourceManager instance
+     */
+    static getInstance(config?: ResourceConfig): ResourceManager;
+    /**
+     * Resolve and normalize configuration with defaults
+     */
+    private static resolveConfig;
+    /**
+     * Private constructor - use getInstance() instead
+     */
+    private constructor();
+    /**
+     * Initialize all resources with lazy loading
+     * Requirements: 5.1 - Automatic resource management
+     */
+    initialize(): Promise<ManagedResources>;
+    /**
+     * Perform the actual initialization steps
+     */
+    private performInitialization;
+    /**
+     * Initialize database connection and schema
+     */
+    private initializeDatabase;
+    /**
+     * Initialize embedder based on stored model info or configuration
+     * Requirements: 5.1 - Embedder initialization based on stored model info from database
+     */
+    private initializeEmbedder;
+    /**
+     * Initialize index manager with model compatibility validation
+     */
+    private initializeIndexManager;
+    /**
+     * Get initialized resources
+     * @throws Error if resources are not initialized
+     */
+    private getResources;
+    /**
+     * Get database connection (lazy initialization)
+     */
+    getDatabase(): Promise<DatabaseConnection>;
+    /**
+     * Get index manager (lazy initialization)
+     */
+    getIndexManager(): Promise<IndexManager>;
+    /**
+     * Get embedder (lazy initialization)
+     */
+    getEmbedder(): Promise<EmbeddingEngine>;
+    /**
+     * Check if resources are initialized
+     */
+    isResourcesInitialized(): boolean;
+    /**
+     * Validate that required files exist for search operations
+     * @throws ResourceError with user-friendly message if files are missing
+     */
+    validateSearchFiles(): Promise<void>;
+    /**
+     * Set up automatic cleanup handlers
+     * Requirements: 5.5 - Automatic cleanup on process exit
+     */
+    private setupCleanupHandlers;
+    /**
+     * Synchronous cleanup for process exit handlers
+     */
+    private performSyncCleanup;
+    /**
+     * Clean up all resources with proper error handling
+     * Requirements: 5.5 - Proper resource disposal and error handling during cleanup
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Clean up all instances (for testing or shutdown)
+     * Requirements: 5.5 - Ensure resources are properly disposed when no longer needed
+     */
+    static cleanupAll(): Promise<void>;
+    /**
+     * Get the number of active resource instances
+     */
+    static getActiveInstanceCount(): number;
+    /**
+     * Check if a specific resource configuration is already managed
+     */
+    static hasInstance(config: ResourceConfig): boolean;
+}
+//# sourceMappingURL=resource-manager.d.ts.map

package/dist/resource-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource-manager.d.ts","sourceRoot":"","sources":["../src/resource-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAA0E,MAAM,SAAS,CAAC;AACrH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAA6B,MAAM,eAAe,CAAC;AAM3E;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,EAAE,eAAe,CAAC;CAC3B;AAED;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAsC;IAC9D,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAS;IAE1C,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAgC;IAChD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,qBAAqB,CAA8B;IAE3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD;;;;;OAKG;IACH,MAAM,CAAC,WAAW,CAAC,MAAM,GAAE,cAAmB,GAAG,eAAe;IAYhE;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAa5B;;OAEG;IACH,OAAO;IAQP;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAgB7C;;OAEG;YACW,qBAAqB;IAyBnC;;OAEG;YACW,kBAAkB;IAUhC;;;OAGG;YACW,kBAAkB;IAyChC;;OAEG;YACW,sBAAsB;IA0BpC;;;OAGG;IACH,OAAO,CAAC,YAAY;IAYpB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAOhD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAO9C;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC;IAO7C;;OAEG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACG,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB1C;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA+C5B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;;OAGG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAqD9B;;;OAGG;WACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBxC;;OAEG;IACH,MAAM,CAAC,sBAAsB,IAAI,MAAM;IAIvC;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO;CAKpD"}