vesper-wizard 2.3.1 → 2.3.2

package/build/search/engine.js

@@ -1,190 +0,0 @@
-import { JITOrchestrator } from "./jit-orchestrator.js";
-import { analyzeDatasetQuery, scoreDatasetAgainstIntent, shouldExcludeByLanguage } from "./query-intent.js";
-import fs from "fs";
-function log(msg) {
-    fs.appendFileSync("debug.log", new Date().toISOString() + " " + msg + "\n");
-}
-export class SearchEngine {
-    metadataStore;
-    vectorStore;
-    embedder;
-    jitOrchestrator;
-    constructor(metadataStore, vectorStore, embedder) {
-        this.metadataStore = metadataStore;
-        this.vectorStore = vectorStore;
-        this.embedder = embedder;
-        this.jitOrchestrator = new JITOrchestrator(metadataStore, vectorStore, embedder);
-    }
-    async search(query, options = {}) {
-        const limit = options.limit || 5;
-        const enableJIT = options.enableJIT !== false; // Default: true
-        const intent = await analyzeDatasetQuery(query);
-        log(`Search request: "${query}" Limit=${limit} Safe=${options.safeOnly} JIT=${enableJIT}`);
-        // 1. Perform local search
-        const localResults = await this.localSearch(query, options, intent);
-        // 2. Check if JIT should be triggered
-        const shouldTrigger = enableJIT && this.shouldTriggerJIT(localResults, query);
-        if (!shouldTrigger) {
-            log(`JIT not triggered. Returning ${localResults.length} local results`);
-            return localResults;
-        }
-        // 3. Trigger JIT fallback
-        console.error(`\nWARNING: Low confidence results (${localResults.length} found, top score: ${localResults[0]?.relevance_score || 0})`);
-        await this.jitOrchestrator.fetchAndIngest(query, 10, intent);
-        // 4. Re-run local search with updated index
-        console.error(`Re-searching with updated library...`);
-        const enhancedResults = await this.localSearch(query, options, intent);
-        const newCount = enhancedResults.length - localResults.length;
-        if (newCount > 0) {
-            console.error(`Found ${newCount} additional results\n`);
-        }
-        return enhancedResults;
-    }
-    /**
-     * Perform hybrid search (Vector + Lexical + Penalties)
-     */
-    async localSearch(query, options, intent) {
-        const limit = options.limit || 5;
-        // 1. Parse Query
-        const words = query.toLowerCase().split(/\s+/);
-        const positiveKeywords = words.filter(w => !w.startsWith("-") && w.length > 2);
-        const negativeKeywords = words.filter(w => w.startsWith("-")).map(w => w.slice(1));
-        // Automatic Penalty Detection: Is this a "Finance" query without "Crypto" mentioned?
-        const financeTerms = ["financial", "finance", "banking", "economy", "stock", "loan", "forecasting", "bank"];
-        const isFinanceQuery = positiveKeywords.some(w => financeTerms.includes(w));
-        const mentionsCrypto = query.toLowerCase().includes("crypto") || query.toLowerCase().includes("bitcoin");
-        // 2. Get query vector
-        const queryVector = await this.embedder.embed(query);
-        log(`Vector generated, length=${queryVector.length}`);
-        // 3. Search in vector store (fetch more candidates for reranking)
-        const matches = this.vectorStore.search(queryVector, 40);
-        log(`Vector search found ${matches.length} matches`);
-        // 4. Score and filter candidates
-        const results = [];
-        for (const match of matches) {
-            const metadata = this.metadataStore.getDataset(match.id);
-            if (!metadata)
-                continue;
-            // Filter: Safe only
-            if (options.safeOnly && metadata.license.category === "restricted")
-                continue;
-            // Filter: Hard language exclusion — completely drop bilingual/multilingual datasets
-            // when user explicitly requested a single language
-            if (shouldExcludeByLanguage(metadata, intent)) {
-                log(`Language exclusion: Dropped ${match.id} (bilingual/multilingual for single-language query)`);
-                continue;
-            }
-            const text = `${metadata.name} ${metadata.description} ${metadata.tags.join(" ")}`.toLowerCase();
-            // Filter: Explicit Negative Keywords
-            if (negativeKeywords.some(neg => text.includes(neg))) {
-                log(`Negative penalty: Dropped ${match.id} due to keyword match`);
-                continue;
-            }
-            // A. Vector Score (0.0 to 1.0)
-            const vectorScore = match.score;
-            // B. Lexical Score (Keyword Match)
-            let lexicalScore = 0;
-            if (positiveKeywords.length > 0) {
-                const matchesCount = positiveKeywords.filter(kw => text.includes(kw)).length;
-                lexicalScore = matchesCount / positiveKeywords.length;
-            }
-            // C. Penalties
-            let penalty = 0;
-            // Penalty: Domain Drift (Finance vs Crypto)
-            // If user asks for finance but NOT crypto, and the result is crypto, penalize heavily
-            if (isFinanceQuery && !mentionsCrypto) {
-                const isCryptoResult = text.includes("crypto") || text.includes("bitcoin") || text.includes("ethereum") || text.includes("blockchain");
-                if (isCryptoResult) {
-                    log(`Penalty applied: Crypto result in Finance query for ${match.id}`);
-                    penalty += 0.4; // Massive penalty
-                }
-            }
-            // Penalty: Weak Lexical Match
-            // If it's a "semantic vibe" match but has ZERO matching keywords from the query
-            if (lexicalScore === 0 && positiveKeywords.length > 1) {
-                penalty += 0.2;
-            }
-            // Penalty: Modality Mismatch
-            // Infer the expected modality from the query and penalize mismatches.
-            // e.g., "anime quotes" is text, so image-classification datasets get penalized.
-            const textIndicators = ["quotes", "text", "nlp", "sentiment", "review", "comment", "caption", "dialogue", "chat", "translation", "summarization", "classification"];
-            const imageIndicators = ["image", "photo", "picture", "vision", "detection", "segmentation", "face", "background"];
-            const queryLower = query.toLowerCase();
-            const queryLooksText = textIndicators.some(t => queryLower.includes(t));
-            const queryLooksImage = imageIndicators.some(t => queryLower.includes(t));
-            if (queryLooksText && !queryLooksImage) {
-                const resultTask = (metadata.task || "").toLowerCase();
-                const isImageResult = resultTask.includes("image") || resultTask.includes("object-detection") ||
-                    text.includes("image classification") || text.includes("image-classification") ||
-                    text.includes("object detection") || text.includes("image segmentation");
-                if (isImageResult) {
-                    log(`Modality penalty: text query but image dataset ${match.id}`);
-                    penalty += 0.35;
-                }
-            }
-            if (queryLooksImage && !queryLooksText) {
-                const resultTask = (metadata.task || "").toLowerCase();
-                const isTextResult = resultTask.includes("text-classification") || resultTask.includes("text-generation") ||
-                    resultTask.includes("translation") || resultTask.includes("summarization") ||
-                    resultTask.includes("question-answering");
-                if (isTextResult) {
-                    log(`Modality penalty: image query but text dataset ${match.id}`);
-                    penalty += 0.35;
-                }
-            }
-            // D. Accessibility Bonuses (Prioritize low-friction sources)
-            let bonus = 0;
-            const sourceBonuses = {
-                "huggingface": 0.1,
-                "uci": 0.1,
-                "github": 0.1,
-                "worldbank": 0.1,
-                "nasa": 0.1
-            };
-            bonus = sourceBonuses[metadata.source] || 0;
-            // Final Combined Score
-            // 70% Vector, 30% Lexical, minus Penalties, plus Bonuses
-            const intentScore = scoreDatasetAgainstIntent(metadata, intent);
-            const finalScore = (vectorScore * 0.7) + (lexicalScore * 0.3) - penalty + bonus + intentScore;
-            metadata.relevance_score = Math.round(finalScore * 100) / 100;
-            metadata.vector_score = Math.round(vectorScore * 100) / 100;
-            metadata.lexical_score = Math.round(lexicalScore * 100) / 100;
-            metadata.accessibility_bonus = bonus;
-            metadata.intent_score = intentScore;
-            results.push(metadata);
-        }
-        // Sort by final score and limit
-        return results
-            .sort((a, b) => b.relevance_score - a.relevance_score)
-            .slice(0, limit);
-    }
-    /**
-     * Determine if JIT should be triggered
-     */
-    shouldTriggerJIT(results, query) {
-        const queryWords = query.trim().split(/\s+/).length;
-        // Condition 1: Zero results (Always trigger JIT)
-        if (results.length === 0) {
-            log(`JIT trigger: No results found for "${query}"`);
-            return true;
-        }
-        // Condition 2: Very few results
-        if (results.length < 3) {
-            log(`JIT trigger: Only ${results.length} results found`);
-            return true;
-        }
-        // Condition 3: Low confidence (top result has low similarity)
-        // Increased threshold from 0.60 to 0.75 for better specific-query matching
-        const topScore = results[0]?.relevance_score || 0;
-        if (topScore < 0.75) {
-            log(`JIT trigger: Low confidence (top score: ${topScore}, threshold: 0.75)`);
-            return true;
-        }
-        // Condition 4: Long specific queries with mediocre matches
-        if (queryWords >= 4 && topScore < 0.85) {
-            log(`JIT trigger: Long query with mediocre top score (${topScore})`);
-            return true;
-        }
-        return false;
-    }
-}

package/build/search/jit-orchestrator.js

@@ -1,262 +0,0 @@
-import { HuggingFaceScraper } from "../metadata/scraper.js";
-import { UCIScraper } from "../metadata/uci-scraper.js";
-import { GitHubScraper } from "../metadata/github-scraper.js";
-import { WorldBankScraper, NASAScraper } from "../metadata/institutional-scrapers.js";
-import { analyzeDatasetQuery, buildIntentSearchQuery } from "./query-intent.js";
-// Common stop words to filter out for better search
-const STOP_WORDS = new Set([
-    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
-    "of", "with", "by", "from", "as", "is", "was", "are", "were", "been",
-    "be", "have", "has", "had", "do", "does", "did", "will", "would",
-    "could", "should", "may", "might", "must", "shall", "can", "need",
-    "about", "into", "through", "during", "before", "after", "above",
-    "below", "between", "under", "again", "further", "then", "once",
-    "here", "there", "when", "where", "why", "how", "all", "each",
-    "few", "more", "most", "other", "some", "such", "no", "nor", "not",
-    "only", "own", "same", "so", "than", "too", "very", "just", "also",
-    "dataset", "datasets", "data", "find", "search", "get", "looking"
-]);
-/**
- * Just-In-Time Orchestrator
- * Automatically fetches and indexes new datasets when local search is insufficient
- */
-export class JITOrchestrator {
-    metadataStore;
-    vectorStore;
-    embedder;
-    lastTriggerTime = new Map();
-    RATE_LIMIT_MS = 30000; // 30 seconds between triggers for same query
-    constructor(metadataStore, vectorStore, embedder) {
-        this.metadataStore = metadataStore;
-        this.vectorStore = vectorStore;
-        this.embedder = embedder;
-    }
-    /**
-     * Simplify a complex user query into keywords that work better with APIs
-     * HuggingFace and other APIs often fail on long multi-word queries
-     */
-    simplifyQuery(query) {
-        // Split into words, lowercase, remove punctuation
-        const words = query.toLowerCase()
-            .replace(/[^\w\s-]/g, "")
-            .split(/\s+/)
-            .filter(w => w.length > 2 && !STOP_WORDS.has(w));
-        // Return unique keywords (max 3 for API-friendly queries)
-        const unique = [...new Set(words)];
-        // If we have a lot of words, prioritize longer/more specific ones
-        if (unique.length > 3) {
-            unique.sort((a, b) => b.length - a.length);
-        }
-        return unique.slice(0, 3);
-    }
-    /**
-     * Check if JIT should be triggered based on rate limiting
-     */
-    canTrigger(query) {
-        const lastTrigger = this.lastTriggerTime.get(query);
-        if (!lastTrigger)
-            return true;
-        const elapsed = Date.now() - lastTrigger;
-        return elapsed > this.RATE_LIMIT_MS;
-    }
-    /**
-     * Main JIT workflow: fetch, save, index, return new datasets
-     */
-    async fetchAndIngest(query, limit = 10, providedIntent) {
-        // Rate limiting check
-        if (!this.canTrigger(query)) {
-            console.error(`[JIT] Query "${query}" was searched recently. Waiting...`);
-            return [];
-        }
-        console.error(`\n[JIT] Searching live sources for: "${query}"`);
-        this.lastTriggerTime.set(query, Date.now());
-        const intent = providedIntent || await analyzeDatasetQuery(query);
-        const keywords = this.simplifyQuery(buildIntentSearchQuery(intent));
-        if (intent.llmBacked || intent.language || intent.task || intent.domain || intent.minRows) {
-            console.error(`[JIT] Intent: ${JSON.stringify({ language: intent.language, task: intent.task, domain: intent.domain, minRows: intent.minRows, searchQuery: intent.searchQuery })}`);
-        }
-        else if (keywords.length > 0) {
-            console.error(`[JIT] Keywords extracted: ${keywords.join(", ")}`);
-        }
-        const newDatasets = [];
-        const existingIds = new Set();
-        const sourceResults = {};
-        try {
-            // Get existing dataset IDs to avoid duplicates
-            const existing = this.metadataStore.getAllDatasets();
-            existing.forEach(ds => existingIds.add(ds.id));
-            let hfResults = await this.scrapeHuggingFace(intent, limit);
-            if (hfResults.length < Math.max(3, Math.floor(limit / 2))) {
-                for (const keyword of keywords) {
-                    if (hfResults.length >= limit)
-                        break;
-                    const results = await this.scrapeHuggingFace(keyword, Math.ceil(limit / Math.max(keywords.length, 1)));
-                    for (const ds of results) {
-                        if (!hfResults.some(existing => existing.id === ds.id)) {
-                            hfResults.push(ds);
-                        }
-                    }
-                }
-            }
-            sourceResults["HuggingFace"] = hfResults.length;
-            console.error(`  [source] HuggingFace: ${hfResults.length} datasets`);
-            for (const ds of hfResults) {
-                if (!existingIds.has(ds.id)) {
-                    newDatasets.push(ds);
-                    existingIds.add(ds.id);
-                }
-            }
-            // 2. Scrape UCI (Open Access)
-            const uciResults = await this.scrapeUCI(query, Math.floor(limit / 2));
-            sourceResults["UCI"] = uciResults.length;
-            console.error(`  [source] UCI: ${uciResults.length} datasets`);
-            for (const ds of uciResults) {
-                if (!existingIds.has(ds.id)) {
-                    newDatasets.push(ds);
-                    existingIds.add(ds.id);
-                }
-            }
-            // 3. Scrape GitHub (Open Access)
-            const githubResults = await this.scrapeGitHub(query, Math.floor(limit / 2));
-            sourceResults["GitHub"] = githubResults.length;
-            console.error(`  [source] GitHub: ${githubResults.length} datasets`);
-            for (const ds of githubResults) {
-                if (!existingIds.has(ds.id)) {
-                    newDatasets.push(ds);
-                    existingIds.add(ds.id);
-                }
-            }
-            // 4. Scrape World Bank (Open Access) - Economic/demographic data
-            const wbResults = await this.scrapeWorldBank(query, Math.floor(limit / 2));
-            sourceResults["WorldBank"] = wbResults.length;
-            console.error(`  [source] World Bank: ${wbResults.length} datasets`);
-            for (const ds of wbResults) {
-                if (!existingIds.has(ds.id)) {
-                    newDatasets.push(ds);
-                    existingIds.add(ds.id);
-                }
-            }
-            // 5. Scrape NASA (Open Access) - Scientific/space data
-            const nasaResults = await this.scrapeNASA(query, Math.floor(limit / 2));
-            sourceResults["NASA"] = nasaResults.length;
-            console.error(`  [source] NASA: ${nasaResults.length} datasets`);
-            for (const ds of nasaResults) {
-                if (!existingIds.has(ds.id)) {
-                    newDatasets.push(ds);
-                    existingIds.add(ds.id);
-                }
-            }
-            // Save and index new datasets
-            if (newDatasets.length > 0) {
-                await this.saveAndIndex(newDatasets);
-                console.error(`[JIT] Indexed ${newDatasets.length} new datasets`);
-            }
-            else {
-                // Provide helpful feedback when no results found
-                const allZero = Object.values(sourceResults).every(v => v === 0);
-                if (allZero) {
-                    console.error(`[JIT] No datasets found across all sources.`);
-                    console.error(`[JIT] Try: broader keywords, or set HF_TOKEN for better HuggingFace access`);
-                }
-                else {
-                    console.error(`[JIT] All found datasets already in index`);
-                }
-            }
-            return newDatasets;
-        }
-        catch (error) {
-            console.error(`ERROR [JIT] Error during fetch and ingest:`, error.message);
-            return [];
-        }
-    }
-    /**
-     * Scrape HuggingFace with free-text search
-     */
-    async scrapeHuggingFace(query, limit) {
-        const scraper = new HuggingFaceScraper();
-        try {
-            return await scraper.scrape(limit, true, query);
-        }
-        catch (error) {
-            console.error(`  ERROR: HuggingFace scrape failed: ${error.message}`);
-            return [];
-        }
-    }
-    /**
-     * Scrape UCI
-     */
-    async scrapeUCI(query, limit) {
-        const scraper = new UCIScraper();
-        try {
-            return await scraper.scrape(query, limit);
-        }
-        catch (error) {
-            console.error(`  ERROR: UCI scrape failed: ${error.message}`);
-            return [];
-        }
-    }
-    /**
-     * Scrape GitHub
-     */
-    async scrapeGitHub(query, limit) {
-        const scraper = new GitHubScraper();
-        try {
-            return await scraper.scrape(query, limit);
-        }
-        catch (error) {
-            console.error(`  ERROR: GitHub scrape failed: ${error.message}`);
-            return [];
-        }
-    }
-    /**
-     * Scrape World Bank
-     */
-    async scrapeWorldBank(query, limit) {
-        const scraper = new WorldBankScraper();
-        try {
-            return await scraper.scrape(query, limit);
-        }
-        catch (error) {
-            console.error(`  ERROR: World Bank scrape failed: ${error.message}`);
-            return [];
-        }
-    }
-    /**
-     * Scrape NASA
-     */
-    async scrapeNASA(query, limit) {
-        const scraper = new NASAScraper();
-        try {
-            return await scraper.scrape(query, limit);
-        }
-        catch (error) {
-            console.error(`  ERROR: NASA scrape failed: ${error.message}`);
-            return [];
-        }
-    }
-    /**
-     * Save datasets to DB and generate embeddings
-     */
-    async saveAndIndex(datasets) {
-        // 1. Save to database
-        this.metadataStore.beginTransaction();
-        try {
-            for (const ds of datasets) {
-                this.metadataStore.saveDataset(ds);
-            }
-            this.metadataStore.commit();
-        }
-        catch (e) {
-            this.metadataStore.rollback();
-            throw e;
-        }
-        // 2. Generate embeddings and update vector store
-        for (const ds of datasets) {
-            const text = `${ds.name} ${ds.description} ${ds.tags.join(" ")}`;
-            const vector = await this.embedder.embed(text);
-            this.vectorStore.add(ds.id, vector);
-        }
-        // 3. Persist vector store to disk
-        this.vectorStore.save();
-    }
-}