vesper-wizard 2.3.1 → 2.3.2

package/build/metadata/rate-limiter.js

@@ -1,128 +0,0 @@
-/**
- * Rate limiting and retry utilities for API requests
- */
-const DEFAULT_OPTIONS = {
-    maxRetries: 5,
-    initialDelay: 1000, // 1 second
-    maxDelay: 60000, // 60 seconds
-    exponentialBase: 2,
-    jitter: true
-};
-/**
- * Calculate delay with exponential backoff and optional jitter
- */
-function calculateDelay(attempt, options) {
-    const exponentialDelay = options.initialDelay * Math.pow(options.exponentialBase, attempt);
-    const delay = Math.min(exponentialDelay, options.maxDelay);
-    if (options.jitter) {
-        // Add random jitter (0-20% of delay) to avoid thundering herd
-        const jitterAmount = delay * 0.2 * Math.random();
-        return Math.floor(delay + jitterAmount);
-    }
-    return Math.floor(delay);
-}
-/**
- * Extract Retry-After header value from response or error
- */
-function getRetryAfter(response) {
-    if (!response)
-        return null;
-    const retryAfter = response.headers.get('Retry-After');
-    if (!retryAfter)
-        return null;
-    // Retry-After can be a number of seconds or an HTTP date
-    const seconds = parseInt(retryAfter, 10);
-    if (!isNaN(seconds)) {
-        return seconds * 1000; // Convert to milliseconds
-    }
-    // Try parsing as HTTP date
-    const date = Date.parse(retryAfter);
-    if (!isNaN(date)) {
-        return Math.max(0, date - Date.now());
-    }
-    return null;
-}
-/**
- * Check if error is a rate limit error (429)
- */
-function isRateLimitError(error) {
-    if (error?.status === 429)
-        return true;
-    if (error?.response?.status === 429)
-        return true;
-    if (error?.message?.includes('rate limit'))
-        return true;
-    if (error?.message?.includes('429'))
-        return true;
-    return false;
-}
-/**
- * Sleep for specified milliseconds
- */
-function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-/**
- * Retry a function with exponential backoff on rate limit errors
- */
-export async function retryWithBackoff(fn, options = {}) {
-    const opts = { ...DEFAULT_OPTIONS, ...options };
-    let lastError;
-    let response = null;
-    for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
-        try {
-            const result = await fn();
-            return result;
-        }
-        catch (error) {
-            lastError = error;
-            // Extract response if available
-            if (error?.response) {
-                response = error.response;
-            }
-            // Only retry on rate limit errors
-            if (!isRateLimitError(error)) {
-                throw error;
-            }
-            // Don't retry on last attempt
-            if (attempt >= opts.maxRetries) {
-                break;
-            }
-            // Calculate delay
-            let delay = getRetryAfter(response);
-            if (!delay) {
-                delay = calculateDelay(attempt, opts);
-            }
-            console.error(`[Rate Limiter] Rate limited (attempt ${attempt + 1}/${opts.maxRetries + 1}). Waiting ${delay}ms...`);
-            await sleep(delay);
-        }
-    }
-    throw lastError;
-}
-/**
- * Add a delay between requests to avoid hitting rate limits
- */
-export async function delayBetweenRequests(ms = 500) {
-    await sleep(ms);
-}
-/**
- * Rate-limited fetch wrapper with automatic retry
- */
-export async function rateLimitedFetch(url, options = {}, retryOptions = {}) {
-    return retryWithBackoff(async () => {
-        const response = await fetch(url, options);
-        if (response.status === 429) {
-            const error = new Error(`Rate limit exceeded: ${response.status}`);
-            error.status = 429;
-            error.response = response;
-            throw error;
-        }
-        if (!response.ok) {
-            const error = new Error(`HTTP error: ${response.status}`);
-            error.status = response.status;
-            error.response = response;
-            throw error;
-        }
-        return response;
-    }, retryOptions);
-}

package/build/metadata/scraper.js

@@ -1,448 +0,0 @@
-import { listDatasets, datasetInfo } from "@huggingface/hub";
-import { categorizeLicense } from "./license.js";
-import { calculateQualityScore } from "./quality.js";
-import { classifyDomain } from "./domain.js";
-import { retryWithBackoff, delayBetweenRequests } from "./rate-limiter.js";
-import { analyzeDatasetQuery, buildIntentSearchQuery, buildHuggingFaceFilterTags, scoreDatasetAgainstIntent, shouldExcludeByLanguage } from "../search/query-intent.js";
-export class HuggingFaceScraper {
-    /**
-     * Bulk discovery: Fetch many datasets quickly without deep details.
-     * Hits the 25k target in minutes.
-     */
-    async scrapeBulk(limit = 1000, queryOrIntent) {
-        const intent = typeof queryOrIntent === "string"
-            ? await analyzeDatasetQuery(queryOrIntent)
-            : queryOrIntent;
-        const query = typeof queryOrIntent === "string" ? queryOrIntent : intent?.searchQuery;
-        const hfQuery = intent ? buildIntentSearchQuery(intent) : query;
-        const filterMsg = query ? `, query: ${query}` : "";
-        console.error(`[Bulk Scraper] Fetching datasets (target limit: ${limit}${filterMsg})...`);
-        const results = [];
-        let processed = 0;
-        try {
-            const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
-            const hfFilterTags = intent ? buildHuggingFaceFilterTags(intent) : [];
-            for await (const ds of listDatasets({
-                limit: limit,
-                additionalFields: ["description", "tags", "downloadsAllTime", "createdAt"],
-                search: { query: hfQuery, tags: hfFilterTags.length > 0 ? hfFilterTags : undefined },
-                ...(hfToken ? { accessToken: hfToken } : {})
-            })) {
-                if (results.length >= limit)
-                    break;
-                processed++;
-                if (processed % 1000 === 0) {
-                    console.error(`[Bulk Scraper] Found ${processed} datasets...`);
-                }
-                const repoId = ds.name || ds.id;
-                const description = ds.description || "";
-                const tags = ds.tags || [];
-                // Filter out non-repo IDs (hex IDs) if they don't have a namespace
-                if (!repoId.includes("/") && /^[a-f0-9]{24}$/.test(repoId))
-                    continue;
-                // Hard skip: Empty or very short description (cannot search semantically)
-                if (!description || description.length < 20)
-                    continue;
-                const createdAt = ds.createdAt;
-                const downloads = ds.downloadsAllTime || ds.downloads || 0;
-                const task = this.extractTask(tags);
-                // Create "skeleton" metadata
-                const metadata = {
-                    id: repoId,
-                    source: "huggingface",
-                    name: repoId.split("/").pop() || repoId,
-                    description: description,
-                    quality_warnings: ["Incomplete metadata: run hydration to get full details"],
-                    downloads: downloads,
-                    likes: ds.likes || 0,
-                    stars: 0,
-                    tags: tags,
-                    last_updated: createdAt instanceof Date ? createdAt.toISOString() : new Date().toISOString(),
-                    task: task,
-                    domain: classifyDomain(description, tags, repoId, task),
-                    languages: this.extractLanguages(tags),
-                    splits: [],
-                    license: {
-                        id: "unknown",
-                        category: "unknown",
-                        usage_restrictions: [],
-                        warnings: ["License not verified yet"]
-                    },
-                    quality_score: 10, // Default low score for skeleton
-                    download_url: `https://huggingface.co/datasets/${repoId}`,
-                    total_examples: 0,
-                    is_structured: false,
-                    has_target_column: false,
-                    is_safe_source: true, // Default to true, will be verified during hydration
-                    has_personal_data: false,
-                    is_paywalled: false,
-                    is_scraped_web_data: false,
-                    uses_https: true,
-                    has_train_split: false,
-                    has_test_split: false,
-                    has_validation_split: false,
-                    description_length: description.length,
-                    has_readme: false,
-                    is_incomplete: true // Flag for Phase 2
-                };
-                // Hard language exclusion
-                if (intent && shouldExcludeByLanguage(metadata, intent))
-                    continue;
-                results.push(metadata);
-            }
-        }
-        catch (e) {
-            console.error("[Bulk Scraper] Error:", e.message);
-        }
-        return results;
-    }
-    async scrape(limit = 100, applyMVPFilters = true, queryOrIntent) {
-        const intent = typeof queryOrIntent === "string"
-            ? await analyzeDatasetQuery(queryOrIntent)
-            : queryOrIntent;
-        const query = typeof queryOrIntent === "string" ? queryOrIntent : intent?.searchQuery;
-        const hfQuery = intent ? buildIntentSearchQuery(intent) : query;
-        const filterMsg = query ? `, query: ${query}` : "";
-        console.error(`Fetching datasets (target limit: ${limit}, MVP filters: ${applyMVPFilters}${filterMsg})...`);
-        const results = [];
-        let processed = 0;
-        let skippedMVP = 0;
-        let rateLimitHits = 0;
-        let otherErrors = 0;
-        try {
-            // Fetch more datasets to account for filtering
-            const fetchLimit = applyMVPFilters ? limit * 30 : limit * 10;
-            // Support HuggingFace token from environment variable
-            const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
-            // CRITICAL: Low concurrency without token to avoid rate limits
-            // With token: 10 concurrent (HF allows more)
-            // Without token: 2 concurrent (stay under ~30 req/min limit)
-            const CONCURRENCY = hfToken ? 10 : 2;
-            const queue = [];
-            if (!hfToken) {
-                console.error(`[HF Scraper] ⚠️ No HF_TOKEN set - using conservative rate limits`);
-            }
-            // Add delay between batches to avoid rate limits
-            const BATCH_DELAY = hfToken ? 500 : 2000;
-            const hfFilterTags = intent ? buildHuggingFaceFilterTags(intent) : [];
-            for await (const ds of listDatasets({
-                limit: fetchLimit,
-                additionalFields: ["description", "tags"],
-                search: { query: hfQuery, tags: hfFilterTags.length > 0 ? hfFilterTags : undefined },
-                ...(hfToken ? { accessToken: hfToken } : {})
-            })) {
-                if (results.length >= limit)
-                    break;
-                processed++;
-                // Add delay every 100 datasets to avoid aggressive rate limits
-                if (processed % 100 === 0 && processed > 0) {
-                    await delayBetweenRequests(hfToken ? 500 : 2000);
-                }
-                const repoId = ds.name || ds.id;
-                // Filter out non-repo IDs (hex IDs) if they don't have a namespace
-                if (!repoId.includes("/")) {
-                    if (/^[a-f0-9]{24}$/.test(repoId))
-                        continue;
-                }
-                const downloads = ds.downloads || 0;
-                const tags = ds.tags || [];
-                const description = ds.description || "";
-                if (processed % 100 === 0) {
-                    console.error(`[HF Scraper] Processed ${processed}, found ${results.length}, workers: ${queue.length}...`);
-                }
-                // Hard skip: Empty or very short description (cannot search semantically)
-                if (!description || description.length < 20)
-                    continue;
-                // Process dataset with concurrency and retry logic
-                const processTask = (async () => {
-                    try {
-                        const fullInfo = await retryWithBackoff(() => datasetInfo({
-                            name: repoId,
-                            additionalFields: ["cardData"],
-                            ...(hfToken ? { accessToken: hfToken } : {})
-                        }), {
-                            maxRetries: 3,
-                            initialDelay: 2000, // Start with 2 seconds for HF API
-                            maxDelay: 30000 // Max 30 seconds
-                        });
-                        const cardData = fullInfo.cardData || {};
-                        // Extract splits from cardData.dataset_info (where HF actually stores them)
-                        // cardData.dataset_info can be an object (single config) or array (multi-config)
-                        let rawSplits = [];
-                        const datasetInfoField = cardData.dataset_info;
-                        if (datasetInfoField) {
-                            const configs = Array.isArray(datasetInfoField) ? datasetInfoField : [datasetInfoField];
-                            for (const config of configs) {
-                                if (config?.splits && Array.isArray(config.splits)) {
-                                    rawSplits = rawSplits.concat(config.splits);
-                                }
-                            }
-                        }
-                        // Fallback: try top-level splits from the SDK (rarely populated)
-                        if (rawSplits.length === 0 && fullInfo.splits) {
-                            rawSplits = fullInfo.splits;
-                        }
-                        const splits = rawSplits.map((s) => ({
-                            name: s.name,
-                            num_examples: s.num_examples || s.numExamples || 0,
-                            size_bytes: s.num_bytes || s.sizeBytes || 0
-                        }));
-                        let totalExamples = splits.reduce((sum, s) => sum + (s.num_examples || 0), 0);
-                        const totalSizeBytes = splits.reduce((sum, s) => sum + (s.size_bytes || 0), 0);
-                        // Fallback: estimate from size_categories when splits give 0
-                        if (totalExamples === 0) {
-                            const sizeCategories = cardData.size_categories;
-                            if (Array.isArray(sizeCategories) && sizeCategories.length > 0) {
-                                const cat = sizeCategories[0];
-                                const rangeMatch = cat.match(/([\d.]+[KMB]?)\s*<\s*n\s*<\s*([\d.]+[KMB]?)/i);
-                                if (rangeMatch) {
-                                    const parseHumanNum = (s) => {
-                                        const m = s.match(/^([\d.]+)([KMB])?$/i);
-                                        if (!m)
-                                            return 0;
-                                        const base = parseFloat(m[1]);
-                                        const suffix = (m[2] || '').toUpperCase();
-                                        if (suffix === 'K')
-                                            return base * 1000;
-                                        if (suffix === 'M')
-                                            return base * 1_000_000;
-                                        if (suffix === 'B')
-                                            return base * 1_000_000_000;
-                                        return base;
-                                    };
-                                    const lo = parseHumanNum(rangeMatch[1]);
-                                    const hi = parseHumanNum(rangeMatch[2]);
-                                    totalExamples = Math.round((lo + hi) / 2);
-                                }
-                            }
-                        }
-                        const totalSizeMB = totalSizeBytes ? Math.round(totalSizeBytes / (1024 * 1024) * 100) / 100 : undefined;
-                        const hasValidationSplit = splits.some((s) => s.name === "validation" || s.name === "val");
-                        const licenseTag = tags.find(t => t.startsWith("license:"));
-                        const licenseId = licenseTag ? licenseTag.replace("license:", "") : fullInfo.license;
-                        const licenseUrl = cardData.license?.[0]?.link || cardData.license_link;
-                        const license = categorizeLicense(licenseId, licenseUrl);
-                        if (license.category === "restricted") {
-                            skippedMVP++;
-                            return;
-                        }
-                        const warnings = [];
-                        if (totalExamples < 50)
-                            warnings.push("Dataset has very few examples (< 50)");
-                        if (description.length < 100)
-                            warnings.push("Short description; results may be less relevant");
-                        const lastUpdated = ds.updatedAt || fullInfo.updatedAt;
-                        if (lastUpdated) {
-                            const updateDate = new Date(lastUpdated);
-                            const fourYearsAgo = new Date();
-                            fourYearsAgo.setFullYear(fourYearsAgo.getFullYear() - 4);
-                            if (updateDate < fourYearsAgo) {
-                                warnings.push(`Stale data: Last updated ${updateDate.getFullYear()}`);
-                            }
-                        }
-                        if (splits.length === 0)
-                            warnings.push("No data splits found; could be a non-standard format");
-                        const descriptionLower = description.toLowerCase();
-                        const tagsLower = tags.map(t => t.toLowerCase()).join(" ");
-                        const hasPersonalData = descriptionLower.includes("personal data") ||
-                            descriptionLower.includes("gdpr") ||
-                            tagsLower.includes("personal-data");
-                        const isPaywalled = descriptionLower.includes("paywall") ||
-                            descriptionLower.includes("paid");
-                        const isScrapedWebData = descriptionLower.includes("scraped") ||
-                            tagsLower.includes("scraped");
-                        const isSafeSource = !isScrapedWebData && !hasPersonalData && !isPaywalled;
-                        if (!isSafeSource)
-                            warnings.push("Contains potentially sensitive or paywalled data sources");
-                        const format = this.extractFormat(tags, cardData);
-                        const columns = this.extractColumns(cardData, splits);
-                        const task = this.extractTask(tags);
-                        const domain = classifyDomain(description, tags, repoId, task);
-                        // REMOVED strict domain filtering that caused search bias
-                        // if (query && domain !== query) return; 
-                        const metadata = {
-                            id: repoId,
-                            source: "huggingface",
-                            name: repoId.split("/").pop() || repoId,
-                            description: description,
-                            quality_warnings: warnings,
-                            downloads: downloads,
-                            likes: ds.likes || 0,
-                            stars: fullInfo.stars || 0,
-                            tags: tags,
-                            last_updated: lastUpdated?.toISOString() || new Date().toISOString(),
-                            task: task,
-                            domain: domain,
-                            languages: this.extractLanguages(tags),
-                            splits,
-                            license,
-                            quality_score: calculateQualityScore({
-                                downloads,
-                                likes: ds.likes || 0,
-                                hasDescription: true,
-                                descriptionLength: description.length,
-                                hasTrainSplit: splits.some((s) => s.name === "train"),
-                                hasTestSplit: splits.some((s) => s.name === "test"),
-                                lastUpdated: lastUpdated?.toISOString() || new Date().toISOString(),
-                                licenseCategory: license.category
-                            }),
-                            download_url: `https://huggingface.co/datasets/${repoId}`,
-                            format,
-                            total_examples: totalExamples,
-                            total_size_bytes: totalSizeBytes,
-                            total_size_mb: totalSizeMB,
-                            columns,
-                            is_structured: columns.length > 0 && splits.some((s) => s.name === "train"),
-                            has_target_column: columns.some(c => c.is_target === true),
-                            is_safe_source: isSafeSource,
-                            has_personal_data: hasPersonalData,
-                            is_paywalled: isPaywalled,
-                            is_scraped_web_data: isScrapedWebData,
-                            uses_https: true,
-                            has_train_split: splits.some((s) => s.name === "train"),
-                            has_test_split: splits.some((s) => s.name === "test"),
-                            has_validation_split: hasValidationSplit,
-                            description_length: description.length,
-                            has_readme: !!(cardData.readme || cardData.readme_content)
-                        };
-                        // Hard language exclusion — drop bilingual/multilingual for single-language queries
-                        if (intent && shouldExcludeByLanguage(metadata, intent)) {
-                            // skip — do not push
-                        }
-                        else {
-                            if (intent) {
-                                metadata.intent_score = scoreDatasetAgainstIntent(metadata, intent);
-                            }
-                            results.push(metadata);
-                        }
-                    }
-                    catch (e) {
-                        // Track all errors for user feedback
-                        if (e?.status === 429 || e?.message?.includes('rate limit')) {
-                            rateLimitHits++;
-                            if (rateLimitHits <= 3) {
-                                console.error(`[HF Scraper] Rate limit hit for ${repoId}`);
-                            }
-                        }
-                        else {
-                            otherErrors++;
-                            // Log first few non-rate-limit errors for debugging
-                            if (otherErrors <= 2) {
-                                console.error(`[HF Scraper] Error for ${repoId}: ${e.message?.slice(0, 80)}`);
-                            }
-                        }
-                    }
-                })();
-                queue.push(processTask);
-                processTask.then(() => {
-                    const idx = queue.indexOf(processTask);
-                    if (idx !== -1)
-                        queue.splice(idx, 1);
-                });
-                if (queue.length >= CONCURRENCY) {
-                    await Promise.race(queue);
-                }
-            }
-            // Wait for remaining tasks
-            await Promise.all(queue);
-        }
-        catch (e) {
-            // Handle rate limit errors with better messaging
-            if (e?.status === 429 || e?.message?.includes('rate limit')) {
-                console.error("[HF Scraper] ❌ Scraping failed due to rate limit:", e.message);
-                console.error("[HF Scraper] 💡 Set HF_TOKEN environment variable for unlimited access");
-            }
-            else {
-                console.error("[HF Scraper] ❌ Scraping failed:", e.message);
-            }
-        }
-        // User-friendly summary
-        console.error(`[HF Scraper] ✅ Complete: ${results.length} datasets found`);
-        if (rateLimitHits > 0) {
-            console.error(`[HF Scraper] ⚠️ ${rateLimitHits} requests rate-limited. Set HF_TOKEN for better results.`);
-        }
-        if (otherErrors > 0) {
-            console.error(`[HF Scraper] ⚠️ ${otherErrors} datasets skipped due to errors`);
-        }
-        return results.sort((a, b) => {
-            const intentDelta = Number(b.intent_score || 0) - Number(a.intent_score || 0);
-            if (intentDelta !== 0)
-                return intentDelta;
-            return b.downloads - a.downloads;
-        });
-    }
-    extractTask(tags) {
-        const taskTags = [
-            "text-classification",
-            "token-classification",
-            "question-answering",
-            "translation",
-            "summarization",
-            "text-generation",
-            "image-classification",
-            "object-detection",
-            "named-entity-recognition",
-            "sentiment-analysis",
-            "machine-translation"
-        ];
-        return tags.find(t => taskTags.includes(t)) || "unknown";
-    }
-    extractLanguages(tags) {
-        const langs = tags
-            .filter(t => t.startsWith("language:"))
-            .map(t => t.replace("language:", ""));
-        const isoLangs = tags.filter(t => t.length === 2 && /^[a-z]{2}$/.test(t));
-        return Array.from(new Set([...langs, ...isoLangs]));
-    }
-    extractFormat(tags, cardData) {
-        // Check tags first
-        const formatTag = tags.find(t => t.startsWith("format:"));
-        if (formatTag) {
-            return formatTag.replace("format:", "").toUpperCase();
-        }
-        // Check cardData
-        if (cardData.format) {
-            return String(cardData.format).toUpperCase();
-        }
-        // Infer from other tags
-        if (tags.includes("parquet"))
-            return "PARQUET";
-        if (tags.includes("csv"))
-            return "CSV";
-        if (tags.includes("json"))
-            return "JSON";
-        if (tags.includes("arrow"))
-            return "ARROW";
-        return undefined;
-    }
-    extractColumns(cardData, splits) {
-        const columns = [];
-        // Try to get columns from cardData
-        if (cardData.columns) {
-            if (Array.isArray(cardData.columns)) {
-                return cardData.columns.map((col) => ({
-                    name: col.name || String(col),
-                    type: col.type,
-                    is_target: col.is_target || false
-                }));
-            }
-        }
-        // Try to infer from features if available
-        if (cardData.features) {
-            const features = cardData.features;
-            if (typeof features === 'object') {
-                for (const [key, value] of Object.entries(features)) {
-                    const feature = value;
-                    columns.push({
-                        name: key,
-                        type: feature?.dtype || feature?.type,
-                        is_target: key.toLowerCase().includes("label") ||
-                            key.toLowerCase().includes("target") ||
-                            key.toLowerCase().includes("y")
-                    });
-                }
-            }
-        }
-        return columns;
-    }
-}