vesper-wizard 2.3.1 → 2.3.3

package/build/gateway/unified-dataset-gateway.js

@@ -1,410 +0,0 @@
-import fs from "fs";
-import path from "path";
-import http from "http";
-import https from "https";
-import { HuggingFaceScraper } from "../metadata/scraper.js";
-import { analyzeDatasetQuery } from "../search/query-intent.js";
-export class UnifiedDatasetGateway {
-    deps;
-    constructor(deps) {
-        this.deps = deps;
-    }
-    getProviderStatuses(includeUnavailable = true) {
-        const hasHfToken = !!(process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN);
-        const hasKaggle = this.deps.dataIngestor.hasKaggleCredentials();
-        const hasDataWorld = this.deps.hasDataWorldToken();
-        const hasBigQuery = !!(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GOOGLE_CLOUD_PROJECT);
-        const providers = [
-            {
-                source: "huggingface",
-                display_name: "Hugging Face",
-                available: true,
-                auth_mode: hasHfToken ? "public-or-server-managed" : "public",
-                supported_operations: ["discover", "download", "info"],
-                requires_end_user_key: false,
-                notes: hasHfToken
-                    ? ["Public datasets are open by default. Gated/private repos can be accessed via the server-managed HF token when configured."]
-                    : ["Public datasets work without any user key. Gated/private repos need an operator or user token."],
-            },
-            {
-                source: "openml",
-                display_name: "OpenML",
-                available: true,
-                auth_mode: "public",
-                supported_operations: ["discover", "download", "info"],
-                requires_end_user_key: false,
-                notes: ["OpenML is exposed as a keyless public provider through the gateway."],
-            },
-            {
-                source: "kaggle",
-                display_name: "Kaggle",
-                available: hasKaggle,
-                auth_mode: hasKaggle ? "server-managed" : "not-configured",
-                supported_operations: ["discover", "download", "info"],
-                requires_end_user_key: false,
-                notes: hasKaggle
-                    ? ["Kaggle is available through server-managed credentials. End users do not need to pass their own key."]
-                    : ["Kaggle support exists, but no server-managed credentials are configured yet."],
-            },
-            {
-                source: "dataworld",
-                display_name: "data.world",
-                available: hasDataWorld,
-                auth_mode: hasDataWorld ? "server-managed" : "not-configured",
-                supported_operations: ["discover", "download", "info"],
-                requires_end_user_key: false,
-                notes: hasDataWorld
-                    ? ["data.world is available through server-managed credentials."]
-                    : ["data.world support exists, but no server-managed token is configured yet."],
-            },
-            {
-                source: "s3",
-                display_name: "Amazon S3",
-                available: true,
-                auth_mode: "public-or-server-managed",
-                supported_operations: ["download", "info"],
-                requires_end_user_key: false,
-                notes: ["Supports keyless download of public S3 objects via s3://bucket/key or HTTPS S3 URLs.", "Bucket listing and search are intentionally not exposed."],
-            },
-            {
-                source: "bigquery",
-                display_name: "BigQuery",
-                available: hasBigQuery,
-                auth_mode: hasBigQuery ? "server-managed" : "not-configured",
-                supported_operations: ["info"],
-                requires_end_user_key: false,
-                notes: hasBigQuery
-                    ? ["BigQuery is reserved for operator-managed connectors. Query execution is not implemented in this patch."]
-                    : ["BigQuery is scaffolded in the gateway contract, but no server-managed GCP configuration is present."],
-            },
-        ];
-        return includeUnavailable ? providers : providers.filter(provider => provider.available);
-    }
-    async discover(options) {
-        const query = String(options.query || "").trim();
-        const requestedSource = options.source || "auto";
-        const limit = Math.max(1, Number(options.limit || 10));
-        const publicOnly = options.publicOnly !== false;
-        if (!query) {
-            throw new Error("query is required");
-        }
-        const notes = [];
-        const providers = this.resolveDiscoverSources(requestedSource, publicOnly, notes);
-        const perSourceLimit = Math.max(5, Math.ceil(limit / Math.max(providers.length, 1)) * 2);
-        const allResults = [];
-        for (const provider of providers) {
-            try {
-                const partial = await this.discoverFromSource(provider, query, perSourceLimit);
-                for (const dataset of partial) {
-                    try {
-                        this.deps.metadataStore.saveDataset(dataset);
-                    }
-                    catch {
-                        // best-effort metadata persistence
-                    }
-                    allResults.push(dataset);
-                }
-            }
-            catch (error) {
-                notes.push(`${provider}: ${(error?.message || error || "Unknown provider error").toString()}`);
-            }
-        }
-        const deduped = new Map();
-        for (const dataset of allResults) {
-            deduped.set(`${dataset.source}:${dataset.id}`, dataset);
-        }
-        const results = Array.from(deduped.values())
-            .sort((a, b) => this.rankDataset(b) - this.rankDataset(a))
-            .slice(0, limit);
-        return {
-            query,
-            requested_source: requestedSource,
-            providers_tried: providers,
-            notes,
-            results,
-        };
-    }
-    async download(options) {
-        const requested = String(options.datasetId || "").trim();
-        if (!requested) {
-            throw new Error("dataset_id is required");
-        }
-        const notes = [];
-        const resolved = this.resolveDatasetReference(requested, options.source || "auto");
-        if (resolved.source === "bigquery") {
-            throw new Error("BigQuery gateway support is scaffolded for operator-managed connectors, but query/download execution is not implemented yet.");
-        }
-        if (resolved.source === "s3") {
-            const localPath = await this.downloadPublicS3Object(resolved.datasetId, options.targetDir);
-            return {
-                dataset_id: requested,
-                resolved_source: "s3",
-                local_path: localPath,
-                notes: ["Downloaded via the keyless S3 gateway path."],
-            };
-        }
-        let source = this.toIngestSource(resolved.source);
-        let datasetId = resolved.datasetId;
-        if (!source) {
-            const metadataMatch = this.lookupKnownDataset(requested);
-            const metadataSource = this.toIngestSource(metadataMatch?.source);
-            if (metadataMatch && metadataSource) {
-                source = metadataSource;
-                datasetId = metadataMatch.id;
-            }
-        }
-        if (!source) {
-            const discovery = await this.discover({ query: requested, source: "auto", limit: 1, publicOnly: false });
-            if (discovery.results.length === 0) {
-                throw new Error(`Unable to resolve provider for '${requested}'. Run unified_dataset_api with operation='discover' first or pass an explicit source.`);
-            }
-            const discoveredSource = this.toIngestSource(discovery.results[0].source);
-            if (!discoveredSource) {
-                throw new Error(`Resolved provider '${discovery.results[0].source}' cannot be downloaded through the dataset ingestor.`);
-            }
-            source = discoveredSource;
-            datasetId = discovery.results[0].id;
-            notes.push(`Auto-resolved provider to ${source}.`);
-        }
-        if (source === "kaggle" && !this.deps.dataIngestor.hasKaggleCredentials()) {
-            throw new Error("Kaggle is configured as a gateway source, but no server-managed credentials are available.");
-        }
-        if (source === "dataworld" && !this.deps.hasDataWorldToken()) {
-            throw new Error("data.world is configured as a gateway source, but no server-managed token is available.");
-        }
-        const localPath = await this.deps.dataIngestor.ensureData(datasetId, source, () => undefined);
-        let copiedTo;
-        if (options.targetDir) {
-            copiedTo = this.copyDownloadOutput(localPath, options.targetDir);
-            notes.push(`Copied dataset output to ${copiedTo}.`);
-        }
-        return {
-            dataset_id: datasetId,
-            resolved_source: source,
-            local_path: localPath,
-            copied_to: copiedTo,
-            notes,
-        };
-    }
-    async info(options) {
-        const requested = String(options.datasetId || "").trim();
-        if (!requested) {
-            throw new Error("dataset_id is required");
-        }
-        const resolved = this.resolveDatasetReference(requested, options.source || "auto");
-        const metadataMatch = this.lookupKnownDataset(requested) || (resolved.datasetId !== requested ? this.lookupKnownDataset(resolved.datasetId) : undefined);
-        if (metadataMatch) {
-            return {
-                dataset_id: requested,
-                resolved_source: metadataMatch.source,
-                notes: [],
-                dataset: metadataMatch,
-            };
-        }
-        if (resolved.source === "s3") {
-            return {
-                dataset_id: requested,
-                resolved_source: "s3",
-                notes: ["S3 info is derived from the object URI. Discovery/listing is intentionally not supported."],
-                dataset: {
-                    id: requested,
-                    source: "s3",
-                    uri: this.toS3HttpsUrl(resolved.datasetId),
-                },
-            };
-        }
-        if (resolved.source === "bigquery") {
-            return {
-                dataset_id: requested,
-                resolved_source: "bigquery",
-                notes: ["BigQuery is reserved for operator-managed connectors. Detailed inspection is not implemented in this patch."],
-            };
-        }
-        const discovery = await this.discover({
-            query: resolved.datasetId,
-            source: resolved.source || "auto",
-            limit: 5,
-            publicOnly: options.publicOnly !== false,
-        });
-        const exact = discovery.results.find(dataset => this.matchesDatasetReference(dataset, requested));
-        return {
-            dataset_id: requested,
-            resolved_source: exact?.source,
-            notes: discovery.notes,
-            dataset: exact || discovery.results[0],
-        };
-    }
-    async discoverFromSource(source, query, limit) {
-        switch (source) {
-            case "huggingface":
-                return await new HuggingFaceScraper().scrape(limit, true, await analyzeDatasetQuery(query));
-            case "openml":
-                return await this.deps.openmlSource.discover(query, limit);
-            case "kaggle":
-                return await this.deps.kaggleSource.discover(query, limit);
-            case "dataworld":
-                return await this.deps.dataworldSource.discover(query, limit);
-            case "s3":
-                throw new Error("S3 does not support search/discovery in the unified gateway. Use a direct s3://bucket/key object reference.");
-            case "bigquery":
-                throw new Error("BigQuery discovery is not implemented in the unified gateway.");
-            default:
-                throw new Error(`Unsupported provider: ${source}`);
-        }
-    }
-    resolveDiscoverSources(source, publicOnly, notes) {
-        if (source !== "auto") {
-            if (source === "kaggle" && !this.deps.dataIngestor.hasKaggleCredentials()) {
-                throw new Error("Kaggle requires server-managed credentials and none are configured.");
-            }
-            if (source === "dataworld" && !this.deps.hasDataWorldToken()) {
-                throw new Error("data.world requires a server-managed token and none is configured.");
-            }
-            if (source === "s3" || source === "bigquery") {
-                throw new Error(`${source} does not currently support discover operation through the gateway.`);
-            }
-            return [source];
-        }
-        const providers = ["huggingface", "openml"];
-        if (!publicOnly && this.deps.dataIngestor.hasKaggleCredentials()) {
-            providers.push("kaggle");
-        }
-        else if (!publicOnly) {
-            notes.push("Kaggle skipped because no server-managed credentials are configured.");
-        }
-        if (!publicOnly && this.deps.hasDataWorldToken()) {
-            providers.push("dataworld");
-        }
-        else if (!publicOnly) {
-            notes.push("data.world skipped because no server-managed token is configured.");
-        }
-        return providers;
-    }
-    resolveDatasetReference(datasetId, source) {
-        const trimmed = datasetId.trim();
-        if (source !== "auto") {
-            if (source === "s3") {
-                return { source, datasetId: trimmed };
-            }
-            return { source, datasetId: this.stripSourcePrefix(trimmed, source) };
-        }
-        if (/^s3:\/\//i.test(trimmed) || /^https?:\/\/[^\s]+\.s3[.-][^\s]+/i.test(trimmed) || /^https?:\/\/s3\.[^\s]+amazonaws\.com\//i.test(trimmed)) {
-            return { source: "s3", datasetId: trimmed };
-        }
-        if (/^kaggle:/i.test(trimmed))
-            return { source: "kaggle", datasetId: trimmed.replace(/^kaggle:/i, "") };
-        if (/^(huggingface|hf):/i.test(trimmed))
-            return { source: "huggingface", datasetId: trimmed.replace(/^(huggingface|hf):/i, "") };
-        if (/^openml:/i.test(trimmed))
-            return { source: "openml", datasetId: trimmed.replace(/^openml:/i, "") };
-        if (/^dataworld:/i.test(trimmed))
-            return { source: "dataworld", datasetId: trimmed.replace(/^dataworld:/i, "") };
-        if (/^bigquery:/i.test(trimmed))
-            return { source: "bigquery", datasetId: trimmed.replace(/^bigquery:/i, "") };
-        if (/^\d+$/.test(trimmed))
-            return { source: "openml", datasetId: trimmed };
-        if (trimmed.includes("/") && !trimmed.includes(":"))
-            return { source: "huggingface", datasetId: trimmed };
-        return { datasetId: trimmed };
-    }
-    stripSourcePrefix(datasetId, source) {
-        if (source === "huggingface") {
-            return datasetId.replace(/^(huggingface|hf):/i, "");
-        }
-        return datasetId.replace(new RegExp(`^${source}:`, "i"), "");
-    }
-    lookupKnownDataset(datasetId) {
-        const candidates = new Set([
-            datasetId,
-            datasetId.replace(/^(huggingface|hf|kaggle|openml|dataworld|bigquery):/i, ""),
-        ]);
-        for (const candidate of candidates) {
-            const dataset = this.deps.metadataStore.getDataset(candidate);
-            if (dataset)
-                return dataset;
-        }
-        return undefined;
-    }
-    matchesDatasetReference(dataset, requested) {
-        const normalizedRequested = requested.replace(/^(huggingface|hf|kaggle|openml|dataworld|bigquery):/i, "").toLowerCase();
-        const fullId = `${dataset.source}:${dataset.id}`.toLowerCase();
-        return dataset.id.toLowerCase() === normalizedRequested || fullId === requested.toLowerCase();
-    }
-    toIngestSource(source) {
-        if (source === "huggingface" || source === "openml" || source === "kaggle" || source === "dataworld") {
-            return source;
-        }
-        return undefined;
-    }
-    rankDataset(dataset) {
-        const relevance = Number(dataset.relevance_score || 0) * 1000;
-        const quality = Number(dataset.quality_score || 0) * 100;
-        const downloads = Number(dataset.downloads || 0);
-        return relevance + quality + downloads;
-    }
-    copyDownloadOutput(localPath, targetDir) {
-        const resolvedTargetDir = path.resolve(targetDir);
-        fs.mkdirSync(resolvedTargetDir, { recursive: true });
-        const destination = path.join(resolvedTargetDir, path.basename(localPath));
-        fs.cpSync(localPath, destination, { recursive: true, force: true });
-        return destination;
-    }
-    async downloadPublicS3Object(datasetId, targetDir) {
-        const httpsUrl = this.toS3HttpsUrl(datasetId);
-        const parsed = new URL(httpsUrl);
-        const fileName = path.basename(parsed.pathname) || "s3-object.bin";
-        const outputDir = path.resolve(targetDir || path.join(this.deps.dataRoot, "data", "raw"));
-        fs.mkdirSync(outputDir, { recursive: true });
-        const outputPath = path.join(outputDir, fileName);
-        await this.downloadToFile(httpsUrl, outputPath);
-        this.deps.metadataStore.registerDownload(datasetId, outputPath, "completed", fs.statSync(outputPath).size);
-        return outputPath;
-    }
-    toS3HttpsUrl(datasetId) {
-        if (/^https?:\/\//i.test(datasetId)) {
-            return datasetId;
-        }
-        const match = datasetId.match(/^s3:\/\/([^/]+)\/(.+)$/i);
-        if (!match) {
-            throw new Error("S3 source expects an s3://bucket/key object reference or a direct HTTPS S3 URL.");
-        }
-        const bucket = match[1];
-        const objectKey = match[2].split("/").map(encodeURIComponent).join("/");
-        return `https://${bucket}.s3.amazonaws.com/${objectKey}`;
-    }
-    async downloadToFile(url, destination) {
-        await new Promise((resolve, reject) => {
-            const transport = url.startsWith("https:") ? https : http;
-            const request = transport.get(url, response => {
-                const statusCode = response.statusCode || 0;
-                const location = response.headers.location;
-                if (statusCode >= 300 && statusCode < 400 && location) {
-                    response.resume();
-                    this.downloadToFile(location, destination).then(resolve).catch(reject);
-                    return;
-                }
-                if (statusCode < 200 || statusCode >= 300) {
-                    response.resume();
-                    reject(new Error(`Download failed with status ${statusCode}`));
-                    return;
-                }
-                const file = fs.createWriteStream(destination);
-                response.pipe(file);
-                file.on("finish", () => {
-                    file.close();
-                    resolve();
-                });
-                file.on("error", error => {
-                    try {
-                        file.close();
-                    }
-                    catch {
-                        // no-op
-                    }
-                    reject(error);
-                });
-            });
-            request.on("error", reject);
-        });
-    }
-}