vesper-wizard 2.3.1 → 2.3.3

package/build/ingestion/hf-downloader.js

@@ -1,171 +0,0 @@
-import { listFiles } from "@huggingface/hub";
-import fs from "fs";
-import path from "path";
-import { RobustDownloader } from "../utils/downloader.js";
-export class HFDownloader {
-    hfToken;
-    downloader;
-    constructor(token) {
-        this.hfToken = token || process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
-        this.downloader = new RobustDownloader();
-    }
-    getToken() {
-        return this.hfToken || process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
-    }
-    /**
-     * Finds the most suitable data file in a repository
-     * Returns the relative path within the repo
-     */
-    async findBestFile(repoId) {
-        try {
-            const token = this.getToken();
-            const files = [];
-            const metadataFiles = [];
-            const blacklist = [
-                ".gitattributes",
-                ".gitignore",
-                ".git",
-                "README.md",
-                "LICENSE",
-                "package.json",
-                "requirements.txt",
-                "setup.py"
-            ];
-            const metadataNamePatterns = [
-                /^dataset_infos?\.json$/i,
-                /^dataset_dict\.json$/i,
-                /^state\.json$/i,
-                /^config\.json$/i,
-                /^metadata\.json$/i,
-                /^stats\.json$/i,
-                /^index\.json$/i
-            ];
-            for await (const file of listFiles({
-                repo: { type: "dataset", name: repoId },
-                recursive: true,
-                ...(token ? { accessToken: token } : {})
-            })) {
-                if (file.type === "file") {
-                    const fileName = path.basename(file.path);
-                    const isMetadataJson = metadataNamePatterns.some(p => p.test(fileName));
-                    if (isMetadataJson) {
-                        metadataFiles.push(file.path);
-                    }
-                    if (!blacklist.includes(fileName) && !fileName.startsWith(".") && !isMetadataJson) {
-                        files.push(file.path);
-                    }
-                }
-            }
-            // Priority logic for data scientists
-            const priorities = [
-                /train.*\.parquet$/i,
-                /data.*\.parquet$/i,
-                /.*\.parquet$/i,
-                /train.*\.csv$/i,
-                /data.*\.csv$/i,
-                /.*\.csv$/i,
-                /train.*\.tsv$/i,
-                /data.*\.tsv$/i,
-                /.*\.tsv$/i,
-                /train.*\.txt$/i,
-                /data.*\.txt$/i,
-                /.*\.txt$/i,
-                /.*\.jsonl$/i,
-                /.*\.ndjson$/i,
-                // Keep plain JSON as lowest priority to avoid selecting metadata-like files.
-                /.*\.json$/i
-            ];
-            for (const pattern of priorities) {
-                const match = files.find(f => pattern.test(f));
-                if (match)
-                    return match;
-            }
-            // Strict fallback: Only return the first file if it has a data-like extension
-            const dataExtensions = [".csv", ".parquet", ".jsonl", ".ndjson", ".tsv", ".txt", ".json", ".avro", ".orc"];
-            const fallback = files.find(f => {
-                const ext = path.extname(f).toLowerCase();
-                return dataExtensions.includes(ext);
-            });
-            if (fallback)
-                return fallback;
-            // Last-resort: allow dataset metadata file, then resolve external raw URLs later.
-            const metadataFallback = metadataFiles.find(f => /dataset_infos?\.json$/i.test(path.basename(f)));
-            return metadataFallback || null;
-        }
-        catch (error) {
-            const msg = String(error?.message || error);
-            if (msg.includes("401") || msg.toLowerCase().includes("unauthorized")) {
-                throw new Error(`Authentication required for dataset '${repoId}'. ` +
-                    `This dataset may be gated or private. ` +
-                    `Use the configure_keys tool to set your HF_TOKEN, then retry.`);
-            }
-            if (msg.includes("403") || msg.toLowerCase().includes("forbidden")) {
-                throw new Error(`Access denied for dataset '${repoId}'. ` +
-                    `You may need to accept the dataset's usage agreement on huggingface.co, ` +
-                    `then set HF_TOKEN via configure_keys tool.`);
-            }
-            if (msg.includes("404") || msg.toLowerCase().includes("not found")) {
-                throw new Error(`Dataset '${repoId}' not found on HuggingFace. Check the dataset ID.`);
-            }
-            console.error(`[HF] Failed to list files for ${repoId}:`, msg);
-            return null;
-        }
-    }
-    /**
-     * Downloads a file from HF to local path
-     */
-    async download(repoId, filePath, targetPath, onProgress) {
-        const token = this.getToken();
-        const url = `https://huggingface.co/datasets/${repoId}/resolve/main/${filePath}`;
-        await this.downloader.download(url, targetPath, {
-            headers: token ? { 'Authorization': `Bearer ${token}` } : {},
-            resume: true,
-            onProgress: (bytes, total) => {
-                if (total > 0 && onProgress) {
-                    onProgress(Math.round((bytes / total) * 100));
-                }
-            }
-        });
-    }
-    /**
-     * If downloaded file is dataset metadata (dataset_infos.json), resolve and download a real data URL.
-     * Returns the actual local data path to use.
-     */
-    async resolveExternalDataFromMetadata(localPath, onProgress) {
-        const ext = path.extname(localPath).toLowerCase();
-        if (ext !== ".json") {
-            return localPath;
-        }
-        try {
-            const raw = fs.readFileSync(localPath, "utf-8");
-            const parsed = JSON.parse(raw);
-            const firstConfig = parsed?.default || Object.values(parsed || {})[0];
-            const checksums = firstConfig?.download_checksums;
-            if (!checksums || typeof checksums !== "object") {
-                return localPath;
-            }
-            const candidateUrls = Object.keys(checksums).filter((u) => /^https?:\/\//i.test(u));
-            if (candidateUrls.length === 0) {
-                return localPath;
-            }
-            const preferred = candidateUrls.find(u => /train|data/i.test(path.basename(u))) || candidateUrls[0];
-            const ext = path.extname(preferred).toLowerCase() || ".csv";
-            const resolvedPath = localPath.replace(/\.json$/i, ext);
-            await this.downloader.download(preferred, resolvedPath, {
-                resume: true,
-                onProgress: (bytes, total) => {
-                    if (total > 0 && onProgress) {
-                        onProgress(Math.round((bytes / total) * 100));
-                    }
-                }
-            });
-            if (fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0) {
-                return resolvedPath;
-            }
-            return localPath;
-        }
-        catch {
-            return localPath;
-        }
-    }
-}

package/build/ingestion/ingestor.js

@@ -1,271 +0,0 @@
-import path from "path";
-import fs from "fs";
-import { spawn } from "child_process";
-import { HFDownloader } from "./hf-downloader.js";
-import { KaggleSource } from "../metadata/kaggle-source.js";
-import { OpenMLSource } from "../metadata/openml-source.js";
-import { DataWorldSource } from "../metadata/dataworld-source.js";
-import { SecureKeysManager } from "../config/secure-keys.js";
-export class DataIngestor {
-    projectRoot;
-    store;
-    rawDataDir;
-    hfDownloader;
-    kaggleSource;
-    openmlSource;
-    dataworldSource;
-    secureKeys;
-    constructor(projectRoot, store) {
-        this.projectRoot = projectRoot;
-        this.store = store;
-        this.rawDataDir = path.join(this.projectRoot, "data", "raw");
-        if (!fs.existsSync(this.rawDataDir)) {
-            fs.mkdirSync(this.rawDataDir, { recursive: true });
-        }
-        this.hfDownloader = new HFDownloader();
-        this.kaggleSource = new KaggleSource();
-        this.openmlSource = new OpenMLSource();
-        this.dataworldSource = new DataWorldSource();
-        this.secureKeys = new SecureKeysManager();
-    }
-    /**
-     * Check if Kaggle credentials are available
-     */
-    hasKaggleCredentials() {
-        if (process.env.KAGGLE_USERNAME && process.env.KAGGLE_KEY)
-            return true;
-        const keys = this.secureKeys.getAll();
-        if (keys.kaggle_username && keys.kaggle_key)
-            return true;
-        const kaggleJsonPath = path.join(process.env.HOME || process.env.USERPROFILE || "", ".kaggle", "kaggle.json");
-        return !!(kaggleJsonPath && fs.existsSync(kaggleJsonPath));
-    }
-    /**
-     * Get helpful error message if Kaggle credentials are missing
-     */
-    getKaggleCredentialError() {
-        return "Kaggle support requires API key. Run 'vespermcp config keys' (30 seconds), or provide ~/.kaggle/kaggle.json.";
-    }
-    toSafeDatasetPath(datasetId) {
-        return datasetId.replace(/[:\/]/g, "_");
-    }
-    /**
-     * Ensures a dataset is available locally
-     */
-    async ensureData(datasetId, source, onProgress) {
-        // 1. Check database for existing download
-        const status = this.store.getDownloadStatus(datasetId);
-        if (status && status.status === 'completed' && fs.existsSync(status.local_path)) {
-            return status.local_path;
-        }
-        if (status && status.status === 'downloading') {
-            console.error(`[Ingestor] Dataset ${datasetId} status is 'downloading'. Attempting to resume/concurrently monitor.`);
-            // In a better system we'd use a lock, but for now we let it resume 
-            // the RobustDownloader handles the actual file locking/range logic.
-        }
-        // 2. Trigger source-specific download
-        if (source === "huggingface") {
-            onProgress?.("Discovering data files on HuggingFace Hub...");
-            const remotePath = await this.hfDownloader.findBestFile(datasetId);
-            if (remotePath) {
-                // Direct file download path (repo has raw data files)
-                const ext = path.extname(remotePath).substring(1).toLowerCase() || "csv";
-                const targetPath = this.getTargetPath(datasetId, ext);
-                this.store.registerDownload(datasetId, targetPath, "downloading");
-                try {
-                    await this.hfDownloader.download(datasetId, remotePath, targetPath, (progress) => {
-                        onProgress?.(`Downloading ${path.basename(remotePath)}...`, progress);
-                    });
-                    const resolvedPath = await this.hfDownloader.resolveExternalDataFromMetadata(targetPath, (progress) => {
-                        onProgress?.("Resolving external dataset file...", progress);
-                    });
-                    const stats = fs.statSync(resolvedPath);
-                    this.completeDownload(datasetId, resolvedPath, stats.size);
-                    return resolvedPath;
-                }
-                catch (e) {
-                    const msg = String(e?.message || e);
-                    // If auth error, propagate immediately with helpful message
-                    if (msg.includes("401") || msg.includes("403") || msg.includes("Authentication") || msg.includes("Access denied")) {
-                        this.failDownload(datasetId, msg);
-                        throw e;
-                    }
-                    // For other download errors, try the fallback
-                    onProgress?.(`Direct download failed (${msg}), trying datasets library fallback...`);
-                }
-            }
-            // Fallback: Use Python datasets library to download and convert
-            // This runs when findBestFile returns null OR when direct download fails (non-auth)
-            if (!fs.existsSync(this.getTargetPath(datasetId, "parquet")) || !this.store.getDownloadStatus(datasetId)?.status?.includes("completed")) {
-                onProgress?.("Using HuggingFace datasets library to download...");
-                const targetPath = this.getTargetPath(datasetId, "parquet");
-                this.store.registerDownload(datasetId, targetPath, "downloading");
-                try {
-                    const result = await this.hfDatasetsFallback(datasetId, targetPath, onProgress);
-                    const stats = fs.statSync(result);
-                    this.completeDownload(datasetId, result, stats.size);
-                    return result;
-                }
-                catch (e) {
-                    this.failDownload(datasetId, e.message);
-                    throw e;
-                }
-            }
-        }
-        else if (source === "kaggle") {
-            if (!this.hasKaggleCredentials()) {
-                const errorMsg = this.getKaggleCredentialError();
-                this.failDownload(datasetId, errorMsg);
-                throw new Error(errorMsg);
-            }
-            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
-            this.store.registerDownload(datasetId, targetDir, "downloading");
-            try {
-                onProgress?.("Downloading from Kaggle...");
-                const result = await this.kaggleSource.download(datasetId, targetDir);
-                const stats = fs.statSync(result.local_path);
-                this.completeDownload(datasetId, result.local_path, stats.size);
-                onProgress?.("Kaggle download complete", 100);
-                return result.local_path;
-            }
-            catch (e) {
-                this.failDownload(datasetId, e.message);
-                throw e;
-            }
-        }
-        else if (source === "openml") {
-            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
-            this.store.registerDownload(datasetId, targetDir, "downloading");
-            try {
-                onProgress?.("Downloading from OpenML...");
-                const result = await this.openmlSource.download(datasetId, targetDir);
-                const stats = fs.statSync(result.local_path);
-                this.completeDownload(datasetId, result.local_path, stats.size);
-                onProgress?.("OpenML download complete", 100);
-                return result.local_path;
-            }
-            catch (e) {
-                this.failDownload(datasetId, e.message);
-                throw e;
-            }
-        }
-        else if (source === "dataworld") {
-            const targetDir = path.join(this.rawDataDir, this.toSafeDatasetPath(datasetId));
-            this.store.registerDownload(datasetId, targetDir, "downloading");
-            try {
-                onProgress?.("Downloading from data.world...");
-                const result = await this.dataworldSource.download(datasetId, targetDir);
-                const stats = fs.statSync(result.local_path);
-                this.completeDownload(datasetId, result.local_path, stats.size);
-                onProgress?.("data.world download complete", 100);
-                return result.local_path;
-            }
-            catch (e) {
-                this.failDownload(datasetId, e.message);
-                throw e;
-            }
-        }
-        throw new Error(`Download logic for ${source} not yet implemented`);
-    }
-    /**
-     * Register a successful download
-     */
-    completeDownload(datasetId, actualPath, sizeBytes) {
-        this.store.registerDownload(datasetId, actualPath, 'completed', sizeBytes);
-    }
-    /**
-     * Register a failed download
-     */
-    failDownload(datasetId, error) {
-        const existing = this.store.getDownloadStatus(datasetId);
-        this.store.registerDownload(datasetId, existing?.local_path || "", 'failed', 0, error);
-    }
-    /**
-     * Generates a safe local filename for a dataset ID
-     */
-    getTargetPath(datasetId, extension = "parquet") {
-        const safeId = this.toSafeDatasetPath(datasetId);
-        return path.join(this.rawDataDir, `${safeId}.${extension}`);
-    }
-    /**
-     * Fallback: Use Python `datasets` library to download a HuggingFace dataset
-     * when no raw data files are found in the repo file listing.
-     */
-    async hfDatasetsFallback(datasetId, targetPath, onProgress) {
-        const pyCmd = process.platform === "win32" ? "py" : "python";
-        // Resolve the fallback script path
-        const homeDir = process.env.HOME || process.env.USERPROFILE || this.projectRoot;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptCandidates = [
-            path.resolve(dataRoot, "python", "hf_fallback.py"),
-            path.resolve(this.projectRoot, "python", "hf_fallback.py"),
-            path.resolve(this.projectRoot, "..", "src", "python", "hf_fallback.py"),
-            path.resolve(this.projectRoot, "..", "python", "hf_fallback.py"),
-        ];
-        let scriptPath = scriptCandidates.find(p => fs.existsSync(p));
-        if (!scriptPath) {
-            scriptPath = scriptCandidates[0]; // Will fail with a clear error
-        }
-        const token = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN || undefined;
-        const payload = {
-            repo_id: datasetId,
-            output_path: targetPath,
-            token: token || null,
-            max_rows: 500000,
-        };
-        onProgress?.("Downloading via datasets library (this may take a moment)...", 30);
-        return new Promise((resolve, reject) => {
-            const proc = spawn(pyCmd, [scriptPath, JSON.stringify(payload)], {
-                env: {
-                    ...process.env,
-                    PYTHONUTF8: "1",
-                    PIP_DISABLE_PIP_VERSION_CHECK: "1",
-                },
-            });
-            let stdout = "";
-            let stderr = "";
-            proc.stdout.on("data", (d) => (stdout += d.toString()));
-            proc.stderr.on("data", (d) => {
-                const msg = d.toString();
-                stderr += msg;
-                // Forward progress info
-                if (msg.includes("Downloading") || msg.includes("Loading")) {
-                    onProgress?.(msg.trim().split("\n").pop() || "Downloading...", 50);
-                }
-            });
-            const timer = setTimeout(() => {
-                try {
-                    proc.kill();
-                }
-                catch { /* no-op */ }
-                reject(new Error(`HuggingFace datasets download timed out after 10 minutes for ${datasetId}`));
-            }, 600000); // 10 min timeout
-            proc.on("close", (code) => {
-                clearTimeout(timer);
-                if (code !== 0) {
-                    let errorMsg = stderr || stdout || `Python exited with code ${code}`;
-                    try {
-                        const parsed = JSON.parse(stdout);
-                        if (parsed.error)
-                            errorMsg = parsed.error;
-                    }
-                    catch { /* use stderr */ }
-                    reject(new Error(`HuggingFace datasets fallback failed: ${errorMsg}`));
-                    return;
-                }
-                try {
-                    const result = JSON.parse(stdout);
-                    if (!result.ok) {
-                        reject(new Error(result.error || "Unknown error from HF fallback"));
-                        return;
-                    }
-                    onProgress?.(`Downloaded ${result.rows?.toLocaleString() || "?"} rows (${result.columns?.length || "?"} columns)`, 90);
-                    resolve(result.path);
-                }
-                catch {
-                    reject(new Error(`Failed to parse HF fallback output: ${stdout}`));
-                }
-            });
-        });
-    }
-}

package/build/ingestion/kaggle-downloader.js

@@ -1,102 +0,0 @@
-import fs from "fs";
-import path from "path";
-import AdmZip from "adm-zip";
-import { RobustDownloader } from "../utils/downloader.js";
-export class KaggleDownloader {
-    username;
-    key;
-    downloader;
-    constructor(username, key) {
-        this.username = username || process.env.KAGGLE_USERNAME || "";
-        this.key = key || process.env.KAGGLE_KEY || "";
-        this.downloader = new RobustDownloader();
-    }
-    /**
-     * Check if Kaggle credentials are available
-     */
-    hasCredentials() {
-        return !!(this.username && this.key);
-    }
-    /**
-     * Get a helpful error message if credentials are missing
-     */
-    getCredentialError() {
-        if (!this.username && !this.key) {
-            return "Kaggle credentials missing. Please set KAGGLE_USERNAME and KAGGLE_KEY environment variables.\n" +
-                "Tip: Get your API token from https://www.kaggle.com/settings -> API -> Create New Token\n" +
-                "Alternative: Download the dataset manually and use analyze_quality() on local files.";
-        }
-        if (!this.username) {
-            return "KAGGLE_USERNAME is missing. Please set it in your MCP config or environment variables.";
-        }
-        if (!this.key) {
-            return "KAGGLE_KEY is missing. Please set it in your MCP config or environment variables.";
-        }
-        return "";
-    }
-    /**
-     * Downloads and extracts a Kaggle dataset
-     * returns the path to the primary data file
-     */
-    async download(repoId, targetDir, onProgress) {
-        if (!this.hasCredentials()) {
-            throw new Error(this.getCredentialError());
-        }
-        const auth = Buffer.from(`${this.username}:${this.key}`).toString('base64');
-        const url = `https://www.kaggle.com/api/v1/datasets/download/${repoId}`;
-        // Ensure target directory exists
-        if (!fs.existsSync(targetDir)) {
-            fs.mkdirSync(targetDir, { recursive: true });
-        }
-        const zipPath = path.join(targetDir, "data.zip");
-        await this.downloader.download(url, zipPath, {
-            headers: {
-                'Authorization': `Basic ${auth}`
-            },
-            resume: true,
-            onProgress: (bytes, total) => {
-                if (total > 0 && onProgress) {
-                    onProgress(Math.round((bytes / total) * 100));
-                }
-            }
-        });
-        // Unzip
-        const zip = new AdmZip(zipPath);
-        zip.extractAllTo(targetDir, true);
-        // Find best file
-        const extractedFiles = this.getAllFiles(targetDir);
-        const priorities = [
-            /.*\.parquet$/i,
-            /.*\.csv$/i,
-            /.*\.jsonl$/i,
-            /.*\.json$/i
-        ];
-        let bestFile = null;
-        for (const pattern of priorities) {
-            const match = extractedFiles.find(f => pattern.test(f) && !f.endsWith(".zip"));
-            if (match) {
-                bestFile = match;
-                break;
-            }
-        }
-        // Cleanup ZIP
-        fs.unlinkSync(zipPath);
-        if (!bestFile) {
-            throw new Error("No suitable data files found in Kaggle archive");
-        }
-        return bestFile;
-    }
-    getAllFiles(dir, allFiles = []) {
-        const files = fs.readdirSync(dir);
-        files.forEach(file => {
-            const name = path.join(dir, file);
-            if (fs.statSync(name).isDirectory()) {
-                this.getAllFiles(name, allFiles);
-            }
-            else {
-                allFiles.push(name);
-            }
-        });
-        return allFiles;
-    }
-}

package/build/install/install-service.js

@@ -1,46 +0,0 @@
-import fs from "fs";
-import path from "path";
-export class InstallService {
-    projectRoot;
-    metadataStore;
-    constructor(projectRoot, metadataStore) {
-        this.projectRoot = projectRoot;
-        this.metadataStore = metadataStore;
-    }
-    /**
-     * Installs a prepared dataset file into the ./datasets directory
-     * @param datasetId The ID of the dataset
-     * @param sourcePath The current location of the processed file
-     * @returns The absolute path to the installed file
-     */
-    async install(datasetId, sourcePath, targetDir) {
-        if (!fs.existsSync(sourcePath)) {
-            throw new Error(`Source file not found for installation: ${sourcePath}`);
-        }
-        const dataset = this.metadataStore.getDataset(datasetId);
-        // Create target directory
-        const installLabel = dataset?.name || datasetId;
-        const sanitizedName = installLabel.replace(/[^a-z0-9]/gi, "_").toLowerCase();
-        // If caller specified a target dir, use it directly
-        // Otherwise use the current working directory
-        const installDir = targetDir
-            ? path.resolve(targetDir)
-            : path.resolve(process.cwd(), sanitizedName);
-        console.error(`[InstallService] Resolved install directory: ${installDir}`);
-        if (!fs.existsSync(installDir)) {
-            fs.mkdirSync(installDir, { recursive: true });
-        }
-        const extension = path.extname(sourcePath);
-        const targetFilename = `${sanitizedName}${extension}`;
-        const targetPath = path.join(installDir, targetFilename);
-        // Copy file
-        fs.copyFileSync(sourcePath, targetPath);
-        // Update metadata
-        const absolutePath = path.resolve(targetPath);
-        if (dataset) {
-            this.metadataStore.updateInstallPath(datasetId, absolutePath);
-        }
-        console.error(`[InstallService] Dataset ${datasetId} installed to ${absolutePath}`);
-        return absolutePath;
-    }
-}