vesper-wizard 2.3.1 → 2.3.3

package/build/jobs/manager.js

@@ -1,136 +0,0 @@
-import { v4 as uuidv4 } from "uuid";
-import { EventEmitter } from "events";
-import { QueueService } from "./queue.js";
-import { ObservabilityService } from "../monitoring/observability.js";
-export class JobManager extends EventEmitter {
-    store;
-    static instance;
-    queue;
-    stats;
-    maxConcurrency = 2;
-    activeWorkers = 0;
-    constructor(store, stats) {
-        super();
-        this.store = store;
-        this.queue = new QueueService();
-        this.stats = stats || new ObservabilityService();
-    }
-    static getInstance(store, stats) {
-        if (!JobManager.instance) {
-            JobManager.instance = new JobManager(store, stats);
-        }
-        return JobManager.instance;
-    }
-    /**
-     * Set max parallel workers
-     */
-    setConcurrency(count) {
-        this.maxConcurrency = count;
-    }
-    /**
-     * Create a new job and add it to the priority queue
-     */
-    createJob(type, priority = 0, metadata, maxAttempts = 3) {
-        const now = new Date().toISOString();
-        const job = {
-            id: uuidv4(),
-            type,
-            status: "pending",
-            priority,
-            progress: 0,
-            status_text: "Job created",
-            attempts: 0,
-            max_attempts: maxAttempts,
-            created_at: now,
-            updated_at: now,
-            metadata: metadata ? JSON.stringify(metadata) : undefined
-        };
-        this.store.saveJob(job);
-        this.queue.enqueue(job);
-        this.emit("jobCreated", job);
-        // Start processing background queue
-        this.processQueue();
-        return job;
-    }
-    /**
-     * Background loop to process queued jobs
-     */
-    async processQueue() {
-        if (this.activeWorkers >= this.maxConcurrency)
-            return;
-        const job = this.queue.dequeue();
-        if (!job)
-            return;
-        this.activeWorkers++;
-        this.updateJob(job.id, { status: "running", status_text: "Picked up by worker" });
-        const startTime = Date.now();
-        const listeners = this.listenerCount("processJob");
-        console.error(`[JobManager] Emitting processJob for ${job.id}. Active listeners: ${listeners}`);
-        // In a real system, we'd have a registry of handlers for each JobType.
-        // For now, we emit an event so the orchestrator can run it.
-        this.emit("processJob", job, async (jobExecutionTask) => {
-            console.error(`[JobManager] Wrapper received jobExecutionTask: ${typeof jobExecutionTask}`);
-            if (typeof jobExecutionTask !== 'function') {
-                console.error(`[JobManager] Error: jobExecutionTask is NOT a function! It is: ${typeof jobExecutionTask}`);
-                this.updateJob(job.id, {
-                    status: "failed",
-                    status_text: "Internal error: jobExecutionTask is not a function"
-                });
-                return;
-            }
-            try {
-                const resultUrl = await jobExecutionTask();
-                const duration = Date.now() - startTime;
-                this.updateJob(job.id, {
-                    status: "completed",
-                    progress: 100,
-                    status_text: "Completed successfully",
-                    result_url: resultUrl || undefined
-                });
-                this.stats.recordJobSuccess(job.type, duration);
-            }
-            catch (error) {
-                if (this.queue.shouldRetry(job)) {
-                    this.updateJob(job.id, {
-                        status: "retrying",
-                        status_text: `Failed: ${error.message}. Retrying...`
-                    });
-                }
-                else {
-                    this.updateJob(job.id, {
-                        status: "failed",
-                        status_text: "Max retries exceeded",
-                        error: error.message || String(error)
-                    });
-                    this.stats.recordJobFailure(job.type, error);
-                }
-            }
-            finally {
-                this.queue.finalize(job.id);
-                this.activeWorkers--;
-                this.processQueue(); // Look for next job
-            }
-        });
-        // Try to start another worker if capacity allows
-        this.processQueue();
-    }
-    /**
-     * Update job status and progress
-     */
-    updateJob(id, updates) {
-        const job = this.store.getJob(id);
-        if (!job)
-            return;
-        // Correctly handle metadata update if it's an object
-        if (updates.metadata && typeof updates.metadata !== 'string') {
-            updates.metadata = JSON.stringify(updates.metadata);
-        }
-        const updatedJob = {
-            ...job,
-            ...updates,
-            updated_at: new Date().toISOString()
-        };
-        this.store.saveJob(updatedJob);
-        this.emit("jobUpdated", updatedJob);
-    }
-}

package/build/jobs/queue.js

@@ -1,59 +0,0 @@
-export class QueueService {
-    queue = [];
-    running = new Map();
-    /**
-     * Adds a job to the queue, sorted by priority.
-     */
-    enqueue(job) {
-        job.status = "queued";
-        this.queue.push(job);
-        this.sortQueue();
-    }
-    /**
-     * Gets the next job from the queue.
-     */
-    dequeue() {
-        const job = this.queue.shift();
-        if (job) {
-            job.status = "running";
-            this.running.set(job.id, job);
-        }
-        return job;
-    }
-    /**
-     * Mark a job as finished (completed or failed permanently).
-     */
-    finalize(id) {
-        this.running.delete(id);
-    }
-    /**
-     * Handles retry logic with exponential backoff (simulated).
-     */
-    shouldRetry(job) {
-        if (job.attempts < job.max_attempts) {
-            job.attempts++;
-            job.status = "retrying";
-            // In a real system, we'd schedule this for later.
-            // For now, we'll re-enqueue it immediately.
-            this.enqueue(job);
-            return true;
-        }
-        return false;
-    }
-    getStats() {
-        return {
-            pending: this.queue.length,
-            running: this.running.size,
-            total: this.queue.length + this.running.size
-        };
-    }
-    sortQueue() {
-        // Priority high to low, then created_at old to new
-        this.queue.sort((a, b) => {
-            if (b.priority !== a.priority) {
-                return b.priority - a.priority;
-            }
-            return new Date(a.created_at).getTime() - new Date(b.created_at).getTime();
-        });
-    }
-}

package/build/jobs/types.js

@@ -1 +0,0 @@
-export {};

package/build/lib/supabase.js

@@ -1,3 +0,0 @@
-import { createClient } from '@supabase/supabase-js';
-export const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY // for MCP, use service_role, not anon
-);

package/build/metadata/dataworld-source.js

@@ -1,89 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-import os from "os";
-export class DataWorldSource {
-    pythonPath = "python";
-    scriptPath;
-    constructor(buildDir = process.cwd()) {
-        const homeDir = os.homedir() || process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "dataworld_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "dataworld_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "dataworld_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else {
-            this.scriptPath = scriptPath0;
-        }
-        if (process.platform === "win32") {
-            const venvPy = path.resolve(buildDir, ".venv", "Scripts", "python.exe");
-            if (fs.existsSync(venvPy)) {
-                this.pythonPath = venvPy;
-            }
-            else {
-                this.pythonPath = "py";
-            }
-        }
-        else {
-            const venvPy = path.resolve(buildDir, ".venv", "bin", "python");
-            if (fs.existsSync(venvPy)) {
-                this.pythonPath = venvPy;
-            }
-        }
-    }
-    async discover(query, limit = 20) {
-        const result = await this.run(["discover", query, String(limit)]);
-        if (!result.ok) {
-            throw new Error(result.error || "data.world discover failed");
-        }
-        return (result.results || []);
-    }
-    async download(datasetRef, targetDir) {
-        const args = ["download", datasetRef];
-        if (targetDir)
-            args.push(targetDir);
-        const result = await this.run(args);
-        if (!result.ok) {
-            throw new Error(result.error || "data.world download failed");
-        }
-        return {
-            local_path: result.local_path,
-            target_dir: result.target_dir,
-        };
-    }
-    run(args) {
-        return new Promise((resolve, reject) => {
-            const proc = spawn(this.pythonPath, [this.scriptPath, ...args], {
-                env: process.env
-            });
-            let stdout = "";
-            let stderr = "";
-            proc.stdout.on("data", (data) => {
-                stdout += data.toString();
-            });
-            proc.stderr.on("data", (data) => {
-                stderr += data.toString();
-            });
-            proc.on("close", (code) => {
-                if (code !== 0) {
-                    return reject(new Error(`data.world engine exited with code ${code}: ${stderr}`));
-                }
-                try {
-                    const parsed = JSON.parse(stdout.trim());
-                    resolve(parsed);
-                }
-                catch (e) {
-                    reject(new Error(`Failed to parse data.world engine output: ${stdout}`));
-                }
-            });
-        });
-    }
-}

package/build/metadata/domain.js

@@ -1,147 +0,0 @@
-const DOMAIN_KEYWORDS = [
-    {
-        domain: "medicine",
-        keywords: ["medical", "medicine", "clinical", "diagnosis", "patient", "disease", "symptom", "treatment", "drug", "pharmaceutical", "healthcare", "hospital", "doctor", "physician", "medical imaging", "radiology", "pathology", "oncology", "cardiology"],
-        weight: 10
-    },
-    {
-        domain: "healthcare",
-        keywords: ["health", "healthcare", "wellness", "public health", "epidemiology", "biomedical", "health data", "medical records", "ehr", "electronic health"],
-        weight: 8
-    },
-    {
-        domain: "security",
-        keywords: ["security", "cybersecurity", "malware", "vulnerability", "threat", "attack", "defense", "encryption", "privacy", "authentication", "authorization", "penetration testing", "intrusion"],
-        weight: 10
-    },
-    {
-        domain: "finance",
-        keywords: ["finance", "financial", "banking", "trading", "stock", "market", "investment", "credit", "loan", "mortgage", "cryptocurrency", "bitcoin", "blockchain", "accounting"],
-        weight: 9
-    },
-    {
-        domain: "education",
-        keywords: ["education", "learning", "student", "academic", "curriculum", "pedagogy", "teaching", "school", "university", "course", "tutorial"],
-        weight: 8
-    },
-    {
-        domain: "science",
-        keywords: ["scientific", "research", "experiment", "laboratory", "physics", "chemistry", "biology", "astronomy", "geology", "environmental"],
-        weight: 7
-    },
-    {
-        domain: "technology",
-        keywords: ["software", "programming", "code", "algorithm", "system", "application", "development", "engineering", "technical"],
-        weight: 5
-    },
-    {
-        domain: "social",
-        keywords: ["social", "society", "community", "demographic", "census", "population", "sociology", "anthropology", "culture"],
-        weight: 6
-    },
-    {
-        domain: "legal",
-        keywords: ["legal", "law", "court", "judicial", "litigation", "contract", "regulation", "compliance", "legislation"],
-        weight: 9
-    },
-    {
-        domain: "business",
-        keywords: ["business", "corporate", "enterprise", "company", "organization", "management", "marketing", "sales", "customer"],
-        weight: 6
-    },
-    {
-        domain: "multimedia",
-        keywords: ["image", "video", "audio", "multimedia", "visual", "photography", "music", "sound"],
-        weight: 7
-    },
-    {
-        domain: "nlp",
-        keywords: ["natural language", "nlp", "text", "language model", "translation", "sentiment", "text classification", "named entity", "ner", "question answering", "qa"],
-        weight: 8
-    },
-    {
-        domain: "computer-vision",
-        keywords: ["computer vision", "cv", "image classification", "object detection", "segmentation", "visual", "optical", "camera", "facial recognition"],
-        weight: 8
-    }
-];
-export function classifyDomain(description, tags, name, task) {
-    const text = `${description} ${name} ${tags.join(" ")} ${task || ""}`.toLowerCase();
-    const scores = new Map();
-    // Initialize scores
-    DOMAIN_KEYWORDS.forEach(dk => {
-        scores.set(dk.domain, 0);
-    });
-    // Score each domain
-    DOMAIN_KEYWORDS.forEach(dk => {
-        let matches = 0;
-        dk.keywords.forEach(keyword => {
-            if (text.includes(keyword.toLowerCase())) {
-                matches++;
-            }
-        });
-        if (matches > 0) {
-            const currentScore = scores.get(dk.domain) || 0;
-            scores.set(dk.domain, currentScore + (matches * dk.weight));
-        }
-    });
-    // Find domain with highest score
-    let maxScore = 0;
-    let bestDomain = "unknown";
-    scores.forEach((score, domain) => {
-        if (score > maxScore) {
-            maxScore = score;
-            bestDomain = domain;
-        }
-    });
-    // If no strong match, check task type for hints
-    if (maxScore === 0 && task) {
-        const taskLower = task.toLowerCase();
-        if (taskLower.includes("image") || taskLower.includes("vision") || taskLower.includes("detection")) {
-            return "computer-vision";
-        }
-        if (taskLower.includes("text") || taskLower.includes("language") || taskLower.includes("nlp")) {
-            return "nlp";
-        }
-    }
-    // Require minimum score threshold to avoid false positives
-    if (maxScore < 5) {
-        return "general";
-    }
-    // Special handling: medicine and healthcare are related
-    if (bestDomain === "medicine" || bestDomain === "healthcare") {
-        const medicineScore = scores.get("medicine") || 0;
-        const healthcareScore = scores.get("healthcare") || 0;
-        if (medicineScore > healthcareScore * 1.5) {
-            return "medicine";
-        }
-        else if (healthcareScore > medicineScore * 1.5) {
-            return "healthcare";
-        }
-        else {
-            // If both are high, prefer medicine for medical-specific terms
-            return (medicineScore >= healthcareScore ? "medicine" : "healthcare");
-        }
-    }
-    return bestDomain;
-}
-export function getDomainDisplayName(domain) {
-    const names = {
-        "medicine": "Medicine",
-        "healthcare": "Healthcare",
-        "security": "Security",
-        "finance": "Finance",
-        "education": "Education",
-        "science": "Science",
-        "technology": "Technology",
-        "social": "Social",
-        "legal": "Legal",
-        "business": "Business",
-        "multimedia": "Multimedia",
-        "nlp": "Natural Language Processing",
-        "computer-vision": "Computer Vision",
-        "general": "General",
-        "unknown": "Unknown"
-    };
-    return names[domain] || "Unknown";
-}

package/build/metadata/github-scraper.js

@@ -1,47 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-const PYTHON_SCRIPT_PATH = path.resolve("src", "python", "github_adapter.py");
-export class GitHubScraper {
-    /**
-     * Search GitHub repositories for datasets using the Python adapter
-     */
-    async scrape(query, limit = 10) {
-        return new Promise((resolve, reject) => {
-            const pythonProcess = spawn("python", [
-                PYTHON_SCRIPT_PATH,
-                "--action", "search",
-                "--query", query,
-                "--limit", String(limit)
-            ]);
-            let output = "";
-            let errorOutput = "";
-            pythonProcess.stdout.on("data", (data) => {
-                output += data.toString();
-            });
-            pythonProcess.stderr.on("data", (data) => {
-                errorOutput += data.toString();
-            });
-            pythonProcess.on("close", (code) => {
-                if (code !== 0) {
-                    console.error(`[GitHubScraper] Process exited with code ${code}: ${errorOutput}`);
-                    resolve([]);
-                    return;
-                }
-                try {
-                    const results = JSON.parse(output);
-                    if (results.error) {
-                        console.error(`[GitHubScraper] Internal error: ${results.error}`);
-                        resolve([]);
-                    }
-                    else {
-                        resolve(results);
-                    }
-                }
-                catch (e) {
-                    console.error(`[GitHubScraper] JSON parse error: ${e.message}. Output: ${output.substring(0, 100)}...`);
-                    resolve([]);
-                }
-            });
-        });
-    }
-}

package/build/metadata/institutional-scrapers.js

@@ -1,49 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-const WB_SCRIPT_PATH = path.resolve("src", "python", "worldbank_adapter.py");
-const NASA_SCRIPT_PATH = path.resolve("src", "python", "nasa_adapter.py");
-export class WorldBankScraper {
-    async scrape(query, limit = 10) {
-        return runAdapter(WB_SCRIPT_PATH, query, limit);
-    }
-}
-export class NASAScraper {
-    async scrape(query, limit = 10) {
-        return runAdapter(NASA_SCRIPT_PATH, query, limit);
-    }
-}
-async function runAdapter(scriptPath, query, limit) {
-    return new Promise((resolve) => {
-        const pythonProcess = spawn("python", [
-            scriptPath,
-            "--action", "search",
-            "--query", query,
-            "--limit", String(limit)
-        ]);
-        let output = "";
-        let errorOutput = "";
-        pythonProcess.stdout.on("data", (data) => { output += data.toString(); });
-        pythonProcess.stderr.on("data", (data) => { errorOutput += data.toString(); });
-        pythonProcess.on("close", (code) => {
-            if (code !== 0) {
-                console.error(`[Adapter] ${path.basename(scriptPath)} exited with code ${code}: ${errorOutput}`);
-                resolve([]);
-                return;
-            }
-            try {
-                const results = JSON.parse(output);
-                if (results.error) {
-                    console.error(`[Adapter] ${path.basename(scriptPath)} error: ${results.error}`);
-                    resolve([]);
-                }
-                else {
-                    resolve(results);
-                }
-            }
-            catch (e) {
-                console.error(`[Adapter] ${path.basename(scriptPath)} JSON error: ${e.message}`);
-                resolve([]);
-            }
-        });
-    });
-}