vesper-wizard 2.3.0 → 2.3.2

package/build/python/vesper/core/download_recipe.py

@@ -1,104 +0,0 @@
-from __future__ import annotations
-
-import json
-import re
-from dataclasses import dataclass, asdict
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-
-DEFAULT_RECIPES_DIR = Path.home() / ".vesper" / "recipes"
-
-
-@dataclass
-class DownloadRecipe:
-    dataset_id: str
-    source: str
-    repo_id: str
-    image_column: Optional[str]
-    download_method: str
-    requires_auth: bool
-    estimated_asset_size_gb: float
-    total_images: int
-    fallback_strategy: list[str]
-
-
-def _safe_name(value: str) -> str:
-    return re.sub(r"[^a-zA-Z0-9._-]+", "_", value).strip("_") or "dataset"
-
-
-def build_download_recipe(dataset_info: Dict[str, Any]) -> Dict[str, Any]:
-    dataset_id = str(dataset_info.get("dataset_id") or dataset_info.get("id") or "")
-    source = str(dataset_info.get("source") or "unknown").lower()
-    repo_id = str(dataset_info.get("repo_id") or dataset_info.get("id") or dataset_id)
-
-    image_column = dataset_info.get("image_column")
-    if not image_column:
-        features = dataset_info.get("features") or {}
-        if isinstance(features, dict):
-            for key in features.keys():
-                lower = str(key).lower()
-                if lower in {"image", "images", "img", "img_path", "image_url", "url"}:
-                    image_column = key
-                    break
-
-    download_method = "url_list"
-    if source == "huggingface":
-        download_method = "hf_dataset_image_feature"
-    elif source == "kaggle":
-        download_method = "kaggle_archive"
-    elif source in {"dataworld", "openml"}:
-        download_method = "direct_file_scan"
-
-    requires_auth = bool(dataset_info.get("requires_auth", source in {"kaggle", "dataworld"}))
-
-    total_images = int(dataset_info.get("total_images") or dataset_info.get("total_examples") or 0)
-    if total_images <= 0:
-        total_images = 1000
-
-    estimated_asset_size_gb = round(float(dataset_info.get("estimated_asset_size_gb") or (total_images * 0.0004)), 3)
-
-    fallback_strategy = dataset_info.get("fallback_strategy") or [
-        "scan_archive_for_images",
-        "extract_url_column_and_download",
-        "export_metadata_only_with_actionable_error",
-    ]
-
-    recipe = DownloadRecipe(
-        dataset_id=dataset_id or repo_id,
-        source=source,
-        repo_id=repo_id,
-        image_column=image_column,
-        download_method=download_method,
-        requires_auth=requires_auth,
-        estimated_asset_size_gb=estimated_asset_size_gb,
-        total_images=total_images,
-        fallback_strategy=list(fallback_strategy),
-    )
-
-    return asdict(recipe)
-
-
-def save_recipe(recipe: Dict[str, Any], recipes_dir: Optional[str] = None) -> str:
-    root = Path(recipes_dir) if recipes_dir else DEFAULT_RECIPES_DIR
-    root.mkdir(parents=True, exist_ok=True)
-
-    dataset_id = str(recipe.get("dataset_id") or recipe.get("repo_id") or "dataset")
-    recipe_dir = root / _safe_name(dataset_id)
-    recipe_dir.mkdir(parents=True, exist_ok=True)
-
-    out_path = recipe_dir / "download_recipe.json"
-    out_path.write_text(json.dumps(recipe, indent=2, ensure_ascii=False), encoding="utf-8")
-    return str(out_path)
-
-
-def get_download_recipe(dataset_id: str, recipes_dir: Optional[str] = None) -> Optional[Dict[str, Any]]:
-    root = Path(recipes_dir) if recipes_dir else DEFAULT_RECIPES_DIR
-    path = root / _safe_name(dataset_id) / "download_recipe.json"
-    if not path.exists():
-        return None
-
-    try:
-        return json.loads(path.read_text(encoding="utf-8"))
-    except Exception:
-        return None

package/build/python/worldbank_adapter.py

@@ -1,99 +0,0 @@
-import sys
-import json
-import argparse
-import urllib.request
-import urllib.parse
-from datetime import datetime
-
-# WB API for indicators (Series)
-# Source 2 is World Development Indicators
-WB_API_URL = "https://api.worldbank.org/v2/indicator"
-
-def search_worldbank(query: str, limit: int = 10):
-    """
-    Search World Bank indicators.
-    """
-    try:
-        # The World Bank Indicators API doesn't have a direct "search" parameter for indicators 
-        # that works exactly like a search engine. We fetch a page and filter by query terms.
-        # Alternatively, we could use the 'qterm' on the documents API, but indicators are more tabular.
-        
-        params = {
-            "format": "json",
-            "per_page": 299, # Max per page to search through more indicators
-            "source": 2
-        }
-        
-        query_string = urllib.parse.urlencode(params)
-        url = f"{WB_API_URL}?{query_string}"
-        
-        req = urllib.request.Request(url)
-        with urllib.request.urlopen(req) as response:
-            data = json.load(response)
-        
-        # WB response is [metadata, data_list]
-        if len(data) < 2:
-            return []
-            
-        indicators = data[1]
-        
-        results = []
-        count = 0
-        
-        query_terms = query.lower().split()
-        
-        for ind in indicators:
-            name = ind.get('name', '')
-            source_note = ind.get('sourceNote', '')
-            text = (name + " " + source_note).lower()
-            
-            # Simple keyword matching
-            if all(term in text for term in query_terms):
-                metadata = {
-                    "id": f"wb:{ind.get('id')}", 
-                    "source": "worldbank",
-                    "name": name,
-                    "description": source_note or "No description available.",
-                    "downloads": 1000, # Placeholder (high relevance for WB)
-                    "likes": 100,
-                    "last_updated": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "quality_score": 95, # Institutional data is high quality
-                    "license": {
-                        "id": "cc-by-4.0",
-                        "name": "Creative Commons Attribution 4.0",
-                        "category": "safe",
-                        "usage_restrictions": [],
-                        "warnings": []
-                    },
-                    "tags": [ind.get('source', {}).get('value')] if ind.get('source') else [],
-                    "total_examples": 0, # Time series length varies
-                    "is_safe_source": True,
-                    "is_structured": True,
-                    "metadata_url": f"https://data.worldbank.org/indicator/{ind.get('id')}",
-                    "domain": "economics"
-                }
-                
-                results.append(metadata)
-                count += 1
-                if count >= limit:
-                    break
-                    
-        return results
-
-    except Exception as e:
-        return {"error": str(e)}
-
-def main():
-    parser = argparse.ArgumentParser(description="World Bank Adapter")
-    parser.add_argument("--action", required=True, choices=["search"])
-    parser.add_argument("--query", required=True)
-    parser.add_argument("--limit", type=int, default=10)
-    
-    args = parser.parse_args()
-    
-    if args.action == "search":
-        results = search_worldbank(args.query, args.limit)
-        print(json.dumps(results))
-
-if __name__ == "__main__":
-    main()

package/build/quality/analyzer.js

@@ -1,93 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-export class QualityAnalyzer {
-    cache;
-    pythonPath = "python"; // Assumes python is in PATH
-    scriptPath;
-    constructor(cache, buildDir = process.cwd()) {
-        // buildDir is the directory containing the compiled JS (e.g., build/)
-        // Priority: 
-        // 1. ~/.vesper/python (stable synced location)
-        // 2. build/python (production)
-        // 3. src/python (development)
-        this.cache = cache;
-        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "quality_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "quality_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "quality_engine.py");
-        const scriptPath3 = path.resolve(buildDir, "..", "python", "quality_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else if (fs.existsSync(scriptPath3)) {
-            this.scriptPath = scriptPath3;
-        }
-        else {
-            // Fallback to stable data path, error will be caught during execution
-            this.scriptPath = scriptPath0;
-            console.error(`[QualityAnalyzer] WARNING: Python script not found!`);
-        }
-        // Detect Python command (Windows may use 'py' instead of 'python')
-        if (process.platform === "win32") {
-            this.pythonPath = "py";
-        }
-    }
-    /**
-     * Run quality analysis on a local file (CSV/Parquet/JSON)
-     * @param datasetId Used for caching
-     */
-    async analyze(filePath, datasetId) {
-        if (this.cache && datasetId) {
-            const cached = await this.cache.getReport(datasetId);
-            if (cached) {
-                console.error(`[QualityAnalyzer] Cache hit for ${datasetId}`);
-                return cached;
-            }
-        }
-        const report = await new Promise((resolve, reject) => {
-            const process = spawn(this.pythonPath, [this.scriptPath, filePath]);
-            let stdout = "";
-            let stderr = "";
-            process.stdout.on("data", (data) => {
-                stdout += data.toString();
-            });
-            process.stderr.on("data", (data) => {
-                stderr += data.toString();
-            });
-            process.on("close", (code) => {
-                if (code !== 0) {
-                    const errorDetails = `Quality Analyzer failed (code ${code})
-Command: ${this.pythonPath} ${this.scriptPath} ${filePath}
-Script path exists: ${fs.existsSync(this.scriptPath)}
-Error output: ${stderr}`;
-                    reject(new Error(errorDetails));
-                    return;
-                }
-                try {
-                    const report = JSON.parse(stdout);
-                    if (report.error) {
-                        reject(new Error(report.error));
-                    }
-                    else {
-                        resolve(report);
-                    }
-                }
-                catch (e) {
-                    reject(new Error(`Failed to parse analyzer output: ${stdout}`));
-                }
-            });
-        });
-        if (this.cache && datasetId) {
-            await this.cache.saveReport(datasetId, report);
-        }
-        return report;
-    }
-}

package/build/quality/image-analyzer.js

@@ -1,114 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-export class ImageAnalyzer {
-    pythonPath = "python";
-    scriptPath;
-    constructor(buildDir = process.cwd()) {
-        // buildDir is the directory containing the compiled JS (e.g., build/)
-        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "image_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "image_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "image_engine.py");
-        const scriptPath3 = path.resolve(buildDir, "..", "python", "image_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else if (fs.existsSync(scriptPath3)) {
-            this.scriptPath = scriptPath3;
-        }
-        else {
-            this.scriptPath = scriptPath0;
-        }
-        // Detect Python command (Windows may use 'py' instead of 'python')
-        if (process.platform === "win32") {
-            this.pythonPath = "py";
-        }
-    }
-    /**
-     * Analyze image quality for a single file or a directory
-     */
-    async analyze(inputPath) {
-        return new Promise((resolve, reject) => {
-            const process = spawn(this.pythonPath, [
-                this.scriptPath,
-                inputPath
-            ]);
-            let stdout = "";
-            let stderr = "";
-            process.stdout.on("data", (data) => {
-                stdout += data.toString();
-            });
-            process.stderr.on("data", (data) => {
-                stderr += data.toString();
-            });
-            process.on("error", (err) => {
-                if (err.code === "ENOENT") {
-                    // Python not found - return a graceful failure report
-                    resolve({
-                        total_images: 1,
-                        ok_images: 0,
-                        failed_images: 1,
-                        details: [{
-                                status: "error",
-                                error: "Python not installed or not in PATH. Please install Python to use image analysis features."
-                            }]
-                    });
-                }
-                else {
-                    reject(new Error(`Failed to start image analysis process: ${err.message}`));
-                }
-            });
-            process.on("close", (code) => {
-                if (code !== 0) {
-                    // Handle case where script fails
-                    resolve({
-                        total_images: 1,
-                        ok_images: 0,
-                        failed_images: 1,
-                        details: [{
-                                status: "error",
-                                error: `Image Analyzer process failed (code ${code}): ${stderr || "Unknown error"}`
-                            }]
-                    });
-                    return;
-                }
-                try {
-                    const result = JSON.parse(stdout);
-                    if (result.error) {
-                        resolve({
-                            total_images: 1,
-                            ok_images: 0,
-                            failed_images: 1,
-                            details: [{
-                                    status: "error",
-                                    error: result.error
-                                }]
-                        });
-                    }
-                    else {
-                        resolve(result);
-                    }
-                }
-                catch (e) {
-                    resolve({
-                        total_images: 1,
-                        ok_images: 0,
-                        failed_images: 1,
-                        details: [{
-                                status: "error",
-                                error: `Failed to parse image analyzer output: ${stdout}`
-                            }]
-                    });
-                }
-            });
-        });
-    }
-}

package/build/quality/media-analyzer.js

@@ -1,115 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-export class MediaAnalyzer {
-    pythonPath = "python";
-    scriptPath;
-    constructor(buildDir = process.cwd()) {
-        // buildDir is the directory containing the compiled JS (e.g., build/)
-        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "media_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "media_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "media_engine.py");
-        const scriptPath3 = path.resolve(buildDir, "..", "python", "media_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else if (fs.existsSync(scriptPath3)) {
-            this.scriptPath = scriptPath3;
-        }
-        else {
-            this.scriptPath = scriptPath0;
-        }
-        // Detect Python command (Windows may use 'py' instead of 'python')
-        if (process.platform === "win32") {
-            this.pythonPath = "py";
-        }
-    }
-    /**
-     * Analyze audio/video quality for a single file or a directory
-     */
-    async analyze(inputPath) {
-        return new Promise((resolve, reject) => {
-            const process = spawn(this.pythonPath, [
-                this.scriptPath,
-                inputPath
-            ]);
-            let stdout = "";
-            let stderr = "";
-            process.stdout.on("data", (data) => {
-                stdout += data.toString();
-            });
-            process.stderr.on("data", (data) => {
-                stderr += data.toString();
-            });
-            process.on("error", (err) => {
-                if (err.code === "ENOENT") {
-                    // Python not found - return a graceful failure report
-                    resolve({
-                        total_files: 1,
-                        ok_files: 0,
-                        failed_files: 1,
-                        details: [{
-                                status: "error",
-                                error: "Python not installed or not in PATH. Please install Python to use media analysis features."
-                            }]
-                    });
-                }
-                else {
-                    reject(new Error(`Failed to start media analysis process: ${err.message}`));
-                }
-            });
-            process.on("close", (code) => {
-                if (code !== 0) {
-                    // Handle case where python exists but script fails
-                    resolve({
-                        total_files: 1,
-                        ok_files: 0,
-                        failed_files: 1,
-                        details: [{
-                                status: "error",
-                                error: `Media Analyzer process failed (code ${code}): ${stderr || "Unknown error"}`
-                            }]
-                    });
-                    return;
-                }
-                try {
-                    const result = JSON.parse(stdout);
-                    if (result.error) {
-                        // Return error as part of report instead of rejecting
-                        resolve({
-                            total_files: 1,
-                            ok_files: 0,
-                            failed_files: 1,
-                            details: [{
-                                    status: "error",
-                                    error: result.error
-                                }]
-                        });
-                    }
-                    else {
-                        resolve(result);
-                    }
-                }
-                catch (e) {
-                    resolve({
-                        total_files: 1,
-                        ok_files: 0,
-                        failed_files: 1,
-                        details: [{
-                                status: "error",
-                                error: `Failed to parse media analyzer output: ${stdout}`
-                            }]
-                    });
-                }
-            });
-        });
-    }
-}

package/build/quality/quality-orchestrator.js

@@ -1,162 +0,0 @@
-import fs from "fs";
-import { ImageAnalyzer } from "./image-analyzer.js";
-import { MediaAnalyzer } from "./media-analyzer.js";
-export class QualityOrchestrator {
-    imageAnalyzer;
-    mediaAnalyzer;
-    constructor(projectRoot = process.cwd()) {
-        this.imageAnalyzer = new ImageAnalyzer(projectRoot);
-        this.mediaAnalyzer = new MediaAnalyzer(projectRoot);
-    }
-    /**
-     * Detect modalities present in a dataset directory
-     */
-    detectModalities(datasetPath) {
-        const modalities = [];
-        if (!fs.existsSync(datasetPath)) {
-            return modalities;
-        }
-        const files = fs.readdirSync(datasetPath);
-        // Check for text/tabular data
-        const hasText = files.some(f => /\.(csv|json|parquet|txt)$/i.test(f));
-        if (hasText)
-            modalities.push("text");
-        // Check for images
-        const hasImages = files.some(f => /\.(jpg|jpeg|png|bmp|webp)$/i.test(f));
-        if (hasImages)
-            modalities.push("image");
-        // Check for audio
-        const hasAudio = files.some(f => /\.(wav|mp3|flac|ogg|m4a)$/i.test(f));
-        if (hasAudio)
-            modalities.push("audio");
-        // Check for video
-        const hasVideo = files.some(f => /\.(mp4|avi|mkv|mov|wmv)$/i.test(f));
-        if (hasVideo)
-            modalities.push("video");
-        return modalities;
-    }
-    /**
-     * Generate a unified quality report for a dataset
-     */
-    async generateReport(datasetId, datasetPath, textQuality) {
-        const modalities = this.detectModalities(datasetPath);
-        const report = {
-            dataset_id: datasetId,
-            modalities,
-            overall_quality_score: 0,
-            recommendations: [],
-            generated_at: new Date().toISOString()
-        };
-        let totalScore = 0;
-        let scoreCount = 0;
-        // Text quality (if provided from existing analysis)
-        if (textQuality) {
-            report.text_quality = {
-                row_count: textQuality.row_count || 0,
-                column_count: textQuality.column_count || 0,
-                missing_percentage: textQuality.missing_percentage || 0,
-                duplicate_percentage: textQuality.duplicate_percentage || 0
-            };
-            // Calculate text quality score (0-100)
-            const textScore = Math.max(0, 100 - (report.text_quality.missing_percentage * 2) - (report.text_quality.duplicate_percentage));
-            totalScore += textScore;
-            scoreCount++;
-            if (report.text_quality.missing_percentage > 20) {
-                report.recommendations.push("High missing data detected. Consider imputation or removal.");
-            }
-            if (report.text_quality.duplicate_percentage > 10) {
-                report.recommendations.push("Significant duplicates found. Run deduplication.");
-            }
-        }
-        // Image quality
-        if (modalities.includes("image")) {
-            try {
-                const imageReport = await this.imageAnalyzer.analyze(datasetPath);
-                report.image_quality = {
-                    total_images: imageReport.total_images,
-                    corrupted_count: imageReport.corrupted_count,
-                    avg_resolution: `${Math.round(imageReport.average_width)}x${Math.round(imageReport.average_height)}`,
-                    blurry_percentage: (imageReport.blurry_count / imageReport.total_images) * 100
-                };
-                // Calculate image quality score
-                const corruptionPenalty = (imageReport.corrupted_count / imageReport.total_images) * 50;
-                const blurPenalty = report.image_quality.blurry_percentage * 0.3;
-                const imageScore = Math.max(0, 100 - corruptionPenalty - blurPenalty);
-                totalScore += imageScore;
-                scoreCount++;
-                if (report.image_quality.corrupted_count > 0) {
-                    report.recommendations.push(`Remove ${imageReport.corrupted_count} corrupted images.`);
-                }
-                if (report.image_quality.blurry_percentage > 15) {
-                    report.recommendations.push("High blur detected. Consider filtering blurry images.");
-                }
-            }
-            catch (e) {
-                console.error("Image analysis failed:", e);
-            }
-        }
-        // Audio quality
-        if (modalities.includes("audio")) {
-            try {
-                const audioReport = await this.mediaAnalyzer.analyze(datasetPath);
-                if ('avg_audio_duration' in audioReport) {
-                    const silentFiles = audioReport.details.filter(d => d.status === "ok" && 'is_silent' in d && d.is_silent).length;
-                    const avgSampleRate = audioReport.details
-                        .filter(d => d.status === "ok" && 'sample_rate' in d)
-                        .reduce((sum, d) => sum + (('sample_rate' in d) ? (d.sample_rate || 0) : 0), 0) / audioReport.ok_files;
-                    report.audio_quality = {
-                        total_files: audioReport.total_files,
-                        avg_duration: audioReport.avg_audio_duration || 0,
-                        avg_sample_rate: avgSampleRate,
-                        silent_percentage: (silentFiles / audioReport.total_files) * 100
-                    };
-                    // Calculate audio quality score
-                    const failurePenalty = (audioReport.failed_files / audioReport.total_files) * 50;
-                    const silentPenalty = report.audio_quality.silent_percentage * 0.5;
-                    const audioScore = Math.max(0, 100 - failurePenalty - silentPenalty);
-                    totalScore += audioScore;
-                    scoreCount++;
-                    if (report.audio_quality.silent_percentage > 10) {
-                        report.recommendations.push("High percentage of silent audio files detected.");
-                    }
-                }
-            }
-            catch (e) {
-                console.error("Audio analysis failed:", e);
-            }
-        }
-        // Video quality
-        if (modalities.includes("video")) {
-            try {
-                const videoReport = await this.mediaAnalyzer.analyze(datasetPath);
-                if ('avg_video_duration' in videoReport) {
-                    const highRiskFiles = videoReport.details.filter(d => d.status === "ok" && d.corruption_risk === "high").length;
-                    report.video_quality = {
-                        total_files: videoReport.total_files,
-                        avg_duration: videoReport.avg_video_duration || 0,
-                        avg_fps: videoReport.avg_fps || 0,
-                        corruption_risk_high: highRiskFiles
-                    };
-                    // Calculate video quality score
-                    const failurePenalty = (videoReport.failed_files / videoReport.total_files) * 50;
-                    const corruptionPenalty = (highRiskFiles / videoReport.total_files) * 30;
-                    const videoScore = Math.max(0, 100 - failurePenalty - corruptionPenalty);
-                    totalScore += videoScore;
-                    scoreCount++;
-                    if (highRiskFiles > 0) {
-                        report.recommendations.push(`${highRiskFiles} video files have high corruption risk.`);
-                    }
-                }
-            }
-            catch (e) {
-                console.error("Video analysis failed:", e);
-            }
-        }
-        // Calculate overall quality score
-        report.overall_quality_score = scoreCount > 0 ? Math.round(totalScore / scoreCount) : 0;
-        if (report.recommendations.length === 0) {
-            report.recommendations.push("Dataset quality is good. No major issues detected.");
-        }
-        return report;
-    }
-}