vesper-wizard 2.3.0 → 2.3.2

package/build/scripts/test-quality.js

@@ -1,43 +0,0 @@
-import { QualityAnalyzer } from "../quality/analyzer.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Quality Analysis Test ===\n");
-    const analyzer = new QualityAnalyzer();
-    const testFile = path.join(process.cwd(), "test_data.csv");
-    // 1. Create a dummy CSV with some issues
-    const csvContent = `id,age,score,category
-1,25,90.5,A
-2,30,88.0,B
-3,,75.0,A
-4,25,90.5,A
-5,150,12.0,C
-6,40,99.9,B`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created test file: ${testFile}`);
-    console.log("Data contains: 1 missing value, 1 duplicate row, 1 outlier (age 150)\n");
-    try {
-        console.log("Running analyzer...");
-        const report = await analyzer.analyze(testFile);
-        console.log("\n=== Analysis Report ===");
-        console.log(`Rows: ${report.row_count}, Columns: ${report.column_count}`);
-        console.log(`Duplicates: ${report.duplicate_rows} (${report.duplicate_percentage.toFixed(1)}%)`);
-        console.log("\nColumn Details:");
-        for (const col of report.columns) {
-            let details = "";
-            if (col.missing_count > 0)
-                details += `Missing: ${col.missing_count} `;
-            if (col.distribution)
-                details += `Mean: ${col.distribution.mean.toFixed(1)} `;
-            console.log(`  - ${col.name} (${col.type}): ${details}`);
-        }
-        console.log("\nWarnings:", report.warnings);
-        // Cleanup
-        fs.unlinkSync(testFile);
-        console.log("\nTest passed!");
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-robust-ingestion.js

@@ -1,41 +0,0 @@
-import { RobustDownloader } from "../utils/downloader.js";
-import fs from "fs";
-import path from "path";
-async function runTest() {
-    console.log("--- Testing Robust Ingestion (Resume Support) ---");
-    const downloader = new RobustDownloader();
-    const testDir = path.resolve("./data/test-ingestion");
-    if (!fs.existsSync(testDir))
-        fs.mkdirSync(testDir, { recursive: true });
-    const targetPath = path.join(testDir, "test-file.txt");
-    const testUrl = "https://raw.githubusercontent.com/google/guava/master/LICENSE";
-    // 1. Pre-fill with partial garbage content
-    console.log("Simulating a partial download (first 50 bytes)...");
-    fs.writeFileSync(targetPath, "This is some existing content that should be ignored by Range 0 but appended if we resume.");
-    const initialSize = fs.statSync(targetPath).size;
-    console.log(`Initial file size: ${initialSize} bytes`);
-    // 2. Attempt Download with Resume
-    console.log("Starting downloader with resume=true...");
-    try {
-        await downloader.download(testUrl, targetPath, {
-            resume: true,
-            onProgress: (bytes, total) => {
-                console.log(`Progress: ${bytes}/${total} bytes`);
-            }
-        });
-        const finalSize = fs.statSync(targetPath).size;
-        console.log(`Final file size: ${finalSize} bytes`);
-        if (finalSize > initialSize) {
-            console.log("\nVERIFICATION_STATUS: ✅ PASS (Resumed and finished)");
-        }
-        else {
-            console.log("\nVERIFICATION_STATUS: ❌ FAIL (File did not grow)");
-        }
-    }
-    catch (e) {
-        console.error("Download failed:", e.message);
-        // Note: github raw might not support ranges perfectly, but the logic check stands
-        console.log("\nVERIFICATION_STATUS: ❌ FAIL");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-schema.js

@@ -1,45 +0,0 @@
-import { QualityAnalyzer } from "../quality/analyzer.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Schema Validation Test ===\n");
-    const analyzer = new QualityAnalyzer();
-    const testFile = path.join(process.cwd(), "test_schema.csv");
-    // Create a CSV with schema issues
-    // 'price': stored as string (should be numeric)
-    // 'mixed': completely broken mixed types
-    const csvContent = `id,price,mixed,valid_str
-1,"100.5",10,A
-2,"50.2","text",B
-3,"75.0",20,A
-4,"200",30,C
-5,"10.99","oops",B`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created test file: ${testFile}`);
-    console.log("Expectations:");
-    console.log("  - 'price' should be detected as Numeric (Stored as String)");
-    console.log("  - 'mixed' should be flagged as Mixed Type");
-    try {
-        console.log("\nRunning analyzer...");
-        const report = await analyzer.analyze(testFile);
-        console.log("\n=== Schema Report ===");
-        for (const col of report.columns) {
-            console.log(`Column: ${col.name}`);
-            console.log(`  Type: ${col.type}`);
-            console.log(`  Inferred: ${col.inferred_type}`);
-            if (col.is_mixed_type)
-                console.log(`  [!] Mixed Type Detected`);
-            if (col.inferred_type !== col.type && !col.type.includes(col.inferred_type)) {
-                console.log(`  [!] Type Mismatch Detected`);
-            }
-        }
-        console.log("\nSchema Warnings:", report.schema_warnings);
-        // Cleanup
-        fs.unlinkSync(testFile);
-        console.log("\nTest passed!");
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-split-validation.js

@@ -1,40 +0,0 @@
-import { DataSplitter } from "../splitting/splitter.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Split Validation Test ===\n");
-    const splitter = new DataSplitter();
-    const trainPath = path.join(process.cwd(), "mock_train.csv");
-    const valPath = path.join(process.cwd(), "mock_val.csv");
-    const testPath = path.join(process.cwd(), "mock_test.csv");
-    // 1. Create splits with LEAKAGE (ID 1 exists in train and val)
-    // And DRIFT (Train is all 'A', Val is all 'B')
-    fs.writeFileSync(trainPath, "id,label\n1,A\n2,A\n3,A");
-    fs.writeFileSync(valPath, "id,label\n1,B\n4,B\n5,B"); // ID 1 is leak
-    fs.writeFileSync(testPath, "id,label\n6,A\n7,B");
-    try {
-        console.log("Running validator...");
-        const report = await splitter.validate({ train: trainPath, val: valPath, test: testPath }, { id_column: "id", target_column: "label" });
-        console.log("\n=== Validation Report ===");
-        console.log(`Leakage Detected: ${report.leakage_detected} (Count: ${report.leakage_count})`);
-        console.log(`Distribution Mismatch: ${report.distribution_mismatch}`);
-        console.log("Warnings:");
-        report.warnings.forEach(w => console.log(`  - ${w}`));
-        // Assertions
-        if (report.leakage_detected && report.distribution_mismatch) {
-            console.log("\nPASS: Detected both Leakage and Drift.");
-        }
-        else {
-            console.error("\nFAIL: Missed issues.");
-        }
-    }
-    catch (e) {
-        console.error("Test failed:", e);
-    }
-    finally {
-        // Cleanup
-        [trainPath, valPath, testPath].forEach(p => { if (fs.existsSync(p))
-            fs.unlinkSync(p); });
-    }
-}
-main().catch(console.error);

package/build/scripts/test-splitter.js

@@ -1,93 +0,0 @@
-import { DataSplitter } from "../splitting/splitter.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Data Splitter Test ===\n");
-    const splitter = new DataSplitter();
-    const testFile = path.join(process.cwd(), "test_splitting.csv");
-    // Create Dummy Data
-    // - id: 1..100
-    // - label: A (50), B (50) for stratification
-    // - date: sequential dates for time split
-    let csvContent = "id,label,date\n";
-    for (let i = 0; i < 50; i++)
-        csvContent += `${i},A,2023-01-${(i % 30) + 1}\n`;
-    for (let i = 50; i < 100; i++)
-        csvContent += `${i},B,2023-02-${(i % 28) + 1}\n`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created test file: ${testFile}`);
-    console.log("Rows: 100 (50 A, 50 B)");
-    // Test 1: Random Split (80/10/10)
-    console.log("\n--- Test 1: Random Split (80/10/10) ---");
-    const config1 = {
-        type: "random",
-        ratios: { train: 0.8, val: 0.1, test: 0.1, holdout: 0 },
-        shuffle: true,
-        random_seed: 42
-    };
-    try {
-        const result1 = await splitter.split(testFile, config1);
-        console.log("Stats:", result1.stats);
-        if (result1.stats.train_rows === 80 && result1.stats.val_rows === 10 && result1.stats.test_rows === 10) {
-            console.log("PASS: Ratios preserved");
-        }
-        else {
-            console.error("FAIL: Incorrect ratios");
-        }
-    }
-    catch (e) {
-        console.error(e);
-    }
-    // Test 2: Stratified Split (60/20/20) on 'label'
-    console.log("\n--- Test 2: Stratified Split (60/20/20) ---");
-    const config2 = {
-        type: "stratified",
-        ratios: { train: 0.6, val: 0.2, test: 0.2, holdout: 0 },
-        target_column: "label",
-        random_seed: 42
-    };
-    try {
-        const result2 = await splitter.split(testFile, config2);
-        console.log("Stats:", result2.stats);
-        if (result2.stats.train_rows === 60 && result2.stats.val_rows === 20 && result2.stats.test_rows === 20) {
-            console.log("PASS: Ratios preserved");
-            // In a deeper test we would verify the class distribution too, but simple count is good for now
-        }
-        else {
-            console.error("FAIL: Incorrect ratios");
-        }
-    }
-    catch (e) {
-        console.error(e);
-    }
-    // Test 3: Time Split (70/15/15) on 'date'
-    console.log("\n--- Test 3: Time Split (70/15/15) ---");
-    const config3 = {
-        type: "time",
-        ratios: { train: 0.7, val: 0.15, test: 0.15, holdout: 0 },
-        time_column: "date"
-    };
-    try {
-        const result3 = await splitter.split(testFile, config3);
-        console.log("Stats:", result3.stats);
-        // Clean up generated files
-        fs.unlinkSync(result3.paths.train);
-        fs.unlinkSync(result3.paths.val);
-        fs.unlinkSync(result3.paths.test);
-        // Clean up previous test files too (using known naming convention)
-        const base = testFile.replace(".csv", "");
-        if (fs.existsSync(base + "_train.csv"))
-            fs.unlinkSync(base + "_train.csv");
-        if (fs.existsSync(base + "_val.csv"))
-            fs.unlinkSync(base + "_val.csv");
-        if (fs.existsSync(base + "_test.csv"))
-            fs.unlinkSync(base + "_test.csv");
-        console.log("PASS: Execution successful");
-    }
-    catch (e) {
-        console.error(e);
-    }
-    // Cleanup input
-    fs.unlinkSync(testFile);
-}
-main().catch(console.error);

package/build/scripts/test-target-detector.js

@@ -1,29 +0,0 @@
-import { TargetDetector } from "../preparation/target-detector.js";
-import path from "path";
-async function testDetector() {
-    // 1. Test existing build dir
-    const detector = new TargetDetector(path.join(process.cwd(), "build"));
-    // 2. Create a dummy CSV for testing
-    const testFile = path.join(process.cwd(), "test_target.csv");
-    const fs = (await import("fs")).default;
-    // Test Case 1: SalePrice (Regression)
-    console.log("--- Test Case 1: SalePrice ---");
-    fs.writeFileSync(testFile, "id,feature1,feature2,SalePrice\n1,10,20,100000\n2,11,21,120000\n3,12,22,110000");
-    let result = await detector.detectTarget(testFile);
-    console.log("Detection:", result.target_column, result.confidence);
-    if (result.target_column) {
-        let val = await detector.validateTarget(testFile, result.target_column);
-        console.log("Validation:", val.problem_type, val.valid);
-    }
-    // Test Case 2: diagnosis (Classification)
-    console.log("\n--- Test Case 2: diagnosis ---");
-    fs.writeFileSync(testFile, "id,age,diagnosis\n1,50,M\n2,60,B\n3,45,M");
-    result = await detector.detectTarget(testFile);
-    console.log("Detection:", result.target_column, result.confidence);
-    if (result.target_column) {
-        let val = await detector.validateTarget(testFile, result.target_column);
-        console.log("Validation:", val.problem_type, val.valid);
-    }
-    fs.unlinkSync(testFile);
-}
-testDetector().catch(console.error);

package/build/scripts/test-uci.js

@@ -1,27 +0,0 @@
-import { UCIScraper } from "../metadata/uci-scraper.js";
-async function runTest() {
-    console.log("--- Testing UCI integration ---");
-    const scraper = new UCIScraper();
-    const query = "iris";
-    console.log(`Searching UCI for: "${query}"...`);
-    const results = await scraper.scrape(query, 5);
-    console.log(`Found ${results.length} datasets.`);
-    if (results.length > 0) {
-        console.log("✅ UCI Scraper returned results.");
-        console.log("Sample Result:");
-        console.log(JSON.stringify(results[0], null, 2));
-        // Validation
-        const sample = results[0];
-        if (sample.id.startsWith("uci:") && sample.source === "uci") {
-            console.log("✅ Metadata schema validation passed.");
-        }
-        else {
-            console.error("❌ Metadata schema validation failed.");
-        }
-    }
-    else {
-        console.error("❌ No results found. Ensure python and ucimlrepo are installed.");
-    }
-    console.log("--- Test Complete ---");
-}
-runTest().catch(console.error);

package/build/scripts/test-unified-quality.js

@@ -1,86 +0,0 @@
-import { QualityOrchestrator } from "../quality/quality-orchestrator.js";
-import path from "path";
-import fs from "fs";
-import { execSync } from "child_process";
-async function runTest() {
-    console.log("--- Testing Unified Quality Report ---");
-    const projectRoot = path.resolve(".");
-    const orchestrator = new QualityOrchestrator(projectRoot);
-    // Create a mixed dataset directory
-    const testDataDir = path.join(projectRoot, "data", "test-unified");
-    if (!fs.existsSync(testDataDir))
-        fs.mkdirSync(testDataDir, { recursive: true });
-    // 1. Create a CSV file (text modality)
-    const csvPath = path.join(testDataDir, "data.csv");
-    fs.writeFileSync(csvPath, "id,name,value\\n1,Alice,10\\n2,Bob,20\\n3,Charlie,30\\n");
-    // 2. Create a test image (using Python)
-    const imagePath = path.join(testDataDir, "test_image.png");
-    const pythonScript = `
-import numpy as np
-from PIL import Image
-img = Image.fromarray(np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8))
-img.save('${imagePath.replace(/\\/g, "\\\\")}')
-    `;
-    fs.writeFileSync(path.join(testDataDir, "gen_image.py"), pythonScript);
-    try {
-        execSync(`python "${path.join(testDataDir, "gen_image.py")}"`);
-    }
-    catch (e) {
-        console.log("⚠️  Could not generate image. Skipping image modality.");
-    }
-    // 3. Create a test audio file (using Python)
-    const audioPath = path.join(testDataDir, "test_audio.wav");
-    const audioScript = `
-import numpy as np
-import soundfile as sf
-sample_rate = 44100
-duration = 1.0
-frequency = 440.0
-t = np.linspace(0, duration, int(sample_rate * duration))
-audio = 0.5 * np.sin(2 * np.pi * frequency * t)
-sf.write('${audioPath.replace(/\\/g, "\\\\")}', audio, sample_rate)
-    `;
-    fs.writeFileSync(path.join(testDataDir, "gen_audio.py"), audioScript);
-    try {
-        execSync(`python "${path.join(testDataDir, "gen_audio.py")}"`);
-    }
-    catch (e) {
-        console.log("⚠️  Could not generate audio. Skipping audio modality.");
-    }
-    // 4. Run Unified Quality Analysis
-    console.log(`\\nAnalyzing multimodal dataset at ${testDataDir}...`);
-    try {
-        const report = await orchestrator.generateReport("test-unified-dataset", testDataDir, null);
-        console.log("\\n📊 Unified Quality Report:");
-        console.log(`- Dataset ID: ${report.dataset_id}`);
-        console.log(`- Modalities: ${report.modalities.join(", ")}`);
-        console.log(`- Overall Quality Score: ${report.overall_quality_score}/100`);
-        if (report.image_quality) {
-            console.log(`\\n🖼️  Image Quality:`);
-            console.log(`  - Total Images: ${report.image_quality.total_images}`);
-            console.log(`  - Avg Resolution: ${report.image_quality.avg_resolution}`);
-        }
-        if (report.audio_quality) {
-            console.log(`\\n🎵 Audio Quality:`);
-            console.log(`  - Total Files: ${report.audio_quality.total_files}`);
-            console.log(`  - Avg Duration: ${report.audio_quality.avg_duration.toFixed(1)}s`);
-        }
-        console.log(`\\n💡 Recommendations:`);
-        report.recommendations.forEach(rec => console.log(`  - ${rec}`));
-        // Verify modalities detected
-        const hasText = report.modalities.includes("text");
-        const hasImage = report.modalities.includes("image");
-        const hasAudio = report.modalities.includes("audio");
-        if (hasText && (hasImage || hasAudio) && report.overall_quality_score > 0) {
-            console.log("\\nVERIFICATION_STATUS: ✅ PASS");
-        }
-        else {
-            console.log("\\nVERIFICATION_STATUS: ⚠️  PARTIAL (Some modalities missing)");
-        }
-    }
-    catch (e) {
-        console.error(`Analysis failed: ${e.message}`);
-        console.log("\\nVERIFICATION_STATUS: ❌ FAIL");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-write.js

@@ -1,14 +0,0 @@
-import fs from 'fs';
-const target = "C:\\Users\\нурбулан\\AppData\\Roaming\\Code\\User\\mcp.json";
-const content = JSON.stringify({ mcpServers: { test: { command: "node" } } }, null, 2);
-try {
-    console.log(`Testing write to: ${target}`);
-    fs.writeFileSync(target, content, "utf8");
-    const stat = fs.statSync(target);
-    console.log(`Success! File size: ${stat.size} bytes`);
-    const readBack = fs.readFileSync(target, "utf8");
-    console.log("Read back content:", readBack);
-}
-catch (e) {
-    console.error("Failed to write:", e);
-}

package/build/scripts/verify-integration.js

@@ -1,57 +0,0 @@
-import { TargetDetector } from "../preparation/target-detector.js";
-import path from "path";
-import fs from "fs";
-async function verifyIntegration() {
-    // 1. Create a dummy CSV with a clear target
-    const testFile = path.join(process.cwd(), "data", "raw", "integration_test.csv");
-    const testId = "integration_test";
-    if (!fs.existsSync(path.dirname(testFile))) {
-        fs.mkdirSync(path.dirname(testFile), { recursive: true });
-    }
-    console.log("Creating test file:", testFile);
-    fs.writeFileSync(testFile, "id,feature1,feature2,SalePrice\n1,10,20,100000\n2,11,21,100000\n3,12,22,110000");
-    // 2. Call the preview_cleaning tool (simulated by calling valid request handler logic or via MCP client if possible)
-    // Since we can't easily call the MCP server from here without a client, we will simulate 
-    // the logic we added to index.ts to ensure it runs without error.
-    try {
-        const { QualityAnalyzer } = await import("../quality/analyzer.js");
-        const { CleaningPlanner } = await import("../cleaning/planner.js");
-        const { CacheService, MockRedisProvider } = await import("../cache/service.js");
-        console.log("Initializing services...");
-        const cacheService = new CacheService(new MockRedisProvider());
-        // Use build/ directory to simulate runtime environment
-        const buildDir = path.join(process.cwd(), "build");
-        const qualityAnalyzer = new QualityAnalyzer(cacheService, buildDir);
-        const cleaningPlanner = new CleaningPlanner(cacheService, buildDir);
-        console.log("Running analysis...");
-        const report = await qualityAnalyzer.analyze(testFile);
-        console.log("Running target detection...");
-        const detector = new TargetDetector(buildDir);
-        const targetResult = await detector.detectTarget(testFile);
-        console.log("Detected:", targetResult);
-        const targetInfo = targetResult.target_column ? {
-            target: targetResult.target_column,
-            confidence: targetResult.confidence
-        } : undefined;
-        console.log("Generating plan...");
-        const plan = await cleaningPlanner.generatePlan(testId, report, undefined, targetInfo);
-        console.log("Plan Operations:", JSON.stringify(plan.operations, null, 2));
-        const hasRename = plan.operations.some(op => op.type === "RenameTarget");
-        if (hasRename) {
-            console.log("✅ SUCCESS: RenameTarget operation found in plan!");
-        }
-        else {
-            console.error("❌ FAILURE: RenameTarget operation NOT found.");
-            process.exit(1);
-        }
-    }
-    catch (e) {
-        console.error("Error during verification:", e);
-        process.exit(1);
-    }
-    finally {
-        if (fs.existsSync(testFile))
-            fs.unlinkSync(testFile);
-    }
-}
-verifyIntegration();

package/build/scripts/verify-priority.js

@@ -1,33 +0,0 @@
-import path from "path";
-import { MetadataStore } from "../metadata/store.js";
-import { VectorStore } from "../search/vector-store.js";
-import { Embedder } from "../search/embedder.js";
-import { SearchEngine } from "../search/engine.js";
-import { formatSearchResults } from "../tools/formatter.js";
-import fs from "fs";
-const query = process.argv[2] || "anime";
-// Use the actual .vesper data path if it exists, otherwise use local data/
-const homeDir = process.env.HOME || process.env.USERPROFILE || process.cwd();
-const vesperDataRoot = path.join(homeDir, ".vesper");
-let dbPath = path.join(vesperDataRoot, "data", "metadata.db");
-let vectorPath = path.join(vesperDataRoot, "data", "vectors.json");
-if (!fs.existsSync(dbPath)) {
-    console.error("Using local project data directory as fallback...");
-    dbPath = path.join(process.cwd(), "data", "metadata.db");
-    vectorPath = path.join(process.cwd(), "data", "vectors.json");
-}
-const metadataStore = new MetadataStore(dbPath);
-const vectorStore = new VectorStore(vectorPath);
-const embedder = Embedder.getInstance();
-const searchEngine = new SearchEngine(metadataStore, vectorStore, embedder);
-async function run() {
-    console.log(`\n=== VERIFYING SOURCE PRIORITIZATION [Query: "${query}"] ===\n`);
-    const results = await searchEngine.search(query, { limit: 5 });
-    if (results.length === 0) {
-        console.log("No results found. Run a search that triggers JIT first!");
-        return;
-    }
-    // Print formatted results to show badges
-    console.log(formatSearchResults(results));
-}
-run().catch(console.error);

package/build/search/embedder.js

@@ -1,34 +0,0 @@
-import { pipeline, env } from "@xenova/transformers";
-// Disable local model check to ensure it downloads if not found
-env.allowLocalModels = false;
-env.useBrowserCache = false;
-export class Embedder {
-    static instance;
-    extractor = null;
-    modelName = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
-    constructor() { }
-    static getInstance() {
-        if (!Embedder.instance) {
-            Embedder.instance = new Embedder();
-        }
-        return Embedder.instance;
-    }
-    async init() {
-        if (this.extractor)
-            return;
-        console.error(`Loading embedding model: ${this.modelName}...`);
-        this.extractor = await pipeline("feature-extraction", this.modelName);
-        console.error("Embedding model loaded successfully.");
-    }
-    async embed(text) {
-        if (!this.extractor) {
-            await this.init();
-        }
-        const result = await this.extractor(text, {
-            pooling: "mean",
-            normalize: true,
-        });
-        // result.data is already a Float32Array in Xenova/transformers
-        return result.data;
-    }
-}