vesper-wizard 2.3.1 → 2.3.2

package/build/scripts/test-kaggle-download.js

@@ -1,34 +0,0 @@
-import { KaggleDownloader } from "../ingestion/kaggle-downloader.js";
-import path from "path";
-import fs from "fs";
-import { fileURLToPath } from "url";
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const projectRoot = path.join(__dirname, "..", "..");
-async function testKaggleDownload() {
-    // Try to get credentials from env
-    const user = process.env.KAGGLE_USERNAME;
-    const key = process.env.KAGGLE_KEY;
-    if (!user || !key) {
-        console.error("KAGGLE_USERNAME and KAGGLE_KEY must be set to run this test.");
-        return;
-    }
-    const downloader = new KaggleDownloader(user, key);
-    const repoId = "shivam2503/diamonds"; // Small classic dataset
-    console.log(`Testing Kaggle Download for ${repoId}...`);
-    const targetDir = path.join(projectRoot, "data", "test", "kaggle_diamonds");
-    if (fs.existsSync(targetDir))
-        fs.rmSync(targetDir, { recursive: true });
-    try {
-        const bestFile = await downloader.download(repoId, targetDir, (p) => {
-            process.stdout.write(`\rProgress: ${p}%`);
-        });
-        console.log(`\nDownload complete!`);
-        console.log(`Best file found: ${bestFile}`);
-        console.log(`Size: ${fs.statSync(bestFile).size} bytes`);
-    }
-    catch (e) {
-        console.error("\nKaggle Download failed:", e.message);
-    }
-}
-testKaggleDownload().catch(console.error);

package/build/scripts/test-large-data.js

@@ -1,50 +0,0 @@
-import { StreamProcessor } from "../data/streaming.js";
-import { WorkerPool } from "../data/worker-pool.js";
-import { Readable } from "stream";
-async function runTest() {
-    console.log("--- Initializing Large Dataset Handling Test ---");
-    // 1. Generate Mock Data (10,000 records)
-    const records = Array.from({ length: 10000 }, (_, i) => ({
-        id: i,
-        name: `Record ${i}`,
-        value: Math.random() * 100,
-        category: i % 2 === 0 ? "A" : "B"
-    }));
-    console.log(`Generated ${records.length} mock records.`);
-    // 2. Test Sampling Strategy
-    console.log("\n--- Testing Sampling Strategy (10%) ---");
-    const isSampled = StreamProcessor.createSampler(10);
-    const sample = records.filter(isSampled);
-    console.log(`Sample size: ${sample.length} (Expected ~1000)`);
-    // 3. Test Streaming Utility
-    console.log("\n--- Testing Streaming Utility (Chunk size: 2000) ---");
-    const stream = Readable.from(records);
-    let chunkCount = 0;
-    const processChunk = async (chunk) => {
-        chunkCount++;
-        console.log(`Processing chunk ${chunkCount} of size ${chunk.length}`);
-        return chunk.map(r => ({ ...r, status: "streamed" }));
-    };
-    const streamedResults = await StreamProcessor.processInChunks(stream, { chunkSize: 2000 }, processChunk);
-    console.log(`Total streamed records: ${streamedResults.length}`);
-    // 4. Test Parallel Worker Pool
-    console.log("\n--- Testing Parallel Worker Pool (Optional) ---");
-    try {
-        const pool = new WorkerPool(4);
-        console.log("Running parallel transformation on all records...");
-        const start = Date.now();
-        const parallelResults = await pool.processParallel(records, 2500);
-        const end = Date.now();
-        console.log(`Parallel processing took ${end - start}ms`);
-    }
-    catch (err) {
-        console.warn(" Parallel Worker Pool skipped or failed (Common with tsx + worker_threads). Streaming and Sampling are still functional.");
-    }
-    if (streamedResults.length === 10000) {
-        console.log("\n✅ Success: Chunked streaming and sampling verified.");
-    }
-    else {
-        console.error("\n❌ Failure: Processing mismatch.");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-mcp-v5.js

@@ -1,74 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { JobManager } from "../jobs/manager.js";
-import { SearchEngine } from "../search/engine.js";
-import { VectorStore } from "../search/vector-store.js";
-import { Embedder } from "../search/embedder.js";
-import { PipelineExecutor } from "../cleaning/executor.js";
-import path from "path";
-import fs from "fs";
-async function testPhase5Tools() {
-    console.log(" Testing Phase 5.2: New MCP Tools Integration\n");
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
-    const store = new MetadataStore(dbPath);
-    const vectorStore = new VectorStore(vectorPath);
-    const embedder = Embedder.getInstance();
-    const searchEngine = new SearchEngine(store, vectorStore, embedder);
-    const jobManager = JobManager.getInstance(store);
-    const executor = new PipelineExecutor();
-    // 1. Test compare_datasets
-    console.log("Step 1: Testing compare_datasets...");
-    const allDs = store.getAllDatasets();
-    if (allDs.length < 2) {
-        console.warn(" ! Need at least 2 datasets in DB to test comparison. Skipping.");
-    }
-    else {
-        const ids = allDs.slice(0, 2).map(d => d.id);
-        console.log(` - Comparing: ${ids.join(", ")}`);
-        // Simulating comparison table logic from index.ts
-        const datasets = ids.map(id => store.getDataset(id)).filter(d => !!d);
-        let comparison = "| Metric | " + datasets.map(d => d.name).join(" | ") + " |\n";
-        comparison += "| Quality Score | " + datasets.map(d => d.quality_score).join(" | ") + " |\n";
-        console.log(comparison);
-    }
-    // 2. Test analyze_quality / preview_cleaning (Local logic)
-    console.log("\nStep 2: Testing Analysis/Preview (Simulated)...");
-    const testFile = path.join(process.cwd(), "e2e_demo_output", "raw_data.csv");
-    if (fs.existsSync(testFile)) {
-        console.log(" - Testing on e2e_demo_output/raw_data.csv");
-        // Logic check
-    }
-    else {
-        console.warn(" ! No test file found at e2e_demo_output/raw_data.csv. Run 'tsx src/scripts/e2e-demo.ts' first.");
-    }
-    // 3. Test prepare_dataset (Async Job)
-    console.log("\nStep 3: Testing prepare_dataset (Autonomous Orchestrator)...");
-    const query = "financial data for stock prediction";
-    // Create job (Logic from index.ts)
-    const job = jobManager.createJob("prepare", 0, { query });
-    console.log(` - Job Created: ${job.id}`);
-    // Register listener for simulated work (Logic from index.ts)
-    jobManager.on("processJob", async (currJob, execute) => {
-        if (currJob.id !== job.id)
-            return;
-        await execute(async () => {
-            console.log(" - [Worker] Starting autonomous preparation task...");
-            await new Promise(r => setTimeout(r, 1000));
-            console.log(" - [Worker] Phase 1: Search complete");
-            await new Promise(r => setTimeout(r, 1000));
-            console.log(" - [Worker] Phase 2: Quality analysis complete");
-            return "data/exports/prepared_dataset.parquet";
-        });
-    });
-    // 4. Test check_job_status (Polling)
-    console.log("\nStep 4: Polling Job Status (Simulating UI Check)...");
-    for (let i = 0; i < 5; i++) {
-        const currentJob = store.getJob(job.id);
-        console.log(` - [TS +${i}s] Status: ${currentJob?.status}, Progress: ${currentJob?.progress}%, Activity: ${currentJob?.status_text}`);
-        if (currentJob?.status === "completed")
-            break;
-        await new Promise(r => setTimeout(r, 800));
-    }
-    console.log("\n Phase 5 tools logic verified.");
-}
-testPhase5Tools().catch(console.error);

package/build/scripts/test-media-analysis.js

@@ -1,61 +0,0 @@
-import { MediaAnalyzer } from "../quality/media-analyzer.js";
-import path from "path";
-import fs from "fs";
-import { execSync } from "child_process";
-async function runTest() {
-    console.log("--- Testing Media Quality Analysis ---");
-    const projectRoot = path.resolve(".");
-    const analyzer = new MediaAnalyzer(projectRoot);
-    // 1. Create a sample audio file using Python (sine wave)
-    const testMediaDir = path.join(projectRoot, "data", "test-media");
-    if (!fs.existsSync(testMediaDir))
-        fs.mkdirSync(testMediaDir, { recursive: true });
-    const audioPath = path.join(testMediaDir, "test_audio.wav");
-    console.log("Generating test audio file (sine wave)...");
-    const pythonScript = `
-import numpy as np
-import soundfile as sf
-sample_rate = 44100
-duration = 2.0
-frequency = 440.0
-t = np.linspace(0, duration, int(sample_rate * duration))
-audio = 0.5 * np.sin(2 * np.pi * frequency * t)
-sf.write('${audioPath.replace(/\\/g, "\\\\")}', audio, sample_rate)
-    `;
-    fs.writeFileSync(path.join(testMediaDir, "gen_audio.py"), pythonScript);
-    try {
-        execSync(`python "${path.join(testMediaDir, "gen_audio.py")}"`);
-    }
-    catch (e) {
-        console.log("⚠️  Could not generate audio (soundfile may not be installed). Skipping audio test.");
-        console.log("VERIFICATION_STATUS: ⚠️  PARTIAL (Audio generation failed)");
-        return;
-    }
-    // 2. Run Analysis
-    console.log(`Analyzing ${audioPath}...`);
-    try {
-        const report = await analyzer.analyze(audioPath);
-        console.log("Analysis Result:");
-        console.log(`- Total Files: ${report.total_files}`);
-        console.log(`- OK Files: ${report.ok_files}`);
-        if ('avg_audio_duration' in report) {
-            console.log(`- Average Audio Duration: ${report.avg_audio_duration}s`);
-            const firstAudio = report.details[0];
-            if (firstAudio && 'sample_rate' in firstAudio) {
-                console.log(`- Sample Rate: ${firstAudio.sample_rate}Hz`);
-                console.log(`- Duration: ${firstAudio.duration}s`);
-            }
-        }
-        if (report.ok_files === 1 && report.total_files === 1) {
-            console.log("\nVERIFICATION_STATUS: ✅ PASS");
-        }
-        else {
-            console.log("\nVERIFICATION_STATUS: ❌ FAIL - Incorrect stats");
-        }
-    }
-    catch (e) {
-        console.error(`Analysis failed: ${e.message}`);
-        console.log("\nVERIFICATION_STATUS: ❌ FAIL");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-monitoring.js

@@ -1,91 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { MonitoringStore } from "../metadata/monitoring-store.js";
-import { MonitoringService } from "../metadata/monitoring-service.js";
-import path from "path";
-async function runTest() {
-    const dbPath = path.resolve("data", "test-monitoring.db");
-    const metadataStore = new MetadataStore(dbPath);
-    const monitoringStore = new MonitoringStore(metadataStore.db);
-    const service = new MonitoringService(monitoringStore, metadataStore);
-    // 1. Setup mock data
-    const dsId = "test/dataset-1";
-    const initialMetadata = {
-        id: dsId,
-        source: "huggingface",
-        name: "Test Dataset",
-        description: "Initial description",
-        downloads: 100,
-        likes: 10,
-        last_updated: "2023-01-01T00:00:00Z",
-        download_url: "https://huggingface.co/datasets/test/dataset-1",
-        quality_score: 85,
-        splits: [{ name: "train", num_examples: 1000 }],
-        license: { id: "mit", category: "safe", usage_restrictions: [], warnings: [] },
-        tags: [],
-        task: "text-classification",
-        languages: ["en"],
-        is_structured: true,
-        has_target_column: true,
-        total_examples: 1000,
-        is_safe_source: true,
-        has_personal_data: false,
-        is_paywalled: false,
-        is_scraped_web_data: false,
-        uses_https: true,
-        has_train_split: true,
-        has_test_split: false,
-        has_validation_split: false,
-        description_length: 100,
-        has_readme: true,
-        quality_warnings: []
-    };
-    metadataStore.saveDataset(initialMetadata);
-    // 2. Setup Monitor and Webhook
-    monitoringStore.saveWebhook({
-        id: "slack-1",
-        name: "General Slack",
-        channel: "slack",
-        url: "https://hooks.slack.com/services/...",
-        enabled: true
-    });
-    monitoringStore.saveMonitor({
-        dataset_id: dsId,
-        enabled: true,
-        auto_reprocess: true,
-        last_checked_version: initialMetadata.last_updated,
-        webhook_ids: ["slack-1"],
-        created_at: new Date().toISOString(),
-        updated_at: new Date().toISOString()
-    });
-    console.log("--- Monitoring Setup Complete ---");
-    // 3. Simulate an update
-    const updatedMetadata = {
-        ...initialMetadata,
-        last_updated: "2023-02-01T00:00:00Z", // New version
-        downloads: 150,
-        quality_score: 90,
-        total_examples: 1200 // Significant change
-    };
-    console.log("Checking for updates...");
-    // Mock fetch function
-    const mockFetch = async (id, source) => {
-        if (id === dsId)
-            return updatedMetadata;
-        return null;
-    };
-    const diffs = await service.checkUpdates(mockFetch);
-    console.log(`Found ${diffs.length} updates.`);
-    if (diffs.length > 0) {
-        const diff = diffs[0];
-        console.log(`Version Change: ${diff.old_version} -> ${diff.new_version}`);
-        console.log(`Impact Score: ${diff.impact_score}`);
-        console.log("Changes:", JSON.stringify(diff.changes, null, 2));
-    }
-    if (diffs.length > 0 && diffs[0].impact_score > 0) {
-        console.log("\n✅ Success: Dataset update detected and diffed correctly.");
-    }
-    else {
-        console.error("\n❌ Failure: Update not detected.");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-observability.js

@@ -1,106 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { JobManager } from "../jobs/manager.js";
-import { ObservabilityService, MockErrorTracker } from "../monitoring/observability.js";
-import path from "path";
-async function runTest() {
-    console.log("--- Initializing Enhanced Observability Test ---");
-    const dbPath = path.resolve("data", "test-obs-10.db");
-    const store = new MetadataStore(dbPath);
-    const errorTracker = new MockErrorTracker();
-    const stats = new ObservabilityService(errorTracker);
-    const manager = JobManager.getInstance(store, stats);
-    manager.setConcurrency(5); // Increased for faster test
-    const jobsToRun = 10;
-    // Listener to simulate job processing
-    manager.on("processJob", async (job, run) => {
-        // Fix: Parse JSON metadata correctly
-        const meta = job.metadata ? JSON.parse(job.metadata) : null;
-        const isFailing = meta === "FAIL";
-        const isSlow = meta === "SLOW";
-        await run(async () => {
-            // Simulated delay: 100ms for fast, 800ms for slow
-            const delay = isSlow ? 800 : 100;
-            await new Promise(r => setTimeout(r, delay));
-            if (isFailing) {
-                console.log(`[Worker] Failing Job: ${job.id}`);
-                throw new Error("Simulated job failure for metrics tracking");
-            }
-            console.log(`[Worker] Finished Job: ${job.id}`);
-        });
-    });
-    console.log(`\nEnqueuing ${jobsToRun} mixed jobs...`);
-    // Enqueue 10 mixed jobs
-    // [Success, Success, FAIL, SlowSuccess, FAIL, Success, Success, SlowSuccess, FAIL, Success] -> 7 Success, 3 Fail
-    const jobConfig = [
-        { type: "clean", meta: "SUCCESS" }, // 1
-        { type: "prepare", meta: "SUCCESS" }, // 2
-        { type: "fusion", meta: "FAIL" }, // 3
-        { type: "clean", meta: "SLOW" }, // 4
-        { type: "prepare", meta: "FAIL" }, // 5
-        { type: "fusion", meta: "SUCCESS" }, // 6
-        { type: "clean", meta: "SUCCESS" }, // 7
-        { type: "prepare", meta: "SLOW" }, // 8
-        { type: "fusion", meta: "FAIL" }, // 9
-        { type: "clean", meta: "SUCCESS" } // 10
-    ];
-    for (const conf of jobConfig) {
-        // Fix: Use maxAttempts: 1 to avoid retries and get immediate metrics
-        manager.createJob(conf.type, 0, conf.meta, 1);
-    }
-    // Monitor for completion
-    return new Promise((resolve) => {
-        const interval = setInterval(() => {
-            const currentStats = stats.getStats();
-            const totalFinished = Object.values(currentStats).reduce((acc, s) => acc + s.successCount + s.failureCount, 0);
-            if (totalFinished >= jobsToRun) {
-                clearInterval(interval);
-                verify(stats, errorTracker, resolve);
-            }
-        }, 500);
-    });
-}
-function verify(stats, errorTracker, resolve) {
-    console.log("\n--- Verification Results ---");
-    const prometheus = stats.getPrometheusMetrics();
-    const summary = stats.getStats();
-    const actualSuccess = Object.values(summary).reduce((acc, s) => acc + s.successCount, 0);
-    const actualFailure = Object.values(summary).reduce((acc, s) => acc + s.failureCount, 0);
-    console.log(`Measured Success: ${actualSuccess} (Expected 7)`);
-    console.log(`Measured Failure: ${actualFailure} (Expected 3)`);
-    let failed = false;
-    if (actualSuccess === 7 && actualFailure === 3) {
-        console.log("✅ Jobs processed/hour (total counts) verified.");
-    }
-    else {
-        console.error("❌ Stats mismatch.");
-        failed = true;
-    }
-    // 2. Verify Latency Histogram
-    const hasLatency = prometheus.includes("job_duration_seconds_sum");
-    const maxLatencyClean = summary.clean?.maxDuration;
-    console.log(`Max Latency (clean): ${maxLatencyClean} (Expected > 100ms)`);
-    if (hasLatency && parseFloat(maxLatencyClean) > 0) {
-        console.log("✅ Latency histogram contains recorded values.");
-    }
-    else {
-        console.error("❌ Latency data missing.");
-        failed = true;
-    }
-    // 3. Mock Sentry / Error Tracking Verification
-    console.log(`Exceptions Captured: ${errorTracker.exceptions.length} (Expected 3)`);
-    if (errorTracker.exceptions.length === 3) {
-        console.log("✅ Error tracking successfully logged all failed jobs.");
-    }
-    else {
-        console.error("❌ Error tracking mismatch.");
-        failed = true;
-    }
-    if (!failed) {
-        console.log("\n✅ Success: Enhanced observability verification complete.");
-    }
-    else {
-        console.error("\n❌ Failure: Observability verification failed.");
-    }
-    resolve();
-}
-runTest().catch(console.error);

package/build/scripts/test-packager.js

@@ -1,55 +0,0 @@
-import { MetadataPackager } from "../export/packager.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Metadata Packager Test ===\n");
-    const packager = new MetadataPackager();
-    const testDir = path.join(process.cwd(), "test_package_input");
-    const outputDir = path.join(process.cwd(), "test_package_output");
-    if (!fs.existsSync(testDir))
-        fs.mkdirSync(testDir);
-    if (fs.existsSync(outputDir))
-        fs.rmSync(outputDir, { recursive: true, force: true });
-    // 1. Create Mock Data
-    const dataPath = path.join(testDir, "train.csv");
-    fs.writeFileSync(dataPath, "id,value\n1,10\n2,20\n");
-    const qualityReport = {
-        score: 85,
-        issues: ["Missing values in column 'label'"]
-    };
-    const metadata = {
-        name: "Test Dataset",
-        version: "1.0.0",
-        description: "A test dataset for packaging",
-        license: "MIT",
-        author: "Vesper Team",
-        tags: ["test", "ml"]
-    };
-    // 2. Create Package
-    console.log("Creating package...");
-    const result = await packager.createPackage(outputDir, [{ path: dataPath, format: "csv", name: "training-data" }], metadata, { qualityReport });
-    console.log("Result:", JSON.stringify(result, null, 2));
-    if (result.success) {
-        console.log("\nVerifying files in output...");
-        const files = fs.readdirSync(outputDir);
-        console.log("Files found:", files);
-        if (files.includes("datapackage.json") && files.includes("train.csv") && files.includes("quality_report.json")) {
-            console.log("PASS: All files present in package");
-            const manifest = JSON.parse(fs.readFileSync(path.join(outputDir, "datapackage.json"), "utf8"));
-            console.log("Manifest resources count:", manifest.resources.length);
-            if (manifest.resources.length === 2 && manifest.resources[0].hash.startsWith("sha256:")) {
-                console.log("PASS: Manifest entry and hash verified");
-            }
-        }
-        else {
-            console.error("FAIL: Missing files in package");
-        }
-    }
-    else {
-        console.error("FAIL: Packaging failed:", result.error);
-    }
-    // Cleanup
-    // fs.rmSync(testDir, { recursive: true, force: true });
-    // fs.rmSync(outputDir, { recursive: true, force: true });
-}
-main().catch(console.error);

package/build/scripts/test-pipeline.js

@@ -1,50 +0,0 @@
-import { PipelineExecutor } from "../cleaning/executor.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Full Pipeline Test ===\n");
-    const executor = new PipelineExecutor();
-    const testFile = path.join(process.cwd(), "test_pipeline.csv");
-    // Create Dirty Data
-    // - duplicate: Duplicate row
-    // - empty: 100% missing (should drop)
-    // - price: "100.5" string (should fix type)
-    const csvContent = `id,price,empty,desc
-1,"100.5",,A
-2,"50.2",,B
-1,"100.5",,A
-3,"75.0",,C`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created dirty file: ${testFile}`);
-    try {
-        console.log("\nRunning Pipeline...");
-        const result = await executor.runPipeline("test-pipeline-ds", testFile);
-        console.log("\n=== Pipeline Report ===");
-        console.log(`Plan Generated: ${result.plan.operations.length} operations`);
-        // Assertions
-        const ops = result.plan.operations.map(o => o.type);
-        console.log("Operations:", ops);
-        const hasDedupe = ops.includes("RemoveDuplicates");
-        const hasDrop = ops.includes("DropColumns");
-        const hasFix = ops.includes("FixTypes");
-        if (hasDedupe && hasDrop && hasFix) {
-            console.log("Plan Logic: PASS (Generated precise cleaning steps)");
-        }
-        else {
-            console.error("Plan Logic: FAIL (Missed some cleaning steps)");
-        }
-        if (result.cleaning_result.success && fs.existsSync(result.final_output_path)) {
-            console.log("Execution: PASS (Created cleaned file)");
-            // Cleanup
-            fs.unlinkSync(testFile);
-            fs.unlinkSync(result.final_output_path);
-        }
-        else {
-            console.error("Execution: FAIL");
-        }
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-planning.js

@@ -1,64 +0,0 @@
-import { CleaningPlanner } from "../cleaning/planner.js";
-async function main() {
-    console.log("=== Vesper Cleaning Planner Test ===\n");
-    const planner = new CleaningPlanner();
-    // Mock Report
-    const mockReport = {
-        row_count: 1000,
-        column_count: 4,
-        duplicate_rows: 50,
-        duplicate_percentage: 5.0,
-        columns: [
-            {
-                name: "id", type: "Int64", inferred_type: "Int64",
-                missing_count: 0, missing_percentage: 0, unique_count: 1000,
-                is_constant: false, is_mixed_type: false
-            },
-            {
-                name: "empty_col", type: "String", inferred_type: "String",
-                missing_count: 950, missing_percentage: 95.0, unique_count: 1,
-                is_constant: false, is_mixed_type: false
-            },
-            {
-                name: "age", type: "String", inferred_type: "Numeric (Stored as String)",
-                missing_count: 10, missing_percentage: 1.0, unique_count: 50,
-                is_constant: false, is_mixed_type: false
-            },
-            {
-                name: "score", type: "Float64", inferred_type: "Float64",
-                missing_count: 5, missing_percentage: 0.5, unique_count: 900,
-                is_constant: false, is_mixed_type: false
-            }
-        ],
-        warnings: [],
-        schema_warnings: [],
-        overall_score: 50
-    };
-    console.log("Input Scenario:");
-    console.log("- 50 duplicate rows -> Expect RemoveDuplicates");
-    console.log("- 'empty_col' (95% missing) -> Expect DropColumns");
-    console.log("- 'age' (String but looks Numeric) -> Expect FixTypes");
-    console.log("- 'score' (Missing values) -> Expect FillMissing");
-    const plan = await planner.generatePlan("test-dataset", mockReport);
-    console.log("\n=== Generated Plan ===");
-    plan.operations.forEach((op, i) => {
-        console.log(`${i + 1}. [${op.type}] Reason: ${op.reason}`);
-        if (op.params && Object.keys(op.params).length > 0) {
-            console.log(`   Params: ${JSON.stringify(op.params)}`);
-        }
-    });
-    // Verification
-    const types = plan.operations.map(o => o.type);
-    const hasDedupe = types.includes("RemoveDuplicates");
-    const hasDrop = types.includes("DropColumns");
-    const hasFix = types.includes("FixTypes");
-    const hasFill = types.includes("FillMissing");
-    if (hasDedupe && hasDrop && hasFix && hasFill) {
-        console.log("\nTest passed! All expected operations generated.");
-    }
-    else {
-        console.error("\nTest failed! Missing expected operations.");
-        console.log("Found:", types);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-privacy.js

@@ -1,38 +0,0 @@
-import { QualityAnalyzer } from "../quality/analyzer.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Privacy Scanner Test ===\n");
-    const analyzer = new QualityAnalyzer();
-    const testFile = path.join(process.cwd(), "test_privacy.csv");
-    // Create a CSV with PII
-    const csvContent = `id,name,contact,notes
-1,John Doe,john@example.com,"User IP: 192.168.1.1"
-2,Jane Smith,(555) 123-4567,"SSN is 123-45-6789"
-3,Bob Jones,bob@work.com,"Safe notes"`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created test file: ${testFile}`);
-    console.log("Expectations:");
-    console.log("  - Should detect 'Email' in 'contact' column");
-    console.log("  - Should detect 'Phone' in 'contact' column");
-    console.log("  - Should detect 'SSN' in 'notes' column");
-    console.log("  - Should detect 'IPv4' in 'notes' column");
-    try {
-        console.log("\nRunning analyzer...");
-        const report = await analyzer.analyze(testFile);
-        console.log("\n=== Privacy Report ===");
-        if (report.pii_warnings && report.pii_warnings.length > 0) {
-            report.pii_warnings.forEach(w => console.log(`[!] ${w}`));
-            console.log("\nTest passed! PII detected.");
-        }
-        else {
-            console.error("\nTest failed! No PII detected.");
-        }
-        // Cleanup
-        fs.unlinkSync(testFile);
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-production-sync.js

@@ -1,36 +0,0 @@
-import { JobManager } from "../jobs/manager.js";
-import { MetadataStore } from "../metadata/store.js";
-import fs from "fs";
-async function testSync() {
-    console.log("Starting Production Sync Test...");
-    const dbPath = "prod_sync.db";
-    if (fs.existsSync(dbPath))
-        fs.unlinkSync(dbPath);
-    const store = new MetadataStore(dbPath);
-    const jobManager = JobManager.getInstance(store);
-    console.log("Attaching listener (same as index.ts)...");
-    jobManager.on("processJob", async (job, execute) => {
-        console.log(`[Listener 1] Received job ${job.id}`);
-        if (typeof execute !== 'function') {
-            console.error(`[Listener 1] ERROR: execute is not a function! It is: ${typeof execute}`);
-            return;
-        }
-        const task = async () => {
-            console.log("[Listener 1] Task running...");
-            return "ok";
-        };
-        await execute(task);
-        console.log("[Listener 1] Task finished.");
-    });
-    console.log("Emitting job...");
-    const job = jobManager.createJob("prepare", 0, { query: "test" });
-    // Wait for the background loop
-    await new Promise(r => setTimeout(r, 1000));
-    const finalJob = store.getJob(job.id);
-    console.log(`Job Result: ${finalJob?.status} - ${finalJob?.status_text}`);
-    store.close();
-    if (fs.existsSync(dbPath))
-        fs.unlinkSync(dbPath);
-    console.log("Test Complete.");
-}
-testSync().catch(console.error);