vesper-wizard 2.3.1 → 2.3.2

package/build/quality/types.js

@@ -1 +0,0 @@
-export {};

package/build/scripts/build-index.js

@@ -1,54 +0,0 @@
-import { Embedder } from "../search/embedder.js";
-import { VectorStore } from "../search/vector-store.js";
-import { MetadataStore } from "../metadata/store.js";
-import path from "path";
-async function main() {
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
-    const metadataStore = new MetadataStore(dbPath);
-    const vectorStore = new VectorStore(vectorPath);
-    const embedder = Embedder.getInstance();
-    const datasets = metadataStore.getAllDatasets();
-    const indexedIds = new Set(vectorStore.getAllIds());
-    // Filter to only new datasets
-    const toIndex = datasets.filter(ds => !indexedIds.has(ds.id));
-    console.error(`Total datasets: ${datasets.length}, Already indexed: ${indexedIds.size}, To index: ${toIndex.length}`);
-    const BATCH_SIZE = 20;
-    let processed = 0;
-    for (let i = 0; i < toIndex.length; i += BATCH_SIZE) {
-        const batch = toIndex.slice(i, i + BATCH_SIZE);
-        try {
-            // Prepare texts for batch embedding
-            const texts = batch.map(ds => [
-                ds.name,
-                ds.description,
-                `Task: ${ds.task}`,
-                `Languages: ${ds.languages?.join(", ") || ""}`,
-                `Tags: ${ds.tags?.join(" ") || ""}`
-            ].join(" ").slice(0, 1500));
-            // Memory-safe sequential embedding (avoids OOM on large libraries)
-            for (let idx = 0; idx < batch.length; idx++) {
-                const ds = batch[idx];
-                try {
-                    const vector = await embedder.embed(texts[idx]);
-                    vectorStore.add(ds.id, vector);
-                }
-                catch (err) {
-                    console.error(`Failed to index ${ds.id}:`, err);
-                }
-            }
-            processed += batch.length;
-            if (processed % 100 === 0 || i + BATCH_SIZE >= toIndex.length) {
-                console.error(`Indexed ${processed}/${toIndex.length} new datasets...`);
-                vectorStore.save();
-            }
-        }
-        catch (err) {
-            console.error(`Batch around ${i} failed:`, err);
-        }
-    }
-    vectorStore.save();
-    console.error("Vector indexing complete.");
-    metadataStore.close();
-}
-main().catch(console.error);

package/build/scripts/check-db.js

@@ -1,73 +0,0 @@
-import Database from "better-sqlite3";
-import path from "path";
-import fs from "fs";
-async function main() {
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    if (!fs.existsSync(dbPath)) {
-        console.error("Database not found. Run 'npm run scrape' first.");
-        process.exit(1);
-    }
-    const db = new Database(dbPath);
-    try {
-        // Get total count
-        const count = db.prepare("SELECT COUNT(*) as count FROM datasets").get();
-        console.log(`\nTotal datasets in database: ${count.count}\n`);
-        // Check which columns exist
-        const tableInfo = db.prepare("PRAGMA table_info(datasets)").all();
-        const columns = tableInfo.map(col => col.name);
-        const hasNewColumns = columns.includes("is_safe_source");
-        // Get basic statistics (works with both old and new schema)
-        const stats = db.prepare(`
-            SELECT 
-                COUNT(*) as total,
-                SUM(downloads) as total_downloads,
-                AVG(quality_score) as avg_quality,
-                SUM(CASE WHEN license_category = 'safe' THEN 1 ELSE 0 END) as safe_licenses,
-                SUM(CASE WHEN has_train_split = 1 THEN 1 ELSE 0 END) as with_train_split
-            FROM datasets
-        `).get();
-        console.log("Statistics:");
-        console.log(`   Total downloads: ${stats.total_downloads?.toLocaleString() || 0}`);
-        console.log(`   Average quality score: ${Math.round(stats.avg_quality || 0)}`);
-        console.log(`   Safe licenses: ${stats.safe_licenses || 0}`);
-        console.log(`   With train split: ${stats.with_train_split || 0}`);
-        // Show extended stats if new schema is available
-        if (hasNewColumns) {
-            const extendedStats = db.prepare(`
-                SELECT 
-                    SUM(CASE WHEN is_safe_source = 1 THEN 1 ELSE 0 END) as safe_sources,
-                    SUM(CASE WHEN is_structured = 1 THEN 1 ELSE 0 END) as structured,
-                    SUM(total_examples) as total_examples
-                FROM datasets
-            `).get();
-            console.log(`   Safe sources: ${extendedStats.safe_sources || 0}`);
-            console.log(`   Structured datasets: ${extendedStats.structured || 0}`);
-            console.log(`   Total examples: ${extendedStats.total_examples?.toLocaleString() || 0}`);
-        }
-        else {
-            console.log(`   WARNING: Database uses old schema. Re-scrape to get extended statistics.`);
-        }
-        console.log();
-        // Top 5 by downloads
-        const top5 = db.prepare(`
-            SELECT id, name, downloads, quality_score, license_category
-            FROM datasets
-            ORDER BY downloads DESC
-            LIMIT 5
-        `).all();
-        console.log("Top 5 datasets by downloads:");
-        top5.forEach((ds, i) => {
-            console.log(`   ${i + 1}. ${ds.id}`);
-            console.log(`      Downloads: ${ds.downloads.toLocaleString()}, Quality: ${ds.quality_score}, License: ${ds.license_category}`);
-        });
-        console.log();
-    }
-    catch (error) {
-        console.error("Error reading database:", error);
-        process.exit(1);
-    }
-    finally {
-        db.close();
-    }
-}
-main();

package/build/scripts/check-jobs.js

@@ -1,24 +0,0 @@
-import Database from "better-sqlite3";
-import path from "path";
-// Checking all plausible databases for jobs
-const dbs = ["metadata.db", "vesper.db", "datasets.db"];
-for (const dbName of dbs) {
-    const dbPath = path.resolve("data", dbName);
-    try {
-        const db = new Database(dbPath);
-        const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='jobs'").all();
-        if (tables.length > 0) {
-            console.log(`\n--- Checking jobs in ${dbName} ---`);
-            const jobs = db.prepare("SELECT * FROM jobs ORDER BY created_at DESC LIMIT 20").all();
-            for (const job of jobs) {
-                if (JSON.stringify(job).toLowerCase().includes("naruto")) {
-                    console.log(JSON.stringify(job, null, 2));
-                }
-            }
-        }
-        db.close();
-    }
-    catch (e) {
-        // Silently skip if DB doesn't exist or table missing
-    }
-}

package/build/scripts/check-naruto.js

@@ -1,17 +0,0 @@
-import Database from "better-sqlite3";
-import path from "path";
-// Trying metadata.db which is larger
-const dbPath = path.resolve("data", "metadata.db");
-const db = new Database(dbPath);
-try {
-    const query = "naruto";
-    const results = db.prepare("SELECT * FROM datasets WHERE name LIKE ? OR description LIKE ?").all(`%${query}%`, `%${query}%`);
-    console.log(`Found ${results.length} results for "${query}" in metadata.db:`);
-    console.log(JSON.stringify(results, null, 2));
-}
-catch (e) {
-    console.error("Error checking database:", e.message);
-}
-finally {
-    db.close();
-}

package/build/scripts/cleanup-kaggle.js

@@ -1,41 +0,0 @@
-#!/usr/bin/env node
-/**
- * Cleanup script to remove all Kaggle datasets from Vesper
- */
-import { fileURLToPath } from "url";
-import path from "path";
-import { MetadataStore } from "../metadata/store.js";
-import { VectorStore } from "../search/vector-store.js";
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const homeDir = process.env.HOME || process.env.USERPROFILE || path.join(__dirname, "..");
-const dataRoot = path.join(homeDir, ".vesper");
-const dbPath = path.join(dataRoot, "data", "metadata.db");
-const vectorPath = path.join(dataRoot, "data", "vectors.json");
-console.log("🧹 Vesper Kaggle Cleanup");
-console.log("========================\n");
-try {
-    const metadataStore = new MetadataStore(dbPath);
-    const vectorStore = new VectorStore(vectorPath);
-    // Get all Kaggle dataset IDs
-    const kaggleIds = metadataStore.getDatasetIdsBySource("kaggle");
-    console.log(`Found ${kaggleIds.length} Kaggle datasets in database`);
-    if (kaggleIds.length === 0) {
-        console.log("✅ No Kaggle datasets to remove");
-        process.exit(0);
-    }
-    // Delete from vector store
-    const vectorsDeleted = vectorStore.deleteMany(kaggleIds);
-    console.log(`🗑️  Deleted ${vectorsDeleted} vectors from vector store`);
-    vectorStore.save();
-    // Delete from metadata database
-    const datasetsDeleted = metadataStore.deleteBySource("kaggle");
-    console.log(`🗑️  Deleted ${datasetsDeleted} datasets from metadata database`);
-    metadataStore.close();
-    console.log("\n✅ Cleanup complete! Kaggle datasets have been removed.");
-    console.log("   You can now search without seeing Kaggle results.");
-}
-catch (error) {
-    console.error("❌ Cleanup failed:", error.message);
-    process.exit(1);
-}

package/build/scripts/demo-full-pipeline.js

@@ -1,62 +0,0 @@
-import { PipelineExecutor } from "../cleaning/executor.js";
-import { ScriptGenerator } from "../cleaning/exporter.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log(" Vesper Dataset Ops Engine: Full Demo\n");
-    const executor = new PipelineExecutor();
-    const exporter = new ScriptGenerator();
-    const demoFile = path.join(process.cwd(), "vesper_demo_data.csv");
-    // 1. Create a Realistic Dirty Dataset
-    // - duplicate: Duplicate Customer
-    // - age: Mixed types ("25", "twenty"), Outliers (200)
-    // - email: PII
-    // - empty_col: 100% missing
-    // - score: Good data
-    const csvContent = `customer_id,age,email,score,empty_col
-C001,25,john.doe@example.com,88.5,
-C002,"30",jane.smith@work.org,92.0,
-C003,200,bob.jones@gmail.com,15.0,
-C001,25,john.doe@example.com,88.5,
-C004,"forty",alice@co.uk,80.0,
-C005,35,,75.0,`;
-    fs.writeFileSync(demoFile, csvContent);
-    console.log(`📦 Created dirty dataset: ${demoFile}`);
-    console.log(`Contains: Duplicates, PII (Emails), Mixed Types (Age), Outliers, Empty Columns.\n`);
-    try {
-        // 2. Run the Auto-Cleaning Pipeline
-        console.log(" Running Auto-Cleaning Pipeline...");
-        const result = await executor.runPipeline("demo-dataset", demoFile);
-        console.log("\n --- Quality Inspection Report ---");
-        console.log(`   Duplicates: ${result.initial_quality.duplicate_rows} rows`);
-        console.log(`   PII Warnings: ${result.initial_quality.pii_warnings?.length || 0}`);
-        if (result.initial_quality.schema_warnings.length > 0) {
-            console.log("   Schema Issues:");
-            result.initial_quality.schema_warnings.forEach(w => console.log(`     ⚠️  ${w}`));
-        }
-        console.log("\n --- Generated Cleaning Plan ---");
-        result.plan.operations.forEach((op, i) => {
-            console.log(`   ${i + 1}. [${op.type}] Reason: ${op.reason}`);
-        });
-        console.log("\n --- Execution Result ---");
-        if (result.cleaning_result.success) {
-            console.log(`    Success! Cleaned file saved to:`);
-            console.log(`      ${result.final_output_path}`);
-        }
-        else {
-            console.error(`    Failed: ${result.cleaning_result.error}`);
-        }
-        // 3. Generate Reproducibility Script
-        console.log("\n --- Reproducibility ---");
-        const pythonScript = exporter.generatePythonScript(result.plan, demoFile);
-        const scriptPath = path.join(process.cwd(), "demo_cleaning_script.py");
-        fs.writeFileSync(scriptPath, pythonScript);
-        console.log(`    Generated Python script: ${scriptPath}`);
-        console.log(`   (You can run this script independently to reproduce these steps!)`);
-        console.log("\n Demo Complete.");
-    }
-    catch (error) {
-        console.error("\n Demo failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/demo-ui.js

@@ -1,58 +0,0 @@
-import { SearchEngine } from "../search/engine.js";
-import { MetadataStore } from "../metadata/store.js";
-import { VectorStore } from "../search/vector-store.js";
-import { Embedder } from "../search/embedder.js";
-import { formatSearchResults, formatDatasetInfo } from "../tools/formatter.js";
-import path from "path";
-/**
- * Demo script to showcase the new formatted UI
- */
-async function main() {
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    const vectorPath = path.join(process.cwd(), "data", "vectors.json");
-    const store = new MetadataStore(dbPath);
-    const vectorStore = new VectorStore(vectorPath);
-    const embedder = Embedder.getInstance();
-    await embedder.init();
-    const engine = new SearchEngine(store, vectorStore, embedder);
-    console.log("\n" + "═".repeat(80));
-    console.log("VESPER UI DEMO - Formatted Search Results");
-    console.log("═".repeat(80) + "\n");
-    // Demo 1: Medical datasets
-    console.log("Demo 1: Medical Dataset Search\n");
-    const medicalResults = await engine.search("diabetes prediction machine learning", {
-        limit: 3,
-        safeOnly: true
-    });
-    console.log(formatSearchResults(medicalResults));
-    // Demo 2: Detailed dataset info
-    if (medicalResults.length > 0) {
-        console.log("\n" + "═".repeat(80));
-        console.log("Demo 2: Detailed Dataset Information");
-        console.log("═".repeat(80) + "\n");
-        console.log(formatDatasetInfo(medicalResults[0]));
-    }
-    // Demo 3: Computer Vision
-    console.log("\n" + "═".repeat(80));
-    console.log("Demo 3: Computer Vision Dataset Search");
-    console.log("═".repeat(80) + "\n");
-    const cvResults = await engine.search("image classification cats dogs", {
-        limit: 3,
-        safeOnly: true
-    });
-    console.log(formatSearchResults(cvResults));
-    // Demo 4: Show quality warnings
-    console.log("\n" + "═".repeat(80));
-    console.log("Demo 4: Niche Query with Quality Warnings");
-    console.log("═".repeat(80) + "\n");
-    const nicheResults = await engine.search("ancient manuscript text recognition", {
-        limit: 3,
-        safeOnly: true
-    });
-    console.log(formatSearchResults(nicheResults));
-    console.log("\n" + "═".repeat(80));
-    console.log("Demo Complete!");
-    console.log("═".repeat(80) + "\n");
-    store.close();
-}
-main().catch(console.error);

package/build/scripts/e2e-demo.js

@@ -1,72 +0,0 @@
-import { QualityAnalyzer } from "../quality/analyzer.js";
-import { PipelineExecutor } from "../cleaning/executor.js";
-import { DataSplitter } from "../splitting/splitter.js";
-import { MetadataPackager } from "../export/packager.js";
-import { DataExporter } from "../export/exporter.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log(" Vesper Data Ops Engine - End-to-End Pipeline Demo\n");
-    const sessionDir = path.join(process.cwd(), "e2e_demo_output");
-    if (fs.existsSync(sessionDir))
-        fs.rmSync(sessionDir, { recursive: true, force: true });
-    fs.mkdirSync(sessionDir);
-    const rawFile = path.join(sessionDir, "raw_data.csv");
-    // --- STEP 0: Create Raw "Dirty" Data ---
-    console.log(" Step 0: Initializing Raw Dataset...");
-    let content = "id,name,age,salary,joined_date\n";
-    content += "1,Alice,25,50000,2023-01-01\n";
-    content += "2,Bob,,60000,2023-01-05\n"; // Missing age
-    content += "3,Charlie,35,70000,2023-01-10\n";
-    content += "1,Alice,25,50000,2023-01-01\n"; // Duplicate
-    content += "4,Diana,40,invalid,2023-02-01\n"; // Type mismatch
-    fs.writeFileSync(rawFile, content);
-    // --- STEP 1: Quality Analysis ---
-    console.log(" Step 1: Running Quality Analysis...");
-    const analyzer = new QualityAnalyzer();
-    const report = await analyzer.analyze(rawFile);
-    console.log(`   - Quality Score: ${report.overall_score}/100`);
-    console.log(`   - Warnings Found: ${report.warnings.length}`);
-    // --- STEP 2: Auto-Cleaning ---
-    console.log(" Step 2: Generating and Executing Cleaning Pipeline...");
-    const executor = new PipelineExecutor();
-    const pipelineResult = await executor.runPipeline("demo-dataset", rawFile);
-    const cleanedFile = pipelineResult.final_output_path;
-    console.log(`   - Cleaned file: ${path.basename(cleanedFile)}`);
-    // --- STEP 3: Smart Splitting ---
-    console.log(" Step 3: Splitting into Train/Val/Test/Holdout (Stratified)...");
-    const splitter = new DataSplitter();
-    const splitResult = await splitter.split(cleanedFile, {
-        type: "random", // Using random since dummy data too small for stratified
-        ratios: { train: 0.6, val: 0.2, test: 0.1, holdout: 0.1 },
-        shuffle: true,
-        random_seed: 42
-    });
-    console.log(`   - Splits created: ${Object.keys(splitResult.paths).join(", ")}`);
-    // --- STEP 4: Export to Parquet ---
-    console.log(" Step 4: Exporting Final Train Set to Parquet...");
-    const exporter = new DataExporter();
-    const parquetFile = path.join(sessionDir, "train_final.parquet");
-    await exporter.export(splitResult.paths.train, parquetFile, "parquet");
-    console.log(`   - Exported: ${path.basename(parquetFile)}`);
-    // --- STEP 5: Metadata Packaging ---
-    console.log(" Step 5: Creating Final Data Package...");
-    const packager = new MetadataPackager();
-    const packageResult = await packager.createPackage(path.join(sessionDir, "vesper_package"), [
-        { path: parquetFile, format: "parquet", name: "train-set" },
-        { path: splitResult.paths.holdout, format: "csv", name: "holdout-set" }
-    ], {
-        name: "Vesper E2E Demo",
-        version: "1.0.0",
-        description: "Automatically cleaned and partitioned dataset.",
-        license: "MIT",
-        author: "Vesper Engine"
-    }, { qualityReport: report });
-    console.log(" Pipeline Finished Successfully!");
-    console.log(` Package Location: ${packageResult.packagePath}`);
-    console.log(` Manifest: datapackage.json created.`);
-}
-main().catch(err => {
-    console.error(" Pipeline Failed:", err);
-    process.exit(1);
-});

package/build/scripts/massive-scrape.js

@@ -1,103 +0,0 @@
-import { HuggingFaceScraper } from "../metadata/scraper.js";
-import { KaggleMetadataScraper } from "../metadata/kaggle-scraper.js";
-import { MetadataStore } from "../metadata/store.js";
-import path from "path";
-/**
- * Realistic massive scraper: Get maximum from HF + extensive Kaggle coverage
- * Phase 1: Bulk Discovery (Skeleton Indexing)
- * Phase 2: Kaggle Discovery
- */
-async function main() {
-    const scraper = new HuggingFaceScraper();
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    const store = new MetadataStore(dbPath);
-    const allDatasets = new Map();
-    try {
-        // Load existing datasets to avoid duplicates
-        console.error(`\nLoading existing datasets from database...`);
-        const existing = store.getAllDatasets();
-        for (const ds of existing) {
-            allDatasets.set(ds.id, ds);
-        }
-        console.error(`Found ${existing.length} existing datasets in database`);
-        // Phase 1: Bulk discovery from HuggingFace
-        console.error(`\nPhase 1: Bulk HuggingFace Discovery (Target: 30,000)`);
-        const hfToken = process.env.HF_TOKEN || process.env.HUGGINGFACE_TOKEN;
-        if (hfToken) {
-            console.error(`Using HuggingFace token (rate limits should be higher)`);
-        }
-        else {
-            console.error(`WARNING: No HF_TOKEN found. Bulk scraping may be slower.`);
-        }
-        const hfLimit = 30000;
-        const hfDatasets = await scraper.scrapeBulk(hfLimit);
-        let newHfCount = 0;
-        for (const ds of hfDatasets) {
-            if (!allDatasets.has(ds.id)) {
-                allDatasets.set(ds.id, ds);
-                newHfCount++;
-            }
-        }
-        console.error(`HuggingFace Bulk Discovery: ${newHfCount} new datasets (${allDatasets.size} total unique)`);
-        // Save HF progress
-        console.error(`Saving HF discovery results to database...`);
-        store.beginTransaction();
-        try {
-            for (const ds of hfDatasets) {
-                store.saveDataset(ds);
-            }
-            store.commit();
-        }
-        catch (e) {
-            store.rollback();
-            console.error("Failed to save HF discovery progress:", e);
-        }
-        // Phase 2: Extensive Kaggle scraping across many categories
-        const kaggleUser = process.env.KAGGLE_USERNAME;
-        const kaggleKey = process.env.KAGGLE_KEY;
-        if (kaggleUser && kaggleKey) {
-            console.error(`\nPhase 2: Extensive Kaggle scraping`);
-            const kaggleScraper = new KaggleMetadataScraper(kaggleUser, kaggleKey);
-            // Comprehensive Kaggle search terms
-            const kaggleSearches = [
-                "machine learning", "deep learning", "data science",
-                "classification", "regression", "clustering", "anomaly detection",
-                "natural language processing", "text classification", "sentiment analysis",
-                "image classification", "object detection", "medical imaging",
-                "time series", "forecasting", "financial forecasting",
-                "healthcare", "medical diagnosis", "e-commerce", "social media"
-            ];
-            for (const search of kaggleSearches) {
-                console.error(`  [Kaggle: "${search}"] Fetching...`);
-                try {
-                    const kaggleDatasets = await kaggleScraper.scrape(search, 100, true);
-                    let newKaggleCount = 0;
-                    for (const ds of kaggleDatasets) {
-                        ds.id = `kaggle:${ds.id}`;
-                        if (!allDatasets.has(ds.id)) {
-                            allDatasets.set(ds.id, ds);
-                            newKaggleCount++;
-                            store.saveDataset(ds); // Save individually to avoid long transactions
-                        }
-                    }
-                    console.error(`    ${newKaggleCount} new datasets (${kaggleDatasets.length} total fetched, ${allDatasets.size} total unique)`);
-                }
-                catch (e) {
-                    console.error(`    ERROR: ${e.message}`);
-                }
-                await new Promise(resolve => setTimeout(resolve, 2000));
-            }
-        }
-        console.error(`\nBulk Discovery Complete!`);
-        console.error(`Total unique datasets in library: ${allDatasets.size}`);
-        console.error(`\nNext step: Run 'npm run index' to update vectors.`);
-    }
-    catch (error) {
-        console.error("\nERROR: Massive scraping failed:", error);
-        process.exit(1);
-    }
-    finally {
-        store.close();
-    }
-}
-main();

package/build/scripts/ops-dashboard.js

@@ -1,33 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { JobManager } from "../jobs/manager.js";
-import path from "path";
-async function showDashboard() {
-    const dbPath = path.resolve("data", "vesper.db");
-    const store = new MetadataStore(dbPath);
-    // In a real app, this would be a shared instance in a long-running process
-    const manager = JobManager.getInstance(store);
-    const stats = manager.stats;
-    console.clear();
-    console.log("==========================================");
-    console.log("      VESPER OPERATIONS DASHBOARD         ");
-    console.log("==========================================\n");
-    const prometheus = stats.getPrometheusMetrics();
-    const summary = stats.getStats();
-    if (Object.keys(summary).length === 0) {
-        console.log("No metrics recorded yet. Process some jobs to see data.");
-    }
-    else {
-        console.log("--- SYSTEM METRICS (JSON) ---");
-        console.log(JSON.stringify(summary, null, 2));
-        console.log("\n--- PROMETHEUS EXPORT (RAW) ---");
-        console.log(prometheus);
-    }
-    console.log("\n==========================================");
-    console.log("  Press Ctrl+C to exit dashboard loop     ");
-    console.log("==========================================");
-    store.close();
-}
-// Simple loop for "real-time" feel (simulated)
-console.log("Starting Dashboard...");
-showDashboard();
-setInterval(showDashboard, 5000);

package/build/scripts/repro-bug.js

@@ -1,37 +0,0 @@
-import { JobManager } from "../jobs/manager.js";
-import { MetadataStore } from "../metadata/store.js";
-import fs from "fs";
-async function repro() {
-    const dbPath = "repro_test.db";
-    if (fs.existsSync(dbPath))
-        fs.unlinkSync(dbPath);
-    const store = new MetadataStore(dbPath);
-    const jobManager = JobManager.getInstance(store);
-    console.log("Setting up listener...");
-    jobManager.on("processJob", async (job, execute) => {
-        console.log(`Listener received job ${job.id}`);
-        const task = async () => {
-            console.log("Running task...");
-            return "success";
-        };
-        try {
-            await execute(task);
-            console.log("Execute finished");
-        }
-        catch (e) {
-            console.error("Execute failed in listener:", e.message);
-        }
-    });
-    console.log("Creating job...");
-    const job = jobManager.createJob("prepare", 0, { query: "test" });
-    console.log(`Job created: ${job.id}`);
-    // Wait for a bit
-    await new Promise(r => setTimeout(r, 2000));
-    const finalJob = store.getJob(job.id);
-    console.log("Final job status:", finalJob?.status);
-    console.log("Final job status text:", finalJob?.status_text);
-    store.close();
-    if (fs.existsSync(dbPath))
-        fs.unlinkSync(dbPath);
-}
-repro().catch(console.error);

package/build/scripts/repro-export-bug.js

@@ -1,56 +0,0 @@
-import path from "path";
-import fs from "fs";
-import { spawnSync } from "child_process";
-const pythonPath = "python";
-const scriptPath = path.join(process.cwd(), "src", "python", "cleaner.py");
-const testDir = path.join(process.cwd(), "test_repro");
-if (!fs.existsSync(testDir))
-    fs.mkdirSync(testDir);
-async function runRepro() {
-    console.log("=== Reproducing CSV Export Bug ===\n");
-    const parquetFile = path.join(testDir, "test_nested.parquet");
-    const csvOutput = path.join(testDir, "test_nested_cleaned.csv");
-    // 1. Create a Parquet file with nested data (Lists/Structs) using Python
-    console.log("Creating nested Parquet file...");
-    const createScript = `
-import polars as pl
-df = pl.DataFrame({
-    "id": [1, 2, 3],
-    "tags": [["a", "b"], ["c"], []],
-    "meta": [{"score": 0.9, "safe": True}, {"score": 0.4, "safe": False}, {"score": 0.1, "safe": True}]
-})
-df.write_parquet(r"${parquetFile}")
-`;
-    fs.writeFileSync(path.join(testDir, "create_data.py"), createScript);
-    spawnSync(pythonPath, [path.join(testDir, "create_data.py")], { stdio: 'inherit' });
-    // 2. Call cleaner.py to convert to CSV
-    console.log("Calling cleaner.py to convert to CSV...");
-    const result = spawnSync(pythonPath, [
-        scriptPath,
-        parquetFile,
-        "[]",
-        "csv"
-    ]);
-    console.log("Exit Code:", result.status);
-    console.log("Stdout:", result.stdout?.toString());
-    console.log("Stderr:", result.stderr?.toString());
-    if (result.status === 0) {
-        try {
-            const data = JSON.parse(result.stdout.toString());
-            if (data.success) {
-                console.log("SUCCESS! Output file:", data.output_path);
-                if (fs.existsSync(data.output_path)) {
-                    console.log("File exists on disk.");
-                }
-            }
-            else {
-                console.error("cleaner.py reported failure:", data.error);
-            }
-        }
-        catch (e) {
-            console.error("Failed to parse JSON output:", e);
-        }
-    }
-}
-runRepro().catch(console.error);
-runRepro().catch(console.error);