vesper-wizard 2.3.1 → 2.3.2

package/build/scripts/scrape-metadata.js

@@ -1,100 +0,0 @@
-import { HuggingFaceScraper } from "../metadata/scraper.js";
-import { KaggleMetadataScraper } from "../metadata/kaggle-scraper.js";
-import { MetadataStore } from "../metadata/store.js";
-import path from "path";
-async function main() {
-    const scraper = new HuggingFaceScraper();
-    const dbPath = path.join(process.cwd(), "data", "metadata.db");
-    const store = new MetadataStore(dbPath);
-    // Get limit from command line args or default to 100
-    let limit = 100;
-    if (process.argv[2] && !process.argv[2].startsWith("-")) {
-        limit = parseInt(process.argv[2], 10);
-        if (isNaN(limit))
-            limit = 100;
-    }
-    // Check if MVP filters should be disabled (--no-mvp flag)
-    const applyMVPFilters = !process.argv.includes("--no-mvp");
-    // Get domain filter if specified (--domain medicine, --domain healthcare, etc.)
-    const domainArgIndex = process.argv.indexOf("--domain");
-    const domainFilter = domainArgIndex !== -1 && process.argv[domainArgIndex + 1]
-        ? process.argv[domainArgIndex + 1]
-        : undefined;
-    try {
-        const datasets = [];
-        const domainMsg = domainFilter ? `, domain: ${domainFilter}` : "";
-        // 1. HuggingFace
-        const hfLimit = limit > 1000 ? limit : 1000; // Aim for at least 1k for "massive"
-        console.error(`Scraping HF: ${hfLimit} datasets with MVP filters: ${applyMVPFilters}${domainMsg}`);
-        const hfDatasets = await scraper.scrape(hfLimit, applyMVPFilters, domainFilter);
-        datasets.push(...hfDatasets);
-        // 2. Kaggle (Optional - only if credentials provided)
-        const kaggleUser = process.env.KAGGLE_USERNAME;
-        const kaggleKey = process.env.KAGGLE_KEY;
-        if (kaggleUser && kaggleKey) {
-            const kaggleLimit = Math.max(100, Math.floor(hfLimit / 4)); // Balance HF/Kaggle ratio
-            console.error(`\nScraping Kaggle: searching for "${domainFilter || 'all'}" (limit: ${kaggleLimit})`);
-            const kaggleScraper = new KaggleMetadataScraper(kaggleUser, kaggleKey);
-            // For massive scraping, we might want to iterate through multiple common search terms if no domainFilter is set
-            const searchTerms = domainFilter ? [domainFilter] : ["machine learning", "data science", "nlp", "computer vision", "healthcare"];
-            const termLimit = Math.floor(kaggleLimit / searchTerms.length);
-            for (const term of searchTerms) {
-                console.error(`[Kaggle] Fetching "${term}"...`);
-                const kaggleDatasets = await kaggleScraper.scrape(term, termLimit);
-                kaggleDatasets.forEach(d => {
-                    d.id = `kaggle:${d.id}`;
-                    if (!datasets.some(existing => existing.id === d.id)) {
-                        datasets.push(d);
-                    }
-                });
-            }
-        }
-        else {
-            console.error("\n[Kaggle] Skipping (KAGGLE_USERNAME/KAGGLE_KEY not set).");
-        }
-        console.error(`\nTotal Scraped: ${datasets.length} datasets.`);
-        let saved = 0;
-        store.beginTransaction(); // Performance optimization for large batch
-        try {
-            for (const ds of datasets) {
-                store.saveDataset(ds);
-                saved++;
-                if (saved % 500 === 0)
-                    console.error(`[DB] Saved ${saved} records...`);
-            }
-            store.commit();
-        }
-        catch (e) {
-            store.rollback();
-            throw e;
-        }
-        console.error(`Successfully saved ${saved} datasets to MetadataStore.`);
-        console.error(`\nSummary:`);
-        console.error(`  - Total examples: ${datasets.reduce((sum, d) => sum + d.total_examples, 0).toLocaleString()}`);
-        console.error(`  - Safe sources: ${datasets.filter(d => d.is_safe_source).length}`);
-        console.error(`  - Structured datasets: ${datasets.filter(d => d.is_structured).length}`);
-        console.error(`  - Permissive licenses: ${datasets.filter(d => d.license.category === "safe").length}`);
-        // Show domain distribution
-        const domainCounts = new Map();
-        datasets.forEach(d => {
-            const domain = d.domain || "unknown";
-            domainCounts.set(domain, (domainCounts.get(domain) || 0) + 1);
-        });
-        if (domainCounts.size > 0) {
-            console.error(`  - Domains:`);
-            Array.from(domainCounts.entries())
-                .sort((a, b) => b[1] - a[1])
-                .forEach(([domain, count]) => {
-                console.error(`    ${domain}: ${count}`);
-            });
-        }
-    }
-    catch (error) {
-        console.error("Scraping failed:", error);
-        process.exit(1);
-    }
-    finally {
-        store.close();
-    }
-}
-main();

package/build/scripts/search-cli.js

@@ -1,26 +0,0 @@
-import path from "path";
-import { MetadataStore } from "../metadata/store.js";
-import { VectorStore } from "../search/vector-store.js";
-import { Embedder } from "../search/embedder.js";
-import { SearchEngine } from "../search/engine.js";
-const query = process.argv[2];
-if (!query) {
-    console.error("Usage: npx tsx src/scripts/search-cli.ts \"your query\"");
-    process.exit(1);
-}
-const dbPath = path.join(process.cwd(), "data", "metadata.db");
-const vectorPath = path.join(process.cwd(), "data", "vectors.json");
-const metadataStore = new MetadataStore(dbPath);
-const vectorStore = new VectorStore(vectorPath);
-const embedder = Embedder.getInstance();
-const searchEngine = new SearchEngine(metadataStore, vectorStore, embedder);
-async function run() {
-    console.error(`Searching for: "${query}"...`);
-    const results = await searchEngine.search(query, { limit: 5, safeOnly: true });
-    if (results.length === 0) {
-        console.log("No results found.");
-        return;
-    }
-    console.log(JSON.stringify(results, null, 2));
-}
-run().catch(console.error);

package/build/scripts/test-bias.js

@@ -1,45 +0,0 @@
-import { QualityAnalyzer } from "../quality/analyzer.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Bias Analysis Test ===\n");
-    const analyzer = new QualityAnalyzer();
-    const testFile = path.join(process.cwd(), "test_bias.csv");
-    // Create a CSV with severe class imbalance
-    // 'label': 90% 'A', 10% 'B' -> Should trigger warning
-    // 'gender': 50/50 -> No warning
-    let csvContent = "id,gender,label\n";
-    for (let i = 0; i < 95; i++)
-        csvContent += `${i},M,A\n`;
-    for (let i = 95; i < 100; i++)
-        csvContent += `${i},F,B\n`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created test file: ${testFile}`);
-    console.log("Expectations:");
-    console.log("  - Should warn about 'Severe imbalance' in 'label' column (95% A)");
-    console.log("  - Should NOT warn about 'gender' (skewed but just a test sample)");
-    try {
-        console.log("\nRunning analyzer...");
-        const report = await analyzer.analyze(testFile);
-        console.log("\n=== Bias Report ===");
-        if (report.class_imbalance_warnings && report.class_imbalance_warnings.length > 0) {
-            report.class_imbalance_warnings.forEach(w => console.log(`[!] ${w}`));
-        }
-        else {
-            console.log("No bias detected.");
-        }
-        const labelWarning = report.class_imbalance_warnings?.find(w => w.includes("'label'"));
-        if (labelWarning) {
-            console.log("\nTest passed! Detected imbalance.");
-        }
-        else {
-            console.error("\nTest failed! Did not detect imbalance.");
-        }
-        // Cleanup
-        fs.unlinkSync(testFile);
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-caching.js

@@ -1,51 +0,0 @@
-import { MockRedisProvider, CacheService } from "../cache/service.js";
-import { CDNService } from "../cache/cdn.js";
-import { CleaningPlanner } from "../cleaning/planner.js";
-async function runTest() {
-    console.log("--- Initializing Caching Layer Test ---");
-    const cacheProvider = new MockRedisProvider();
-    const cache = new CacheService(cacheProvider);
-    const planner = new CleaningPlanner(cache);
-    const cdn = new CDNService("data/cdn_mock", "https://cdn.vesper.ai");
-    const dsId = "test/cache-dataset";
-    const mockReport = {
-        row_count: 1000,
-        column_count: 2,
-        duplicate_rows: 5,
-        duplicate_percentage: 0.5,
-        columns: [
-            { name: "col1", type: "Utf8", inferred_type: "String", missing_count: 0, missing_percentage: 0, unique_count: 1000, is_constant: false, is_mixed_type: false }
-        ],
-        warnings: [],
-        schema_warnings: [],
-        overall_score: 80
-    };
-    // 1. Test Planning Cache
-    console.log("\n--- Testing Cleaning Plan Cache ---");
-    console.log("First Run (Cache Miss)...");
-    const start1 = Date.now();
-    await planner.generatePlan(dsId, mockReport);
-    console.log(`First run took ${Date.now() - start1}ms`);
-    console.log("Second Run (Cache Hit)...");
-    const start2 = Date.now();
-    await planner.generatePlan(dsId, mockReport);
-    console.log(`Second run took ${Date.now() - start2}ms (Expected to be near 0ms)`);
-    // 2. Test TTL / Hash mismatch
-    console.log("\n--- Testing Cache Sensitivity (Config Change) ---");
-    console.log("Running with a ruleSet (New Cache Key)...");
-    const start3 = Date.now();
-    await planner.generatePlan(dsId, mockReport, { id: "rules-1", name: "Custom", rules: [] });
-    console.log(`Config change run took ${Date.now() - start3}ms (Cache Miss)`);
-    // 3. Test CDN Upload
-    console.log("\n--- Testing CDN Mock ---");
-    const reportContent = JSON.stringify(mockReport, null, 2);
-    const url = await cdn.upload("report_latest.json", reportContent);
-    console.log(`Report uploaded to CDN: ${url}`);
-    if (url.startsWith("https://cdn.vesper.ai")) {
-        console.log("\n✅ Success: Caching and CDN layers verified.");
-    }
-    else {
-        console.error("\n❌ Failure: CDN URL generation mismatch.");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-cleaning.js

@@ -1,76 +0,0 @@
-import { DataCleaner } from "../cleaning/cleaner.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Auto-Cleaning Pipeline Test ===\n");
-    const cleaner = new DataCleaner();
-    const testFile = path.join(process.cwd(), "test_cleaning.csv");
-    // Create Dirty Data
-    // - id: Duplicate rows (1)
-    // - age: Missing values, Outlier (200), Wrong Type ("thirty")
-    // - score: Good float
-    // - unnecessary: Column to drop
-    const csvContent = `id,age,score,unnecessary
-1,25,88.5,trash
-2,,92.0,trash
-3,200,15.0,trash
-1,25,88.5,trash
-4,"30",80.0,trash`;
-    fs.writeFileSync(testFile, csvContent);
-    console.log(`Created dirty file: ${testFile}`);
-    // Define Cleaning Plan
-    const ops = [
-        {
-            type: "DropColumns",
-            params: { columns: ["unnecessary"] },
-            reason: "Not useful"
-        },
-        {
-            type: "RemoveDuplicates",
-            params: {},
-            reason: "Duplicate rows"
-        },
-        {
-            type: "FillMissing",
-            params: { column: "age", method: "constant", value: 0 },
-            reason: "Impute missing age"
-        },
-        // Note: Polars can't easily fix "thirty" to 30 automatically without specific logic, 
-        // so we just cast what we can. "30" string -> 30 int works.
-        {
-            type: "FixTypes",
-            params: { column: "age", type: "float" },
-            reason: "Convert age to number"
-        },
-        {
-            type: "RemoveOutliers",
-            params: { column: "age", method: "iqr", threshold: 1.5 },
-            reason: "Remove age > 100"
-        }
-    ];
-    try {
-        console.log("Executing cleaning plan...");
-        const result = await cleaner.clean(testFile, ops);
-        console.log("\n=== Cleaning Result ===");
-        console.log(`Success: ${result.success}`);
-        console.log(`Output: ${result.output_path || "None"}`);
-        console.log(`Rows Affected: ${result.rows_affected}`);
-        console.log("\nLogs:");
-        result.logs.forEach(l => console.log(`  - ${l}`));
-        // Validate Output File Exists
-        if (fs.existsSync(result.output_path)) {
-            console.log("\nConverted file created successfully.");
-            // Cleanup
-            fs.unlinkSync(testFile);
-            fs.unlinkSync(result.output_path);
-            console.log("Test passed!");
-        }
-        else {
-            console.error("\nTest failed! Output file missing.");
-        }
-    }
-    catch (error) {
-        console.error("\nTest failed:", error);
-    }
-}
-main().catch(console.error);

package/build/scripts/test-cloud-storage.js

@@ -1,48 +0,0 @@
-import { StorageManager } from "../cloud/storage-manager.js";
-import fs from "fs";
-import path from "path";
-async function main() {
-    console.log("=== Vesper Cloud Storage Test ===\n");
-    const vaultDir = path.join(process.cwd(), "test_vault");
-    const adapter = StorageManager.createAdapter({
-        type: "local",
-        options: { basePath: vaultDir }
-    });
-    const testFile = "test_upload.txt";
-    fs.writeFileSync(testFile, "Hello Cloud Storage!");
-    try {
-        console.log("Testing upload...");
-        const remotePath = "datasets/v1/test.txt";
-        const url = await adapter.upload(testFile, remotePath);
-        console.log("Upload URL:", url);
-        const expectedFile = path.join(vaultDir, remotePath);
-        if (fs.existsSync(expectedFile)) {
-            console.log("PASS: File successfully uploaded to local vault.");
-            const content = fs.readFileSync(expectedFile, "utf8");
-            if (content === "Hello Cloud Storage!") {
-                console.log("PASS: Content integrity verified.");
-            }
-        }
-        else {
-            console.error("FAIL: 0Uploaded file not found in vault.");
-        }
-        console.log("\nTesting signed URL...");
-        const signedUrl = await adapter.getSignedUrl(remotePath);
-        console.log("Signed URL:", signedUrl);
-        console.log("\nTesting deletion...");
-        await adapter.delete(remotePath);
-        if (!fs.existsSync(expectedFile)) {
-            console.log("PASS: File successfully deleted from vault.");
-        }
-    }
-    catch (e) {
-        console.error("FAIL: Cloud storage test failed:", e);
-    }
-    finally {
-        if (fs.existsSync(testFile))
-            fs.unlinkSync(testFile);
-        if (fs.existsSync(vaultDir))
-            fs.rmSync(vaultDir, { recursive: true, force: true });
-    }
-}
-main().catch(console.error);

package/build/scripts/test-compliance.js

@@ -1,58 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { ComplianceStore } from "../compliance/store.js";
-import { ComplianceService } from "../compliance/service.js";
-import path from "path";
-import fs from "fs";
-async function runTest() {
-    const dbPath = path.resolve("data", "test-compliance.db");
-    const metadataStore = new MetadataStore(dbPath);
-    const complianceStore = new ComplianceStore(metadataStore.db);
-    const service = new ComplianceService(complianceStore);
-    const dsId = "org/health-data";
-    const medDataset = {
-        id: dsId,
-        source: "huggingface",
-        name: "Patient Records (Mock)",
-        domain: "healthcare",
-        has_personal_data: true,
-        quality_warnings: ["Potential PII detected in column 'patient_name'"],
-        last_updated: new Date().toISOString(),
-        license: { id: "proprietary", category: "restricted", usage_restrictions: [], warnings: [] },
-        // ... rest
-    };
-    console.log("--- Initial Compliance Check (Expecting Failure) ---");
-    const gdpr1 = await service.verifyGDPR(medDataset);
-    console.log(`GDPR Passed: ${gdpr1.passed}`);
-    console.log("Issues:", JSON.stringify(gdpr1.issues, null, 2));
-    const hipaa1 = await service.verifyHIPAA(medDataset);
-    console.log(`HIPAA Passed: ${hipaa1.passed}`);
-    console.log("Issues:", JSON.stringify(hipaa1.issues, null, 2));
-    console.log("\n--- Logging Operations (Audit Trail) ---");
-    service.logOperation("admin-123", dsId, "ComplianceCheck", { result: "Failed" });
-    service.logOperation("admin-123", dsId, "Clean", { rules: "PII-Masking" });
-    console.log("\n--- Resolving Compliance Issues ---");
-    complianceStore.saveConsent({
-        dataset_id: dsId,
-        consent_obtained: true,
-        source: "patient-portal",
-        last_verified: new Date().toISOString()
-    });
-    // Simulate de-identification
-    const cleanDataset = { ...medDataset, has_personal_data: false, quality_warnings: [] };
-    const gdpr2 = await service.verifyGDPR(cleanDataset);
-    console.log(`GDPR Passed (After Fix): ${gdpr2.passed}`);
-    const hipaa2 = await service.verifyHIPAA(cleanDataset);
-    console.log(`HIPAA Passed (After Fix): ${hipaa2.passed}`);
-    console.log("\n--- Exporting Audit Log ---");
-    const csv = service.exportAuditLog(dsId);
-    fs.writeFileSync("compliance-audit.csv", csv);
-    console.log("Audit log saved to compliance-audit.csv");
-    console.log("Log Snippet:\n", csv.split("\n").slice(0, 3).join("\n"));
-    if (gdpr2.passed && hipaa2.passed && csv.includes("Clean")) {
-        console.log("\n✅ Success: Compliance checks and audit trail verified.");
-    }
-    else {
-        console.error("\n❌ Failure: Compliance workflow incomplete.");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-conversion.js

@@ -1,64 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-import { PipelineExecutor } from "../cleaning/executor.js";
-import { InstallService } from "../install/install-service.js";
-import path from "path";
-import fs from "fs";
-async function runTest() {
-    console.log("--- Testing Format Conversion Export ---");
-    const projectRoot = path.resolve(".");
-    const metadataStore = new MetadataStore(path.join(projectRoot, "data", "metadata.db"));
-    const pipelineExecutor = new PipelineExecutor(projectRoot);
-    const installService = new InstallService(projectRoot, metadataStore);
-    // 1. Create a mock JSON dataset
-    const datasetId = "test/json-dataset";
-    const rawDir = path.join(projectRoot, "data", "raw");
-    if (!fs.existsSync(rawDir))
-        fs.mkdirSync(rawDir, { recursive: true });
-    const jsonPath = path.join(rawDir, "test_data.json");
-    const testData = [
-        { id: 1, name: "Alice", value: 10.5 },
-        { id: 2, name: "Bob", value: 20.1 }
-    ];
-    fs.writeFileSync(jsonPath, JSON.stringify(testData));
-    // Register in local_files
-    metadataStore.registerDownload(datasetId, jsonPath, "completed", fs.statSync(jsonPath).size);
-    // Register in datasets to satisfy inner check
-    metadataStore.saveDataset({
-        id: datasetId,
-        source: "huggingface",
-        name: "Test JSON",
-        description: "A test JSON file",
-        license: { category: "safe", id: "mit", usage_restrictions: [], warnings: [] },
-        last_updated: new Date().toISOString(),
-        quality_score: 80,
-        download_url: "http://example.com",
-        quality_warnings: []
-    });
-    console.log("Mock dataset registered.");
-    // 2. Simulate export_dataset with format="csv"
-    console.log(`Converting ${jsonPath} to CSV...`);
-    const downloadStatus = metadataStore.getDownloadStatus(datasetId);
-    if (!downloadStatus)
-        throw new Error("Dataset not found in DB");
-    let sourcePath = downloadStatus.local_path;
-    const requestedFormat = "csv";
-    const currentExt = path.extname(sourcePath).substring(1).toLowerCase();
-    if (currentExt !== requestedFormat) {
-        console.log(`Mismatch detected. Running pipeline conversion...`);
-        const result = await pipelineExecutor.runPipeline(datasetId, sourcePath, requestedFormat);
-        sourcePath = result.final_output_path;
-    }
-    // 3. Install
-    const targetDir = path.join(projectRoot, "test-conversion-export");
-    const finalPath = await installService.install(datasetId, sourcePath, targetDir);
-    console.log(`✅ Final Export Path: ${finalPath}`);
-    if (finalPath.endsWith(".csv") && fs.existsSync(finalPath)) {
-        const content = fs.readFileSync(finalPath, "utf8");
-        console.log("CSV Content Preview:\n" + content);
-        console.log("\nVERIFICATION_STATUS: ✅ PASS");
-    }
-    else {
-        console.error("\nVERIFICATION_STATUS: ❌ FAIL - Result is not a CSV or file missing");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-custom-rules.js

@@ -1,58 +0,0 @@
-import { CleaningPlanner } from "../cleaning/planner.js";
-import { NLP_PRESET, HEALTHCARE_PRESET } from "../cleaning/rules.js";
-import { RuleEvaluator } from "../cleaning/evaluator.js";
-async function runTest() {
-    const planner = new CleaningPlanner();
-    const evaluator = new RuleEvaluator();
-    const mockReport = {
-        row_count: 1000,
-        column_count: 3,
-        duplicate_rows: 0,
-        duplicate_percentage: 0,
-        columns: [
-            { name: "text", type: "Utf8", inferred_type: "String", missing_count: 0, missing_percentage: 0, unique_count: 1000, is_constant: false, is_mixed_type: false },
-            { name: "email", type: "Utf8", inferred_type: "String", missing_count: 0, missing_percentage: 0, unique_count: 1000, is_constant: false, is_mixed_type: false },
-            { name: "date", type: "Utf8", inferred_type: "Date", missing_count: 0, missing_percentage: 0, unique_count: 1000, is_constant: false, is_mixed_type: false }
-        ],
-        warnings: [],
-        schema_warnings: [],
-        overall_score: 90
-    };
-    console.log("--- Testing NLP Preset ---");
-    const nlpPlan = await planner.generatePlan("nlp-ds", mockReport, NLP_PRESET);
-    console.log(`Plan generated with ${nlpPlan.operations.length} operations.`);
-    nlpPlan.operations.forEach((op) => console.log(`- [${op.type}] ${op.reason}`));
-    console.log("\n--- Testing Healthcare Preset ---");
-    const hcPlan = await planner.generatePlan("hc-ds", mockReport, HEALTHCARE_PRESET);
-    console.log(`Plan generated with ${hcPlan.operations.length} operations.`);
-    hcPlan.operations.forEach((op) => console.log(`- [${op.type}] ${op.reason}`));
-    console.log("\n--- Testing RuleEvaluator Logic ---");
-    const record = {
-        text: "Check out https://google.com",
-        email: "user@example.com",
-        date: "2023-01-01"
-    };
-    console.log("Original Record:", JSON.stringify(record));
-    let processed = { ...record };
-    // Apply NLP Rules
-    for (const rule of NLP_PRESET.rules) {
-        if (evaluator.matches(processed, rule.condition)) {
-            processed = evaluator.apply(processed, rule);
-        }
-    }
-    // Apply Healthcare Rules
-    for (const rule of HEALTHCARE_PRESET.rules) {
-        if (evaluator.matches(processed, rule.condition)) {
-            processed = evaluator.apply(processed, rule);
-        }
-    }
-    console.log("Processed Record:", JSON.stringify(processed));
-    if (processed.text === "check out " && processed.email.includes("...")) {
-        console.log("\n✅ Success: Rules correctly applied and record transformed.");
-    }
-    else {
-        console.error("\n❌ Failure: Record transformation mismatch.");
-        console.log("Expected text to be lowercased and URL removed, and email to be masked.");
-    }
-}
-runTest().catch(console.error);

package/build/scripts/test-db-opt.js

@@ -1,63 +0,0 @@
-import { MetadataStore } from "../metadata/store.js";
-async function runTest() {
-    console.log("--- Initializing Database Optimization Test ---");
-    const store = new MetadataStore("data/vesper_test_opt.db");
-    // 1. Check if jobs_archive table exists
-    console.log("Verifying tables and indexes...");
-    // 2. Insert mock jobs (some old, some new)
-    const now = new Date();
-    const oldDate = new Date();
-    oldDate.setDate(now.getDate() - 40); // 40 days ago
-    const jobs = [
-        {
-            id: "job-new-1",
-            type: "clean",
-            status: "completed",
-            priority: 0,
-            progress: 100,
-            status_text: "Done",
-            attempts: 1,
-            max_attempts: 3,
-            created_at: now.toISOString(),
-            updated_at: now.toISOString()
-        },
-        {
-            id: "job-old-1",
-            type: "prepare",
-            status: "completed",
-            priority: 0,
-            progress: 100,
-            status_text: "Archivable",
-            attempts: 1,
-            max_attempts: 3,
-            created_at: oldDate.toISOString(),
-            updated_at: oldDate.toISOString()
-        }
-    ];
-    console.log("Inserting mock jobs...");
-    jobs.forEach(j => store.saveJob(j));
-    // 3. Test Archiving
-    console.log("\n--- Testing Archiving Logic (cutoff 30 days) ---");
-    const archivedCount = store.archiveOldJobs(30);
-    console.log(`Archived ${archivedCount} jobs (indicator).`);
-    // Verify
-    const newJob = store.getJob("job-new-1");
-    const oldJob = store.getJob("job-old-1");
-    if (newJob)
-        console.log("✅ New job remains in active table.");
-    if (!oldJob)
-        console.log("✅ Old job removed from active table.");
-    // Manually check archive via query if possible (not exposed, but we can check if it failed)
-    // 4. Test Optimization
-    console.log("\n--- Testing Maintenance Methods ---");
-    try {
-        store.optimize();
-        console.log("✅ Vacuum and Analyze completed.");
-    }
-    catch (e) {
-        console.error("❌ Optimization failed:", e);
-    }
-    console.log("\n✅ Success: Database optimizations verified.");
-    store.close();
-}
-runTest().catch(console.error);

package/build/scripts/test-export-custom.js

@@ -1,33 +0,0 @@
-import { InstallService } from "../install/install-service.js";
-import { MetadataStore } from "../metadata/store.js";
-import path from "path";
-import fs from "fs";
-async function runTest() {
-    console.log("--- Testing Custom Export Path ---");
-    const projectRoot = path.resolve(".");
-    const metadataStore = new MetadataStore(path.join(projectRoot, "data", "metadata.db"));
-    const installService = new InstallService(projectRoot, metadataStore);
-    // Find a naruto dataset
-    const dataset = metadataStore.db.prepare("SELECT * FROM datasets WHERE name LIKE '%naruto%' LIMIT 1").get();
-    if (!dataset) {
-        console.error("Naruto dataset not found. Please run a search first.");
-        return;
-    }
-    const customDir = path.join(projectRoot, "naruto-quotes");
-    const mockFile = path.join(projectRoot, "data", "raw", "naruto_test_export.csv");
-    if (!fs.existsSync(path.dirname(mockFile))) {
-        fs.mkdirSync(path.dirname(mockFile), { recursive: true });
-    }
-    fs.writeFileSync(mockFile, "quote,character\nBelieve it!,Naruto");
-    console.log(`Exporting ${dataset.id} to ${customDir}...`);
-    const finalPath = await installService.install(dataset.id, mockFile, customDir);
-    console.log(`✅ Success! Exported to: ${finalPath}`);
-    if (fs.existsSync(finalPath) && finalPath.includes("naruto-quotes")) {
-        console.log("Path verification: PASSED");
-    }
-    else {
-        console.error("Path verification: FAILED");
-    }
-    console.log("\n--- Test Complete ---");
-}
-runTest().catch(console.error);