vesper-wizard 2.0.5 → 2.0.7

package/build/cleaning/executor.js

@@ -0,0 +1,62 @@
+import { QualityAnalyzer } from "../quality/analyzer.js";
+import { CleaningPlanner } from "./planner.js";
+import { DataCleaner } from "./cleaner.js";
+export class PipelineExecutor {
+    analyzer;
+    planner;
+    cleaner;
+    constructor(dataRoot = process.cwd(), buildDir) {
+        // Use buildDir for analyzer and cleaner (where Python scripts are), dataRoot for data operations
+        this.analyzer = new QualityAnalyzer(undefined, buildDir || dataRoot);
+        this.planner = new CleaningPlanner(undefined, buildDir || dataRoot);
+        this.cleaner = new DataCleaner(buildDir || dataRoot);
+    }
+    /**
+     * Run the full Auto-Cleaning Pipeline on a dataset file
+     */
+    async runPipeline(datasetId, filePath, outputFormat = "parquet", onProgress) {
+        // ... (logging setup)
+        const log = (m) => {
+            console.error(`[Pipeline] ${m}`);
+            if (onProgress)
+                onProgress(m);
+        };
+        log(`Analyzing quality for ${datasetId}...`);
+        const qualityReport = await this.analyzer.analyze(filePath);
+        // 2. Generate Plan
+        log(`Generating cleaning plan...`);
+        const plan = await this.planner.generatePlan(datasetId, qualityReport);
+        const rules = plan.operations;
+        // If no cleaning needed, we still might need format conversion
+        const needsConversion = !filePath.toLowerCase().endsWith(`.${outputFormat}`);
+        if (rules.length === 0 && !needsConversion) {
+            log(`No cleaning or conversion needed.`);
+            return {
+                initial_quality: qualityReport,
+                plan,
+                cleaning_result: {
+                    success: true,
+                    rows_affected: 0,
+                    columns_affected: 0,
+                    logs: ["No operations generated."]
+                }
+            };
+        }
+        // 3. Execute Plan (includes conversion if requested)
+        log(`Executing ${rules.length} operations (Format: ${outputFormat})...`);
+        rules.forEach(op => console.error(`  - ${op.type}: ${op.reason}`));
+        const cleaningResult = await this.cleaner.clean(filePath, rules, outputFormat);
+        if (cleaningResult.success) {
+            log(`Cleaning complete. Output: ${cleaningResult.output_path}`);
+        }
+        else {
+            log(`Cleaning failed: ${cleaningResult.error}`);
+        }
+        return {
+            initial_quality: qualityReport,
+            plan,
+            cleaning_result: cleaningResult,
+            final_output_path: cleaningResult.output_path
+        };
+    }
+}

package/build/cleaning/exporter.js

@@ -0,0 +1,87 @@
+export class ScriptGenerator {
+    /**
+     * Generate a standalone Python script for the cleaning plan
+     */
+    generatePythonScript(plan, inputPath) {
+        const timestamp = new Date().toISOString().split('T')[0];
+        let script = `"""
+Vesper Auto-Cleaning Script
+Generated: ${timestamp}
+Dataset ID: ${plan.dataset_id}
+"""
+
+import polars as pl
+import numpy as np
+
+def clean_dataset(file_path):
+    print(f"Loading {file_path}...")
+    
+    # Load Data
+    if file_path.endswith(".csv"):
+        df = pl.read_csv(file_path, ignore_errors=True)
+    elif file_path.endswith(".parquet"):
+        df = pl.read_parquet(file_path)
+    else:
+        raise ValueError("Unsupported format")
+
+    print(f"Initial shape: {df.shape}")
+
+`;
+        // Generate code for each operation
+        plan.operations.forEach((op, index) => {
+            script += `    # Step ${index + 1}: ${op.type}\n`;
+            script += `    # Reason: ${op.reason}\n`;
+            script += this.generateOpCode(op);
+            script += `    print(f"After Step ${index + 1} (${op.type}): {df.shape}")\n\n`;
+        });
+        script += `    # Save Output
+    output_path = file_path.replace(".csv", "_cleaned_repro.csv").replace(".parquet", "_cleaned_repro.parquet")
+    if file_path.endswith(".csv"):
+        df.write_csv(output_path)
+    else:
+        df.write_parquet(output_path)
+        
+    print(f"Done! Saved to {output_path}")
+
+if __name__ == "__main__":
+    # Default input path from generation time, can be overridden
+    INPUT_PATH = r"${inputPath}"
+    clean_dataset(INPUT_PATH)
+`;
+        return script;
+    }
+    generateOpCode(op) {
+        const p = op.params;
+        switch (op.type) {
+            case "RemoveDuplicates":
+                return `    df = df.unique()\n`;
+            case "DropColumns":
+                return `    cols_to_drop = ${JSON.stringify(p.columns)}\n    existing_cols = [c for c in cols_to_drop if c in df.columns]\n    if existing_cols:\n        df = df.drop(existing_cols)\n`;
+            case "FillMissing":
+                if (p.method === "constant") {
+                    const val = typeof p.value === 'string' ? `"${p.value}"` : p.value;
+                    return `    df = df.with_columns(pl.col("${p.column}").fill_null(${val}))\n`;
+                }
+                else if (p.method === "mean") {
+                    return `    mean_val = df["${p.column}"].mean()\n    df = df.with_columns(pl.col("${p.column}").fill_null(mean_val))\n`;
+                }
+                else if (p.method === "median") {
+                    return `    median_val = df["${p.column}"].median()\n    df = df.with_columns(pl.col("${p.column}").fill_null(median_val))\n`;
+                }
+                return `    # Unknown fill method for ${p.column}\n`;
+            case "FixTypes":
+                if (p.type === "float")
+                    return `    df = df.with_columns(pl.col("${p.column}").cast(pl.Float64, strict=False))\n`;
+                if (p.type === "int")
+                    return `    df = df.with_columns(pl.col("${p.column}").cast(pl.Int64, strict=False))\n`;
+                if (p.type === "string")
+                    return `    df = df.with_columns(pl.col("${p.column}").cast(pl.Utf8))\n`;
+                return `    # Unknown type conversion for ${p.column}\n`;
+            case "RemoveOutliers":
+                // IQR implementation inline
+                return `    q1 = df["${p.column}"].quantile(0.25)\n    q3 = df["${p.column}"].quantile(0.75)\n    iqr = q3 - q1\n    lower = q1 - (1.5 * iqr)\n    upper = q3 + (1.5 * iqr)\n    df = df.filter((pl.col("${p.column}") >= lower) & (pl.col("${p.column}") <= upper))\n`;
+            default:
+                return `    # Operation ${op.type} not fully supported in export yet\n`;
+        }
+    }
+}

package/build/cleaning/planner.js

@@ -0,0 +1,127 @@
+import { TargetDetector } from "../preparation/target-detector.js";
+export class CleaningPlanner {
+    cache;
+    targetDetector;
+    constructor(cache, buildDir = process.cwd()) {
+        this.cache = cache;
+        this.targetDetector = new TargetDetector(buildDir);
+    }
+    /**
+     * Generate a cleaning plan based on the quality report and optional custom rules
+     */
+    async generatePlan(datasetId, report, ruleSet, targetInfo) {
+        if (this.cache) {
+            const cached = await this.cache.getPlan(datasetId, { report, ruleSet });
+            if (cached) {
+                console.error(`[CleaningPlanner] Cache hit for ${datasetId}`);
+                return cached;
+            }
+        }
+        const ops = [];
+        let estimatedRowsSaved = 0;
+        let estimatedColsSaved = 0;
+        // 1. Remove Duplicates (Global)
+        if (report.duplicate_rows > 0) {
+            ops.push({
+                type: "RemoveDuplicates",
+                params: {},
+                reason: `Found ${report.duplicate_rows} exact duplicate rows`
+            });
+            estimatedRowsSaved += report.duplicate_rows;
+        }
+        // 2. Column-level operations
+        for (const col of report.columns) {
+            // A. Drop Empty / Useless Columns
+            if (col.missing_percentage > 90 || col.is_constant) {
+                ops.push({
+                    type: "DropColumns",
+                    params: { columns: [col.name] },
+                    reason: col.is_constant ? "Column is constant (zero variance)" : `High missing values (${col.missing_percentage.toFixed(1)}%)`
+                });
+                estimatedColsSaved++;
+                continue;
+            }
+            // B. Fix Types
+            if (this.shouldFixType(col)) {
+                const targetType = col.inferred_type.toLowerCase().includes("numeric") ? "float" : "string";
+                ops.push({
+                    type: "FixTypes",
+                    params: { column: col.name, type: targetType },
+                    reason: `Inferred type is ${col.inferred_type} but stored as ${col.type}`
+                });
+            }
+            // C. Impute Missing Values
+            if (col.missing_count > 0) {
+                let method = "constant";
+                let value = "unknown";
+                if (col.inferred_type.includes("Numeric") || col.type.includes("Int") || col.type.includes("Float")) {
+                    method = "median";
+                    value = 0;
+                }
+                else {
+                    method = "constant";
+                    value = "missing";
+                }
+                ops.push({
+                    type: "FillMissing",
+                    params: { column: col.name, method, value },
+                    reason: `${col.missing_count} missing values`
+                });
+            }
+        }
+        // 3. Target Unification
+        if (targetInfo && targetInfo.target !== "target" && targetInfo.confidence > 0.7) {
+            ops.push({
+                type: "RenameTarget",
+                params: { old_name: targetInfo.target, new_name: "target" },
+                reason: `Detected target '${targetInfo.target}' with high confidence (${targetInfo.confidence.toFixed(2)})`
+            });
+            estimatedColsSaved++; // Logic fix: effectively "saving" a column by standardizing it
+        }
+        // 4. Apply Custom Rules
+        if (ruleSet) {
+            for (const rule of ruleSet.rules) {
+                const targets = rule.condition.column === "*"
+                    ? report.columns.map(c => c.name)
+                    : [rule.condition.column];
+                for (const targetCol of targets) {
+                    const colStats = report.columns.find(c => c.name === targetCol);
+                    if (!colStats)
+                        continue;
+                    ops.push({
+                        type: rule.action.type,
+                        params: { ...rule.action.params, column: targetCol },
+                        reason: `Custom Rule: ${rule.name} - ${rule.description}`
+                    });
+                }
+            }
+        }
+        const plan = {
+            dataset_id: datasetId,
+            operations: ops,
+            estimated_impact: {
+                rows_saved: estimatedRowsSaved,
+                columns_saved: estimatedColsSaved,
+                quality_score_improvement: 10 + (ops.length * 5)
+            }
+        };
+        if (this.cache) {
+            await this.cache.savePlan(datasetId, { report, ruleSet }, plan);
+        }
+        return plan;
+    }
+    async generateRules(datasetId, report, ruleSet, targetInfo) {
+        const plan = await this.generatePlan(datasetId, report, ruleSet, targetInfo);
+        return plan.operations;
+    }
+    shouldFixType(col) {
+        if (col.inferred_type && col.inferred_type.includes("Numeric") && (col.type.includes("String") || col.type.includes("Utf8"))) {
+            return true;
+        }
+        return false;
+    }
+    isNumeric(col) {
+        const t = col.type.toLowerCase();
+        return t.includes("int") || t.includes("float") || t.includes("numeric");
+    }
+}

package/build/cleaning/rules.js

@@ -0,0 +1,57 @@
+// --- Domain Presets ---
+export const NLP_PRESET = {
+    id: "preset-nlp",
+    name: "NLP Data Prep",
+    domain: "nlp",
+    rules: [
+        {
+            id: "nlp-1",
+            name: "Normalize Case",
+            description: "Convert all text to lowercase",
+            condition: { column: "*", operator: "is_null", value: false }, // Apply to all non-null if type is string (logic in evaluator)
+            action: { type: "NormalizeText", params: { case: "lower" } }
+        },
+        {
+            id: "nlp-2",
+            name: "Remove URLs",
+            description: "Strip http/https links",
+            condition: { column: "*", operator: "contains", value: "http" },
+            action: { type: "Replace", params: { pattern: "https?://\\S+", replacement: "" } }
+        }
+    ]
+};
+export const HEALTHCARE_PRESET = {
+    id: "preset-healthcare",
+    name: "Healthcare (HIPAA) Prep",
+    domain: "healthcare",
+    rules: [
+        {
+            id: "hc-1",
+            name: "Mask Emails",
+            description: "Identify and mask email addresses",
+            condition: { column: "*", operator: "contains", value: "@" },
+            action: { type: "CustomMask", params: { method: "hash", salt: "vesper-pii" } }
+        },
+        {
+            id: "hc-2",
+            name: "Normalize Dates",
+            description: "Ensure ISO-8601 for DOB/Admit dates",
+            condition: { column: "date", operator: "is_null", value: false },
+            action: { type: "FixTypes", params: { type: "date" } }
+        }
+    ]
+};
+export const FINANCE_PRESET = {
+    id: "preset-finance",
+    name: "Financial Data Prep",
+    domain: "finance",
+    rules: [
+        {
+            id: "fin-1",
+            name: "Currency Cleanup",
+            description: "Remove currency symbols and parse as float",
+            condition: { column: "amount", operator: "matches_regex", value: "[\\$\\€\\£]" },
+            action: { type: "FixTypes", params: { type: "float", strip: "[^0-9\\.]" } }
+        }
+    ]
+};

package/build/cleaning/types.js

@@ -0,0 +1 @@
+export {};

package/build/cloud/adapters/local.js

@@ -0,0 +1,37 @@
+import fs from "fs";
+import path from "path";
+/**
+ * LocalAdapter mimics cloud storage by copying files to a local vault directory.
+ */
+export class LocalAdapter {
+    vaultPath;
+    constructor(vaultPath) {
+        this.vaultPath = path.resolve(vaultPath);
+        if (!fs.existsSync(this.vaultPath)) {
+            fs.mkdirSync(this.vaultPath, { recursive: true });
+        }
+    }
+    async upload(localPath, remotePath) {
+        if (!fs.existsSync(localPath)) {
+            throw new Error(`Local file not found: ${localPath}`);
+        }
+        const destPath = path.join(this.vaultPath, remotePath);
+        const destDir = path.dirname(destPath);
+        if (!fs.existsSync(destDir)) {
+            fs.mkdirSync(destDir, { recursive: true });
+        }
+        fs.copyFileSync(localPath, destPath);
+        // Returns a file URI as the "url"
+        return `file://${destPath}`;
+    }
+    async delete(remotePath) {
+        const destPath = path.join(this.vaultPath, remotePath);
+        if (fs.existsSync(destPath)) {
+            fs.unlinkSync(destPath);
+        }
+    }
+    async getSignedUrl(remotePath, expiresValue) {
+        // For local, just return the file URI
+        return `file://${path.join(this.vaultPath, remotePath)}`;
+    }
+}

package/build/cloud/adapters/s3.js

@@ -0,0 +1,24 @@
+/**
+ * S3Adapter Stub.
+ * Note: Requires @aws-sdk/client-s3 to be installed for full functionality.
+ */
+export class S3Adapter {
+    bucket;
+    region;
+    credentials;
+    constructor(bucket, region, credentials) {
+        this.bucket = bucket;
+        this.region = region;
+        this.credentials = credentials;
+    }
+    async upload(localPath, remotePath) {
+        console.warn("S3Adapter: Full implementation requires @aws-sdk/client-s3. This is a stub.");
+        return `https://${this.bucket}.s3.${this.region}.amazonaws.com/${remotePath}`;
+    }
+    async delete(remotePath) {
+        console.warn("S3Adapter: Delete stub called.");
+    }
+    async getSignedUrl(remotePath) {
+        return `https://${this.bucket}.s3.${this.region}.amazonaws.com/${remotePath}?stub=true`;
+    }
+}

package/build/cloud/adapters/supabase.js

@@ -0,0 +1,49 @@
+import fs from "fs";
+import { readFile } from "fs/promises";
+import { createClient } from "@supabase/supabase-js";
+export class SupabaseAdapter {
+    bucket;
+    client;
+    constructor(bucket, supabaseUrl, supabaseServiceRoleKey) {
+        this.bucket = bucket;
+        const resolvedUrl = supabaseUrl || process.env.SUPABASE_URL;
+        const resolvedServiceRoleKey = supabaseServiceRoleKey || process.env.SUPABASE_SERVICE_ROLE_KEY;
+        if (!resolvedUrl || !resolvedServiceRoleKey) {
+            throw new Error("Supabase requires SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY");
+        }
+        this.client = createClient(resolvedUrl, resolvedServiceRoleKey, {
+            auth: {
+                persistSession: false,
+            },
+        });
+    }
+    async upload(localPath, remotePath) {
+        if (!fs.existsSync(localPath)) {
+            throw new Error(`Local file not found: ${localPath}`);
+        }
+        const fileBuffer = await readFile(localPath);
+        const { error } = await this.client.storage
+            .from(this.bucket)
+            .upload(remotePath, fileBuffer, { upsert: true });
+        if (error) {
+            throw new Error(`Supabase upload failed: ${error.message}`);
+        }
+        const { data } = this.client.storage.from(this.bucket).getPublicUrl(remotePath);
+        return data.publicUrl;
+    }
+    async delete(remotePath) {
+        const { error } = await this.client.storage.from(this.bucket).remove([remotePath]);
+        if (error) {
+            throw new Error(`Supabase delete failed: ${error.message}`);
+        }
+    }
+    async getSignedUrl(remotePath, expiresValue = 3600) {
+        const { data, error } = await this.client.storage
+            .from(this.bucket)
+            .createSignedUrl(remotePath, expiresValue);
+        if (error || !data?.signedUrl) {
+            throw new Error(`Supabase signed URL failed: ${error?.message || "No signed URL returned"}`);
+        }
+        return data.signedUrl;
+    }
+}

package/build/cloud/storage-manager.js

@@ -0,0 +1,26 @@
+import { LocalAdapter } from "./adapters/local.js";
+import { S3Adapter } from "./adapters/s3.js";
+import { SupabaseAdapter } from "./adapters/supabase.js";
+export class StorageManager {
+    /**
+     * Creates an adapter based on configuration
+     */
+    static createAdapter(config) {
+        switch (config.type) {
+            case "local":
+                return new LocalAdapter(config.options.basePath || "./storage_vault");
+            case "s3":
+                if (!config.options.bucket || !config.options.region) {
+                    throw new Error("S3 requires bucket and region");
+                }
+                return new S3Adapter(config.options.bucket, config.options.region, config.options.credentials);
+            case "supabase":
+                if (!config.options.bucket) {
+                    throw new Error("Supabase requires bucket");
+                }
+                return new SupabaseAdapter(config.options.bucket, config.options.supabaseUrl, config.options.supabaseServiceRoleKey);
+            default:
+                throw new Error(`Unsupported storage type: ${config.type}`);
+        }
+    }
+}

package/build/cloud/types.js

@@ -0,0 +1 @@
+export {};

package/build/compliance/service.js

@@ -0,0 +1,73 @@
+import { v4 as uuidv4 } from "uuid";
+export class ComplianceService {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    async verifyGDPR(dataset) {
+        const issues = [];
+        // 1. Check for PII metadata flag
+        if (dataset.has_personal_data) {
+            issues.push("Dataset explicitly flagged as containing personal data.");
+        }
+        // 2. Check quality warnings for PII
+        if (dataset.quality_warnings) {
+            const piiWarnings = dataset.quality_warnings.filter(w => w.toLowerCase().includes("pii") || w.toLowerCase().includes("personal"));
+            if (piiWarnings.length > 0) {
+                issues.push(...piiWarnings);
+            }
+        }
+        // 3. Check for consent record
+        const consent = this.store.getConsent(dataset.id);
+        if (!consent || !consent.consent_obtained) {
+            issues.push("No valid consent record found for this dataset.");
+        }
+        const result = {
+            dataset_id: dataset.id,
+            standard: "GDPR",
+            passed: issues.length === 0,
+            issues,
+            timestamp: new Date().toISOString()
+        };
+        this.store.saveCheck(result);
+        return result;
+    }
+    async verifyHIPAA(dataset) {
+        const issues = [];
+        // 1. De-identification check (simulated)
+        // If domain is medical but no de-identification flag exists
+        if (dataset.domain === "healthcare" || dataset.domain === "medical") {
+            if (dataset.has_personal_data) {
+                issues.push("Medical dataset contains personal data (not de-identified).");
+            }
+        }
+        const result = {
+            dataset_id: dataset.id,
+            standard: "HIPAA",
+            passed: issues.length === 0,
+            issues,
+            timestamp: new Date().toISOString()
+        };
+        this.store.saveCheck(result);
+        return result;
+    }
+    logOperation(userId, datasetId, operation, metadata = {}) {
+        const event = {
+            id: uuidv4(),
+            user_id: userId,
+            dataset_id: datasetId,
+            operation,
+            timestamp: new Date().toISOString(),
+            details: JSON.stringify(metadata)
+        };
+        this.store.saveAudit(event);
+    }
+    exportAuditLog(datasetId) {
+        const logs = this.store.getAuditLogs(datasetId);
+        if (logs.length === 0)
+            return "No audit logs found.";
+        const header = "ID,Timestamp,User,Operation,Dataset,Details\n";
+        const rows = logs.map(l => `${l.id},${l.timestamp},${l.user_id},${l.operation},${l.dataset_id},"${l.details.replace(/"/g, '""')}"`).join("\n");
+        return header + rows;
+    }
+}

package/build/compliance/store.js

@@ -0,0 +1,80 @@
+export class ComplianceStore {
+    db;
+    constructor(db) {
+        this.db = db;
+        this.init();
+    }
+    init() {
+        this.db.exec(`
+            CREATE TABLE IF NOT EXISTS audit_logs (
+                id TEXT PRIMARY KEY,
+                user_id TEXT,
+                dataset_id TEXT,
+                operation TEXT,
+                timestamp TEXT,
+                details TEXT
+            );
+
+            CREATE TABLE IF NOT EXISTS compliance_checks (
+                dataset_id TEXT,
+                standard TEXT,
+                passed BOOLEAN,
+                issues TEXT, -- JSON array
+                timestamp TEXT,
+                PRIMARY KEY (dataset_id, standard)
+            );
+
+            CREATE TABLE IF NOT EXISTS consent_records (
+                dataset_id TEXT PRIMARY KEY,
+                consent_obtained BOOLEAN,
+                source TEXT,
+                last_verified TEXT
+            );
+        `);
+    }
+    saveAudit(event) {
+        const stmt = this.db.prepare(`
+            INSERT INTO audit_logs (id, user_id, dataset_id, operation, timestamp, details)
+            VALUES (?, ?, ?, ?, ?, ?)
+        `);
+        stmt.run(event.id, event.user_id, event.dataset_id, event.operation, event.timestamp, event.details);
+    }
+    getAuditLogs(datasetId) {
+        let query = "SELECT * FROM audit_logs";
+        const params = [];
+        if (datasetId) {
+            query += " WHERE dataset_id = ?";
+            params.push(datasetId);
+        }
+        query += " ORDER BY timestamp DESC";
+        return this.db.prepare(query).all(...params);
+    }
+    saveCheck(result) {
+        const stmt = this.db.prepare(`
+            INSERT INTO compliance_checks (dataset_id, standard, passed, issues, timestamp)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(dataset_id, standard) DO UPDATE SET
+                passed=excluded.passed,
+                issues=excluded.issues,
+                timestamp=excluded.timestamp
+        `);
+        stmt.run(result.dataset_id, result.standard, result.passed ? 1 : 0, JSON.stringify(result.issues), result.timestamp);
+    }
+    saveConsent(record) {
+        const stmt = this.db.prepare(`
+            INSERT INTO consent_records (dataset_id, consent_obtained, source, last_verified)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(dataset_id) DO UPDATE SET
+                consent_obtained=excluded.consent_obtained,
+                source=excluded.source,
+                last_verified=excluded.last_verified
+        `);
+        stmt.run(record.dataset_id, record.consent_obtained ? 1 : 0, record.source, record.last_verified);
+    }
+    getConsent(datasetId) {
+        const row = this.db.prepare("SELECT * FROM consent_records WHERE dataset_id = ?").get(datasetId);
+        if (!row)
+            return null;
+        return { ...row, consent_obtained: Boolean(row.consent_obtained) };
+    }
+}

package/build/compliance/types.js

@@ -0,0 +1 @@
+export {};