vesper-wizard 2.3.1 → 2.3.2

package/build/export/exporter.js

@@ -1,82 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-import { ensurePythonPackages, resolvePythonCommand } from "../utils/python-runtime.js";
-export class DataExporter {
-    buildDir;
-    scriptPath;
-    constructor(buildDir = process.cwd()) {
-        this.buildDir = buildDir;
-        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "export_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "export_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "export_engine.py");
-        const scriptPath3 = path.resolve(buildDir, "..", "python", "export_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else if (fs.existsSync(scriptPath3)) {
-            this.scriptPath = scriptPath3;
-        }
-        else {
-            this.scriptPath = scriptPath0;
-        }
-    }
-    /**
-     * Exports a dataset file to a specified format
-     */
-    async export(inputFile, outputFile, format, options = {}) {
-        const pythonRequirements = [
-            { module: "polars", packageName: "polars" },
-        ];
-        if (format === "feather") {
-            pythonRequirements.push({ module: "pyarrow", packageName: "pyarrow" });
-        }
-        if (format === "tfrecord") {
-            pythonRequirements.push({ module: "tensorflow", packageName: "tensorflow" });
-        }
-        const pythonPath = await ensurePythonPackages(this.buildDir, pythonRequirements).catch(() => resolvePythonCommand(this.buildDir));
-        return new Promise((resolve, reject) => {
-            if (!fs.existsSync(inputFile)) {
-                reject(new Error(`Input file not found: ${inputFile}`));
-                return;
-            }
-            const args = [this.scriptPath, inputFile, outputFile, format, JSON.stringify(options)];
-            const childProcess = spawn(pythonPath, args, {
-                env: {
-                    ...process.env,
-                    PYTHONIOENCODING: "utf-8",
-                },
-            });
-            let stdout = "";
-            let stderr = "";
-            childProcess.stdout.on("data", (data) => stdout += data.toString());
-            childProcess.stderr.on("data", (data) => stderr += data.toString());
-            childProcess.on("close", (code) => {
-                if (code !== 0) {
-                    reject(new Error(`Export failed: ${stderr || stdout}`));
-                    return;
-                }
-                try {
-                    const result = JSON.parse(stdout);
-                    if (result.error) {
-                        reject(new Error(result.error));
-                    }
-                    else {
-                        resolve(result);
-                    }
-                }
-                catch (e) {
-                    reject(new Error(`Failed to parse export output: ${stdout}`));
-                }
-            });
-        });
-    }
-}

package/build/export/packager.js

@@ -1,100 +0,0 @@
-import fs from "fs";
-import path from "path";
-import crypto from "crypto";
-/**
- * MetadataPackager bundles data files with metadata and quality reports.
- * Follows a simplified Frictionless Data Package standard.
- */
-export class MetadataPackager {
-    /**
-     * Creates a data package in the target directory
-     */
-    async createPackage(targetDir, dataFiles, metadata, extraArtifacts) {
-        try {
-            if (!fs.existsSync(targetDir)) {
-                fs.mkdirSync(targetDir, { recursive: true });
-            }
-            const resources = [];
-            // 1. Process Data Files
-            for (const file of dataFiles) {
-                if (!fs.existsSync(file.path)) {
-                    throw new Error(`Data file not found: ${file.path}`);
-                }
-                const fileName = path.basename(file.path);
-                const destPath = path.join(targetDir, fileName);
-                // Copy file to package dir
-                fs.copyFileSync(file.path, destPath);
-                // Compute hash and size
-                const fileBuffer = fs.readFileSync(destPath);
-                const hash = crypto.createHash("sha256").update(fileBuffer).digest("hex");
-                const stats = fs.statSync(destPath);
-                resources.push({
-                    name: file.name,
-                    path: fileName,
-                    format: file.format,
-                    mediatype: this.getMediaType(file.format),
-                    bytes: stats.size,
-                    hash: `sha256:${hash}`
-                });
-            }
-            // 2. Add Extra Artifacts
-            if (extraArtifacts?.qualityReport) {
-                const qrPath = path.join(targetDir, "quality_report.json");
-                fs.writeFileSync(qrPath, JSON.stringify(extraArtifacts.qualityReport, null, 2));
-                resources.push({
-                    name: "quality-report",
-                    path: "quality_report.json",
-                    format: "json"
-                });
-            }
-            if (extraArtifacts?.cleaningLog) {
-                const clPath = path.join(targetDir, "cleaning_log.json");
-                fs.writeFileSync(clPath, JSON.stringify(extraArtifacts.cleaningLog, null, 2));
-                resources.push({
-                    name: "cleaning-log",
-                    path: "cleaning_log.json",
-                    format: "json"
-                });
-            }
-            // 3. Generate Manifest (datapackage.json)
-            const manifest = {
-                profile: "tabular-data-package",
-                name: metadata.name.toLowerCase().replace(/\s+/g, "-"),
-                title: metadata.name,
-                description: metadata.description,
-                version: metadata.version,
-                homepage: "https://github.com/vesper-data",
-                license: metadata.license,
-                author: metadata.author,
-                keywords: metadata.tags,
-                created: new Date().toISOString(),
-                resources: resources
-            };
-            const manifestPath = path.join(targetDir, "datapackage.json");
-            fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
-            return {
-                success: true,
-                packagePath: targetDir,
-                manifest: manifest
-            };
-        }
-        catch (e) {
-            return {
-                success: false,
-                packagePath: targetDir,
-                manifest: null,
-                error: e.message
-            };
-        }
-    }
-    getMediaType(format) {
-        switch (format.toLowerCase()) {
-            case "csv": return "text/csv";
-            case "parquet": return "application/x-parquet";
-            case "jsonl": return "application/x-jsonlines";
-            case "arrow": return "application/x-apache-arrow-file";
-            case "json": return "application/json";
-            default: return "application/octet-stream";
-        }
-    }
-}

package/build/export/types.js

@@ -1 +0,0 @@
-export {};

package/build/fusion/aligner.js

@@ -1,56 +0,0 @@
-export class SchemaAligner {
-    config;
-    constructor(config) {
-        this.config = config;
-    }
-    /**
-     * Finds common columns across datasets and creates a mapping to canonical names.
-     */
-    align(datasets) {
-        const alignment = {};
-        for (const ds of datasets) {
-            alignment[ds.id] = {};
-            if (!ds.columns)
-                continue;
-            for (const col of ds.columns) {
-                const canonicalName = this.getCanonicalName(col.name);
-                if (canonicalName) {
-                    alignment[ds.id][col.name] = canonicalName;
-                }
-            }
-        }
-        return alignment;
-    }
-    /**
-     * Resolves the most specific common type for a canonical column.
-     */
-    resolveType(canonicalName, types) {
-        if (this.config.type_overrides[canonicalName]) {
-            return this.config.type_overrides[canonicalName];
-        }
-        const uniqueTypes = new Set(types.map(t => t.toLowerCase()));
-        if (uniqueTypes.size === 1) {
-            return Array.from(uniqueTypes)[0];
-        }
-        if (uniqueTypes.has("float") || uniqueTypes.has("number")) {
-            return "number";
-        }
-        return "string"; // Default to string for mixed types
-    }
-    getCanonicalName(colName) {
-        // Direct match with any canonical name
-        for (const canonical in this.config.column_aliases) {
-            if (canonical.toLowerCase() === colName.toLowerCase()) {
-                return canonical;
-            }
-            // Match with aliases
-            const aliases = this.config.column_aliases[canonical];
-            if (aliases.some((a) => a.toLowerCase() === colName.toLowerCase())) {
-                return canonical;
-            }
-        }
-        // If no alias, use the name itself if it's already one of the canonical names we expect
-        // This is a bit recursive, let's simplify.
-        return null;
-    }
-}

package/build/fusion/deduplicator.js

@@ -1,69 +0,0 @@
-import * as crypto from "crypto";
-export class Deduplicator {
-    config;
-    seenHashes = new Set();
-    seenTexts = new Map(); // column -> tokens[]
-    constructor(config) {
-        this.config = config;
-    }
-    /**
-     * Checks if a record is an exact duplicate based on all columns.
-     */
-    isExactDuplicate(record) {
-        if (!this.config.dedupe_config.exact)
-            return false;
-        const hash = crypto
-            .createHash("md5")
-            .update(JSON.stringify(record))
-            .digest("hex");
-        if (this.seenHashes.has(hash)) {
-            return true;
-        }
-        this.seenHashes.add(hash);
-        return false;
-    }
-    /**
-     * Checks if a record is a fuzzy duplicate based on configured columns.
-     */
-    isFuzzyDuplicate(record) {
-        if (!this.config.dedupe_config.fuzzy)
-            return false;
-        if (!this.config.dedupe_config.fuzzy_columns.length)
-            return false;
-        for (const col of this.config.dedupe_config.fuzzy_columns) {
-            const text = String(record[col] || "");
-            if (!text || text.length < 10)
-                continue; // Skip short/empty tags
-            const tokens = this.tokenize(text);
-            if (this.isSimilar(col, tokens)) {
-                return true;
-            }
-            // In a real implementation, we'd use a more efficient data structure (like LSH)
-            // For now, we store tokens and compare (O(N^2) in worst case, use with caution)
-            // But we'll keep it simple for this phase.
-            // Actually, let's just store the tokens.
-        }
-        return false;
-    }
-    tokenize(text) {
-        return new Set(text.toLowerCase().split(/\s+/).filter(t => t.length > 2));
-    }
-    isSimilar(column, tokens) {
-        if (!this.seenTexts.has(column)) {
-            this.seenTexts.set(column, []);
-        }
-        const stored = this.seenTexts.get(column);
-        for (const existingSerialized of stored) {
-            const existing = new Set(JSON.parse(existingSerialized));
-            const intersection = new Set([...tokens].filter(t => existing.has(t)));
-            const union = new Set([...tokens, ...existing]);
-            const jaccard = intersection.size / union.size;
-            if (jaccard >= this.config.dedupe_config.fuzzy_threshold) {
-                return true;
-            }
-        }
-        // Add to seen
-        stored.push(JSON.stringify([...tokens]));
-        return false;
-    }
-}

package/build/fusion/engine.js

@@ -1,69 +0,0 @@
-import { spawn } from "child_process";
-import path from "path";
-import fs from "fs";
-export class DataFusionEngine {
-    pythonPath = "python";
-    scriptPath;
-    constructor(buildDir = process.cwd()) {
-        const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
-        const dataRoot = path.join(homeDir, ".vesper");
-        const scriptPath0 = path.resolve(dataRoot, "python", "fusion_engine.py");
-        const scriptPath1 = path.resolve(buildDir, "python", "fusion_engine.py");
-        const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "fusion_engine.py");
-        const scriptPath3 = path.resolve(buildDir, "..", "python", "fusion_engine.py");
-        if (fs.existsSync(scriptPath0)) {
-            this.scriptPath = scriptPath0;
-        }
-        else if (fs.existsSync(scriptPath1)) {
-            this.scriptPath = scriptPath1;
-        }
-        else if (fs.existsSync(scriptPath2)) {
-            this.scriptPath = scriptPath2;
-        }
-        else if (fs.existsSync(scriptPath3)) {
-            this.scriptPath = scriptPath3;
-        }
-        else {
-            this.scriptPath = scriptPath0;
-        }
-        if (process.platform === "win32") {
-            this.pythonPath = "py";
-        }
-    }
-    async fuse(sourcePaths, outputPath, options = {}) {
-        return new Promise((resolve, reject) => {
-            if (!Array.isArray(sourcePaths) || sourcePaths.length < 2) {
-                reject(new Error("At least 2 source paths are required for fusion"));
-                return;
-            }
-            const args = [
-                this.scriptPath,
-                JSON.stringify(sourcePaths),
-                outputPath,
-                JSON.stringify(options),
-            ];
-            const processRef = spawn(this.pythonPath, args);
-            let stdout = "";
-            let stderr = "";
-            processRef.stdout.on("data", (data) => (stdout += data.toString()));
-            processRef.stderr.on("data", (data) => (stderr += data.toString()));
-            processRef.on("close", (code) => {
-                if (code !== 0) {
-                    reject(new Error(`Fusion failed: ${stderr || stdout}`));
-                    return;
-                }
-                try {
-                    const result = JSON.parse(stdout);
-                    if (result.error) {
-                        reject(new Error(result.error));
-                        return;
-                    }
-                    resolve(result);
-                }
-                catch (e) {
-                    reject(new Error(`Failed to parse fusion output: ${stdout}`));
-                }
-            });
-        });
-    }
-}

package/build/fusion/harmonizer.js

@@ -1,39 +0,0 @@
-export class LabelHarmonizer {
-    config;
-    constructor(config) {
-        this.config = config;
-    }
-    /**
-     * Maps a raw label value to a unified canonical value.
-     */
-    harmonize(label) {
-        let harmonized = label;
-        // 1. apply explicit mapping
-        if (this.config.label_map && label in this.config.label_map) {
-            harmonized = this.config.label_map[label];
-        }
-        // 2. apply multi-class to binary conversion
-        if (this.config.multi_to_binary) {
-            const { positive_classes, positive_label, negative_label } = this.config.multi_to_binary;
-            const isPositive = positive_classes.some((pc) => String(pc).toLowerCase() === String(harmonized).toLowerCase());
-            return isPositive ? positive_label : negative_label;
-        }
-        return harmonized;
-    }
-    /**
-     * Checks if a label distribution is balanced enough (placeholder for quality scoring).
-     */
-    checkBalance(distribution) {
-        const warnings = [];
-        const total = Object.values(distribution).reduce((a, b) => a + b, 0);
-        if (total === 0)
-            return warnings;
-        for (const [label, count] of Object.entries(distribution)) {
-            const pct = count / total;
-            if (pct < 0.05) {
-                warnings.push(`Extreme minority class detected: "${label}" (${(pct * 100).toFixed(1)}%)`);
-            }
-        }
-        return warnings;
-    }
-}

package/build/fusion/orchestrator.js

@@ -1,86 +0,0 @@
-import { SchemaAligner } from "./aligner.js";
-import { LabelHarmonizer } from "./harmonizer.js";
-import { Deduplicator } from "./deduplicator.js";
-export class FusionOrchestrator {
-    config;
-    aligner;
-    harmonizer;
-    deduplicator;
-    constructor(config) {
-        this.config = config;
-        this.aligner = new SchemaAligner(config);
-        this.harmonizer = new LabelHarmonizer(config);
-        this.deduplicator = new Deduplicator(config);
-    }
-    /**
-     * Fuses multiple datasets into one.
-     * @param datasets Metadata of datasets to fuse
-     * @param dataRecords Actual records from all datasets (flattened)
-     */
-    async fuse(datasets, dataRecords) {
-        const stats = {
-            total_input_rows: dataRecords.length,
-            total_output_rows: 0,
-            duplicates_removed: 0,
-            fuzzy_duplicates_removed: 0,
-            schema_overlaps: [],
-            label_distribution: {}
-        };
-        const warnings = [];
-        const alignmentMap = this.aligner.align(datasets);
-        // Find canonical columns common to at least one dataset (in this simple version)
-        const allCanonicalCols = new Set();
-        for (const dsId in alignmentMap) {
-            for (const canonical of Object.values(alignmentMap[dsId])) {
-                allCanonicalCols.add(canonical);
-            }
-        }
-        stats.schema_overlaps = Array.from(allCanonicalCols);
-        const fusedData = [];
-        for (const { datasetId, record } of dataRecords) {
-            const alignedRecord = {};
-            const colMap = alignmentMap[datasetId];
-            if (!colMap) {
-                warnings.push(`No alignment found for dataset ${datasetId}`);
-                continue;
-            }
-            // 1. Align columns
-            for (const [sourceCol, val] of Object.entries(record)) {
-                const canonical = colMap[sourceCol];
-                if (canonical) {
-                    alignedRecord[canonical] = val;
-                }
-            }
-            // 2. Harmonize label
-            if (this.config.target_column && alignedRecord[this.config.target_column] !== undefined) {
-                alignedRecord[this.config.target_column] = this.harmonizer.harmonize(alignedRecord[this.config.target_column]);
-            }
-            // 3. Deduplicate
-            if (this.deduplicator.isExactDuplicate(alignedRecord)) {
-                stats.duplicates_removed++;
-                continue;
-            }
-            if (this.deduplicator.isFuzzyDuplicate(alignedRecord)) {
-                stats.fuzzy_duplicates_removed++;
-                continue;
-            }
-            // Update distribution ONLY for kept records
-            if (this.config.target_column && alignedRecord[this.config.target_column] !== undefined) {
-                const labelStr = String(alignedRecord[this.config.target_column]);
-                stats.label_distribution[labelStr] = (stats.label_distribution[labelStr] || 0) + 1;
-            }
-            fusedData.push(alignedRecord);
-        }
-        stats.total_output_rows = fusedData.push(); // Wait, push returns new length
-        stats.total_output_rows = fusedData.length;
-        // Add balance warnings
-        const balanceWarnings = this.harmonizer.checkBalance(stats.label_distribution);
-        warnings.push(...balanceWarnings);
-        return {
-            success: true,
-            output_path: "fused_dataset.json", // Placeholder
-            stats,
-            warnings
-        };
-    }
-}

package/build/fusion/types.js

@@ -1 +0,0 @@
-export {};