vesper-wizard 2.0.5 → 2.0.7

package/src/python/export_engine.py

@@ -0,0 +1,243 @@
+import sys
+import json
+import polars as pl
+import os
+import time
+
+# Optional imports for extra formats
+try:
+    import pyarrow as pa
+    import pyarrow.feather as pf
+    HAS_PYARROW = True
+except ImportError:
+    HAS_PYARROW = False
+
+try:
+    import tensorflow as tf
+    HAS_TENSORFLOW = True
+except ImportError:
+    HAS_TENSORFLOW = False
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _load(file_path: str, options: dict) -> pl.DataFrame:
+    """Load any supported input format into a Polars DataFrame."""
+    sample_rows = options.get("sample_rows")  # int | None
+    columns = options.get("columns")          # list[str] | None
+
+    ext = os.path.splitext(file_path)[1].lower()
+    if ext == ".csv":
+        df = pl.read_csv(file_path, ignore_errors=True)
+    elif ext == ".tsv":
+        df = pl.read_csv(file_path, separator="\t", ignore_errors=True)
+    elif ext == ".txt":
+        # Heuristic delimiter detection for plain text tabular files.
+        sep = ","
+        try:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                first_line = fh.readline()
+                if "\t" in first_line:
+                    sep = "\t"
+        except Exception:
+            sep = ","
+        df = pl.read_csv(file_path, separator=sep, ignore_errors=True)
+    elif ext in (".parquet", ".pq"):
+        df = pl.read_parquet(file_path)
+    elif ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        df = pl.read_ipc(file_path)
+    elif ext == ".jsonl":
+        df = pl.read_ndjson(file_path)
+    else:
+        raise ValueError(f"Unsupported input format: {ext}")
+
+    if len(df) == 0:
+        raise ValueError("empty CSV")
+
+    # Column selection (before sampling for speed)
+    if columns:
+        valid = [c for c in columns if c in df.columns]
+        if valid:
+            df = df.select(valid)
+
+    # Optional sampling
+    if sample_rows and sample_rows < len(df):
+        seed = options.get("random_seed", 42)
+        df = df.sample(n=sample_rows, seed=seed)
+
+    return df
+
+
+def _safe_csv_df(df: pl.DataFrame) -> pl.DataFrame:
+    """Stringify complex columns so CSV doesn't choke."""
+    for col in df.columns:
+        dtype = df.schema[col]
+        is_simple = (
+            dtype.is_numeric()
+            or dtype.is_temporal()
+            or str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
+        )
+        if not is_simple:
+            def safe_serialize(val):
+                try:
+                    if hasattr(val, "to_list"):
+                        return json.dumps(val.to_list())
+                    if hasattr(val, "to_dict"):
+                        return json.dumps(val.to_dict())
+                    return json.dumps(val)
+                except Exception:
+                    return str(val)
+            df = df.with_columns(
+                pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8)
+            )
+    return df
+
+
+def _write_preview(df: pl.DataFrame, output_path: str, n: int = 500):
+    """Write a small CSV preview next to the exported file."""
+    preview_path = os.path.splitext(output_path)[0] + "_preview.csv"
+    preview_df = _safe_csv_df(df.head(min(n, len(df))))
+    preview_df.write_csv(preview_path)
+    return preview_path
+
+
+# ---------------------------------------------------------------------------
+# Main export function
+# ---------------------------------------------------------------------------
+
+def export_data(file_path: str, output_path: str, format: str, options: dict | None = None):
+    options = options or {}
+    t0 = time.perf_counter()
+
+    # ---- Load ----
+    try:
+        df = _load(file_path, options)
+    except Exception as e:
+        return {"error": f"Failed to load input file: {str(e)}"}
+
+    output_dir = os.path.dirname(output_path)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir, exist_ok=True)
+
+    preview_path = None
+    generate_preview = options.get("preview", False)
+
+    try:
+        # ---- Feather (Arrow IPC) – fastest binary format ----
+        if format == "feather":
+            if not HAS_PYARROW:
+                return {"error": "pyarrow is not installed. Run: pip install pyarrow"}
+            compression = options.get("compression", "lz4")
+            if compression in ("uncompressed", "none", "None", None):
+                compression = "uncompressed"
+            # Polars write_ipc uses Arrow IPC (= Feather v2) under the hood
+            arrow_table = df.to_arrow()
+            pf.write_feather(arrow_table, output_path, compression=compression)
+            if generate_preview:
+                preview_path = _write_preview(df, output_path)
+
+        # ---- Parquet – best compression, big-data friendly ----
+        elif format == "parquet":
+            compression = options.get("compression", "snappy")
+            if compression in ("uncompressed", "none", "None", None):
+                compression = "uncompressed"
+            df.write_parquet(output_path, compression=compression)
+            if generate_preview:
+                preview_path = _write_preview(df, output_path)
+
+        # ---- CSV – human-readable fallback ----
+        elif format == "csv":
+            df = _safe_csv_df(df)
+            df.write_csv(output_path)
+
+        # ---- JSONL ----
+        elif format == "jsonl":
+            df.write_ndjson(output_path)
+            if generate_preview:
+                preview_path = _write_preview(df, output_path)
+
+        # ---- Arrow IPC (legacy name kept for compat) ----
+        elif format in ("arrow", "ipc"):
+            compression = options.get("compression", "uncompressed")
+            if compression == "uncompressed":
+                compression = None
+            df.write_ipc(output_path, compression=compression)
+            if generate_preview:
+                preview_path = _write_preview(df, output_path)
+
+        # ---- TFRecord ----
+        elif format == "tfrecord":
+            if not HAS_TENSORFLOW:
+                return {"error": "TensorFlow is not installed. Cannot export to TFRecord."}
+            with tf.io.TFRecordWriter(output_path) as writer:
+                pdf = df.to_pandas()
+                for _, row in pdf.iterrows():
+                    feature = {}
+                    for col, value in row.items():
+                        if value is None:
+                            continue
+                        if isinstance(value, int):
+                            feature[col] = tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+                        elif isinstance(value, float):
+                            feature[col] = tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+                        elif isinstance(value, str):
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode("utf-8")]))
+                        elif isinstance(value, bytes):
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+                        else:
+                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value).encode("utf-8")]))
+                    example = tf.train.Example(features=tf.train.Features(feature=feature))
+                    writer.write(example.SerializeToString())
+
+        else:
+            return {"error": f"Unknown export format: {format}"}
+
+        elapsed = round(time.perf_counter() - t0, 3)
+        file_size_mb = round(os.path.getsize(output_path) / (1024 * 1024), 2)
+
+        result = {
+            "success": True,
+            "output_path": output_path,
+            "rows": len(df),
+            "columns": len(df.columns),
+            "format": format,
+            "compression": options.get("compression", "default"),
+            "file_size_mb": file_size_mb,
+            "elapsed_seconds": elapsed,
+        }
+        if preview_path:
+            result["preview_path"] = preview_path
+
+        return result
+
+    except Exception as e:
+        return {"error": f"Export failed: {str(e)}"}
+
+
+def main():
+    if len(sys.argv) < 4:
+        print(
+            json.dumps({"error": "Usage: export_engine.py <input_file> <output_file> <format> [options_json]"}),
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    fmt = sys.argv[3]
+
+    options = {}
+    if len(sys.argv) > 4:
+        try:
+            options = json.loads(sys.argv[4])
+        except Exception:
+            pass
+
+    result = export_data(input_file, output_file, fmt, options)
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

package/src/python/framework_adapters.py

@@ -0,0 +1,100 @@
+import os
+import json
+import warnings
+
+# --- PyTorch Adapter ---
+try:
+    import torch
+    from torch.utils.data import Dataset
+    import polars as pl
+    
+    class VesperPyTorchDataset(Dataset):
+        """
+        PyTorch Dataset wrapper for Vesper exported files (Parquet/CSV/Arrow).
+        Efficiently loads data using Polars and converts to Tensors on demand.
+        """
+        def __init__(self, file_path, target_col=None, transform=None):
+            self.file_path = file_path
+            self.target_col = target_col
+            self.transform = transform
+            
+            # Auto-detect format
+            if file_path.endswith(".parquet"):
+                self.df = pl.read_parquet(file_path)
+            elif file_path.endswith(".csv"):
+                self.df = pl.read_csv(file_path, ignore_errors=True)
+            elif file_path.endswith(".arrow"):
+                self.df = pl.read_ipc(file_path)
+            else:
+                raise ValueError(f"Unsupported file format for PyTorch loader: {file_path}")
+                
+            self.data = self.df.to_pandas() # Convert to pandas for easier row access in __getitem__ (Polars slice can be slow row-wise)
+
+        def __len__(self):
+            return len(self.data)
+
+        def __getitem__(self, idx):
+            row = self.data.iloc[idx]
+            
+            # Simple assumption: all numeric columns except target are features
+            # In production, metadata would tell us which columns are features
+            if self.target_col and self.target_col in row:
+                y = row[self.target_col]
+                x = row.drop(self.target_col).values
+                
+                # Convert to tensors
+                x = torch.tensor(x, dtype=torch.float32)
+                # Auto-detect target type (scalar vs class index)
+                if isinstance(y, (int, float)):
+                     y = torch.tensor(y, dtype=torch.float32) # Regression/Binary
+                else:
+                    # TODO: Label encoding if string
+                    pass
+                
+                sample = (x, y)
+            else:
+                # Unsupervised
+                x = torch.tensor(row.values, dtype=torch.float32)
+                sample = x
+
+            if self.transform:
+                sample = self.transform(sample)
+
+            return sample
+
+except ImportError:
+    class VesperPyTorchDataset:
+        def __init__(self, *args, **kwargs):
+            raise ImportError("PyTorch or Polars not installed.")
+
+# --- HuggingFace Adapter ---
+try:
+    from datasets import load_dataset as hf_load_dataset
+    
+    def load_vesper_dataset(file_path):
+        """
+        Loads a Vesper export into a Hugging Face Dataset.
+        Supported: Parquet, CSV, JSONL, Arrow.
+        """
+        output_format = "parquet" # Default fallback
+        if file_path.endswith(".csv"): output_format = "csv"
+        elif file_path.endswith(".jsonl"): output_format = "json"
+        elif file_path.endswith(".arrow"): output_format = "arrow"
+        
+        # 'arrow' format in HF might need custom script, but usually parquet/csv/json are native
+        if output_format == "arrow":
+             # Use pandas/polars to read then convert to HF dataset
+             import polars as pl
+             from datasets import Dataset
+             df = pl.read_ipc(file_path).to_pandas()
+             return Dataset.from_pandas(df)
+        
+        return hf_load_dataset(output_format, data_files=file_path, split="train")
+
+except ImportError:
+    def load_vesper_dataset(*args, **kwargs):
+        raise ImportError("HuggingFace 'datasets' library not installed.")
+
+if __name__ == "__main__":
+    print("Vesper Framework Adapters Library")
+    print("Usage: import this module in your training script.")