vesper-wizard 2.3.1 → 2.3.3

package/src/python/dataworld_engine.py

@@ -1,208 +0,0 @@
-import sys
-import json
-import argparse
-import tempfile
-import os
-import urllib.request
-import urllib.error
-import urllib.parse
-from typing import Dict, Any, List
-
-def _get_token() -> str:
-    token = os.environ.get("DW_AUTH_TOKEN")
-    if not token:
-        raise ValueError("DW_AUTH_TOKEN environment variable is required for data.world")
-    return token
-
-def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
-    owner_field = ds.get("owner", "")
-    if isinstance(owner_field, dict):
-        owner = owner_field.get("id") or owner_field.get("name") or ""
-    else:
-        owner = owner_field or ""
-
-    id_str = ds.get("id", "")
-    title = ds.get("title", "")
-
-    if (not owner or not id_str) and isinstance(ds.get("resourceLink"), str):
-        # Expected format includes /<owner>/<dataset-id>
-        parts = ds["resourceLink"].strip("/").split("/")
-        if len(parts) >= 2:
-            owner = owner or parts[-2]
-            id_str = id_str or parts[-1]
-
-    if isinstance(id_str, str) and "/" in id_str and not owner:
-        split_ref = id_str.split("/", 1)
-        owner = split_ref[0]
-        id_str = split_ref[1]
-
-    if not owner and not id_str:
-        owner = "unknown"
-        id_str = "unknown"
-
-    if not title:
-        title = f"{owner}/{id_str}"
-    
-    return {
-        "id": f"dataworld:{owner}/{id_str}",
-        "name": title,
-        "source": "dataworld",
-        "description": ds.get("description", f"data.world dataset {title}"),
-        "author": owner,
-        "license": {
-            "id": "Unknown",
-            "category": "unknown",
-            "commercial_use": None,
-            "warnings": []
-        },
-        "tags": ds.get("tags", []) + ["dataworld"],
-        "downloads": 0,
-        "likes": 0,
-        "created_at": ds.get("created", ""),
-        "updated_at": ds.get("updated", ""),
-        "size_bytes": 0,
-        "quality_score": 0.8,
-        "domain": "general",
-        "is_gated": False,
-        "is_nsfw": False,
-        "description_length": len(ds.get("description", "")),
-        "has_readme": False,
-        "download_url": f"https://data.world/{owner}/{id_str}",
-    }
-
-def discover(query: str, limit: int = 20) -> Dict[str, Any]:
-    try:
-        token = _get_token()
-        
-        # data.world simple search API
-        url = f"https://api.data.world/v0/search/resources?size={limit}"
-        
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-            "Accept": "application/json"
-        }
-        
-        # Search datasets and include community results to improve recall
-        body = {
-            "query": query,
-            "category": ["dataset"],
-            "includeCommunityResults": True,
-        }
-        
-        req = urllib.request.Request(url, data=json.dumps(body).encode('utf-8'), headers=headers, method="POST")
-        
-        with urllib.request.urlopen(req) as response:
-            data = json.loads(response.read().decode('utf-8'))
-            
-        records = data.get("records", [])
-
-        # Fallback to advanced endpoint if simple search returns nothing
-        if not records:
-            adv_url = f"https://api.data.world/v0/search?size={limit}"
-            adv_body = {
-                "query": query,
-                "category": ["dataset"],
-            }
-            adv_req = urllib.request.Request(
-                adv_url,
-                data=json.dumps(adv_body).encode("utf-8"),
-                headers=headers,
-                method="POST",
-            )
-            with urllib.request.urlopen(adv_req) as response:
-                adv_data = json.loads(response.read().decode("utf-8"))
-            records = adv_data.get("records", [])
-
-        items = [_dataset_to_dict(r) for r in records]
-        
-        return {"ok": True, "results": items, "count": len(items)}
-    except Exception as e:
-        return {"ok": False, "error": f"data.world discover failed: {str(e)}"}
-
-def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
-    try:
-        token = _get_token()
-        
-        # dataset_ref is expected to be "dataworld:owner/id"
-        if dataset_ref.startswith("dataworld:"):
-            ref = dataset_ref.split(":", 1)[1]
-        else:
-            ref = dataset_ref
-            
-        parts = ref.split("/")
-        if len(parts) != 2:
-            return {"ok": False, "error": f"Invalid data.world dataset ID format. Expected owner/id, got {ref}"}
-            
-        owner, dataset_id = parts
-        
-        if not target_dir:
-            target_dir = tempfile.mkdtemp(prefix="vesper_dataworld_")
-            
-        os.makedirs(target_dir, exist_ok=True)
-        
-        # First, get the dataset metadata to find the files
-        url = f"https://api.data.world/v0/datasets/{owner}/{dataset_id}"
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "Accept": "application/json"
-        }
-        
-        req = urllib.request.Request(url, headers=headers)
-        with urllib.request.urlopen(req) as response:
-            dataset_meta = json.loads(response.read().decode('utf-8'))
-            
-        files = dataset_meta.get("files", [])
-        if not files:
-            return {"ok": False, "error": "No files found in this dataset"}
-            
-        # Find the best file to download (prefer csv, parquet, jsonl)
-        best_file = None
-        for ext in [".parquet", ".csv", ".jsonl", ".json"]:
-            for f in files:
-                if f.get("name", "").lower().endswith(ext):
-                    best_file = f
-                    break
-            if best_file:
-                break
-                
-        if not best_file:
-            best_file = files[0] # Just take the first one if no preferred format
-            
-        filename = best_file.get("name")
-        
-        # Download the file
-        download_url = f"https://api.data.world/v0/file_download/{owner}/{dataset_id}/{urllib.parse.quote(filename)}"
-        
-        file_path = os.path.join(target_dir, filename)
-        
-        download_req = urllib.request.Request(download_url, headers=headers)
-        with urllib.request.urlopen(download_req) as response, open(file_path, 'wb') as out_file:
-            out_file.write(response.read())
-            
-        return {
-            "ok": True, 
-            "local_path": file_path, 
-            "target_dir": target_dir
-        }
-    except Exception as e:
-        return {"ok": False, "error": f"data.world download failed: {str(e)}"}
-
-def main():
-    parser = argparse.ArgumentParser(description="Vesper data.world Engine")
-    parser.add_argument("action", choices=["discover", "download"])
-    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
-    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
-
-    args = parser.parse_args()
-
-    if args.action == "discover":
-        limit = int(args.arg2) if args.arg2 else 20
-        result = discover(args.arg1, limit)
-        print(json.dumps(result))
-    elif args.action == "download":
-        result = download(args.arg1, args.arg2)
-        print(json.dumps(result))
-
-if __name__ == "__main__":
-    main()

package/src/python/export_engine.py

@@ -1,288 +0,0 @@
-import sys
-import json
-import polars as pl
-import os
-import time
-
-# Optional imports for extra formats
-try:
-    import pyarrow as pa
-    import pyarrow.feather as pf
-    HAS_PYARROW = True
-except ImportError:
-    HAS_PYARROW = False
-
-try:
-    import tensorflow as tf
-    HAS_TENSORFLOW = True
-except ImportError:
-    HAS_TENSORFLOW = False
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _load(file_path: str, options: dict) -> pl.DataFrame:
-    """Load any supported input format into a Polars DataFrame."""
-    sample_rows = options.get("sample_rows")  # int | None
-    columns = options.get("columns")          # list[str] | None
-
-    ext = os.path.splitext(file_path)[1].lower()
-    if ext == ".csv":
-        df = pl.read_csv(file_path, ignore_errors=True)
-    elif ext == ".tsv":
-        df = pl.read_csv(file_path, separator="\t", ignore_errors=True)
-    elif ext == ".txt":
-        # Heuristic delimiter detection for plain text tabular files.
-        sep = ","
-        try:
-            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
-                first_line = fh.readline()
-                if "\t" in first_line:
-                    sep = "\t"
-        except Exception:
-            sep = ","
-        df = pl.read_csv(file_path, separator=sep, ignore_errors=True)
-    elif ext in (".parquet", ".pq"):
-        df = pl.read_parquet(file_path)
-    elif ext in (".feather", ".ftr", ".arrow", ".ipc"):
-        df = pl.read_ipc(file_path)
-    elif ext == ".jsonl":
-        df = pl.read_ndjson(file_path)
-    elif ext == ".json":
-        # Auto-detect: array-of-objects vs NDJSON vs nested structures
-        try:
-            import json as _json
-            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
-                raw_text = fh.read(512)  # peek
-            stripped = raw_text.lstrip()
-            if stripped.startswith("["):
-                # Array of objects — standard JSON
-                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
-                    data = _json.load(fh)
-                if isinstance(data, list) and len(data) > 0:
-                    df = pl.DataFrame(data)
-                else:
-                    raise ValueError("JSON file is empty or not an array of objects")
-            elif stripped.startswith("{"):
-                # Could be NDJSON or a single object wrapping rows
-                try:
-                    df = pl.read_ndjson(file_path)
-                except Exception:
-                    with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
-                        data = _json.load(fh)
-                    # Try common wrapper patterns: {"data": [...]}, {"rows": [...]}, etc.
-                    rows = None
-                    if isinstance(data, dict):
-                        for key in ("data", "rows", "records", "items", "results", "entries"):
-                            if key in data and isinstance(data[key], list):
-                                rows = data[key]
-                                break
-                        if rows is None:
-                            # Last resort: try to use the dict values
-                            rows = [data]
-                    if rows and len(rows) > 0:
-                        df = pl.DataFrame(rows)
-                    else:
-                        raise ValueError("Could not parse JSON structure into tabular data")
-            else:
-                raise ValueError("JSON file does not start with [ or {")
-        except pl.exceptions.ComputeError as ce:
-            raise ValueError(f"Failed to parse JSON: {ce}")
-    elif ext == ".xlsx":
-        try:
-            df = pl.read_excel(file_path)
-        except Exception as e:
-            raise ValueError(f"Failed to read Excel file: {e}")
-    else:
-        raise ValueError(f"Unsupported input format: {ext}")
-
-    if len(df) == 0:
-        raise ValueError("empty CSV")
-
-    # Column selection (before sampling for speed)
-    if columns:
-        valid = [c for c in columns if c in df.columns]
-        if valid:
-            df = df.select(valid)
-
-    # Optional sampling
-    if sample_rows and sample_rows < len(df):
-        seed = options.get("random_seed", 42)
-        df = df.sample(n=sample_rows, seed=seed)
-
-    return df
-
-
-def _safe_csv_df(df: pl.DataFrame) -> pl.DataFrame:
-    """Stringify complex columns so CSV doesn't choke."""
-    for col in df.columns:
-        dtype = df.schema[col]
-        is_simple = (
-            dtype.is_numeric()
-            or dtype.is_temporal()
-            or str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
-        )
-        if not is_simple:
-            def safe_serialize(val):
-                try:
-                    if hasattr(val, "to_list"):
-                        return json.dumps(val.to_list())
-                    if hasattr(val, "to_dict"):
-                        return json.dumps(val.to_dict())
-                    return json.dumps(val)
-                except Exception:
-                    return str(val)
-            df = df.with_columns(
-                pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8)
-            )
-    return df
-
-
-def _write_preview(df: pl.DataFrame, output_path: str, n: int = 500):
-    """Write a small CSV preview next to the exported file."""
-    preview_path = os.path.splitext(output_path)[0] + "_preview.csv"
-    preview_df = _safe_csv_df(df.head(min(n, len(df))))
-    preview_df.write_csv(preview_path)
-    return preview_path
-
-
-# ---------------------------------------------------------------------------
-# Main export function
-# ---------------------------------------------------------------------------
-
-def export_data(file_path: str, output_path: str, format: str, options: dict | None = None):
-    options = options or {}
-    t0 = time.perf_counter()
-
-    # ---- Load ----
-    try:
-        df = _load(file_path, options)
-    except Exception as e:
-        return {"error": f"Failed to load input file: {str(e)}"}
-
-    output_dir = os.path.dirname(output_path)
-    if output_dir and not os.path.exists(output_dir):
-        os.makedirs(output_dir, exist_ok=True)
-
-    preview_path = None
-    generate_preview = options.get("preview", False)
-
-    try:
-        # ---- Feather (Arrow IPC) – fastest binary format ----
-        if format == "feather":
-            if not HAS_PYARROW:
-                return {"error": "pyarrow is not installed. Run: pip install pyarrow"}
-            compression = options.get("compression", "lz4")
-            if compression in ("uncompressed", "none", "None", None):
-                compression = "uncompressed"
-            # Polars write_ipc uses Arrow IPC (= Feather v2) under the hood
-            arrow_table = df.to_arrow()
-            pf.write_feather(arrow_table, output_path, compression=compression)
-            if generate_preview:
-                preview_path = _write_preview(df, output_path)
-
-        # ---- Parquet – best compression, big-data friendly ----
-        elif format == "parquet":
-            compression = options.get("compression", "snappy")
-            if compression in ("uncompressed", "none", "None", None):
-                compression = "uncompressed"
-            df.write_parquet(output_path, compression=compression)
-            if generate_preview:
-                preview_path = _write_preview(df, output_path)
-
-        # ---- CSV – human-readable fallback ----
-        elif format == "csv":
-            df = _safe_csv_df(df)
-            df.write_csv(output_path)
-
-        # ---- JSONL ----
-        elif format == "jsonl":
-            df.write_ndjson(output_path)
-            if generate_preview:
-                preview_path = _write_preview(df, output_path)
-
-        # ---- Arrow IPC (legacy name kept for compat) ----
-        elif format in ("arrow", "ipc"):
-            compression = options.get("compression", "uncompressed")
-            if compression == "uncompressed":
-                compression = None
-            df.write_ipc(output_path, compression=compression)
-            if generate_preview:
-                preview_path = _write_preview(df, output_path)
-
-        # ---- TFRecord ----
-        elif format == "tfrecord":
-            if not HAS_TENSORFLOW:
-                return {"error": "TensorFlow is not installed. Cannot export to TFRecord."}
-            with tf.io.TFRecordWriter(output_path) as writer:
-                pdf = df.to_pandas()
-                for _, row in pdf.iterrows():
-                    feature = {}
-                    for col, value in row.items():
-                        if value is None:
-                            continue
-                        if isinstance(value, int):
-                            feature[col] = tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-                        elif isinstance(value, float):
-                            feature[col] = tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
-                        elif isinstance(value, str):
-                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode("utf-8")]))
-                        elif isinstance(value, bytes):
-                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-                        else:
-                            feature[col] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value).encode("utf-8")]))
-                    example = tf.train.Example(features=tf.train.Features(feature=feature))
-                    writer.write(example.SerializeToString())
-
-        else:
-            return {"error": f"Unknown export format: {format}"}
-
-        elapsed = round(time.perf_counter() - t0, 3)
-        file_size_mb = round(os.path.getsize(output_path) / (1024 * 1024), 2)
-
-        result = {
-            "success": True,
-            "output_path": output_path,
-            "rows": len(df),
-            "columns": len(df.columns),
-            "format": format,
-            "compression": options.get("compression", "default"),
-            "file_size_mb": file_size_mb,
-            "elapsed_seconds": elapsed,
-        }
-        if preview_path:
-            result["preview_path"] = preview_path
-
-        return result
-
-    except Exception as e:
-        return {"error": f"Export failed: {str(e)}"}
-
-
-def main():
-    if len(sys.argv) < 4:
-        print(
-            json.dumps({"error": "Usage: export_engine.py <input_file> <output_file> <format> [options_json]"}),
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    input_file = sys.argv[1]
-    output_file = sys.argv[2]
-    fmt = sys.argv[3]
-
-    options = {}
-    if len(sys.argv) > 4:
-        try:
-            options = json.loads(sys.argv[4])
-        except Exception:
-            pass
-
-    result = export_data(input_file, output_file, fmt, options)
-    print(json.dumps(result))
-
-
-if __name__ == "__main__":
-    main()

package/src/python/framework_adapters.py

@@ -1,100 +0,0 @@
-import os
-import json
-import warnings
-
-# --- PyTorch Adapter ---
-try:
-    import torch
-    from torch.utils.data import Dataset
-    import polars as pl
-    
-    class VesperPyTorchDataset(Dataset):
-        """
-        PyTorch Dataset wrapper for Vesper exported files (Parquet/CSV/Arrow).
-        Efficiently loads data using Polars and converts to Tensors on demand.
-        """
-        def __init__(self, file_path, target_col=None, transform=None):
-            self.file_path = file_path
-            self.target_col = target_col
-            self.transform = transform
-            
-            # Auto-detect format
-            if file_path.endswith(".parquet"):
-                self.df = pl.read_parquet(file_path)
-            elif file_path.endswith(".csv"):
-                self.df = pl.read_csv(file_path, ignore_errors=True)
-            elif file_path.endswith(".arrow"):
-                self.df = pl.read_ipc(file_path)
-            else:
-                raise ValueError(f"Unsupported file format for PyTorch loader: {file_path}")
-                
-            self.data = self.df.to_pandas() # Convert to pandas for easier row access in __getitem__ (Polars slice can be slow row-wise)
-
-        def __len__(self):
-            return len(self.data)
-
-        def __getitem__(self, idx):
-            row = self.data.iloc[idx]
-            
-            # Simple assumption: all numeric columns except target are features
-            # In production, metadata would tell us which columns are features
-            if self.target_col and self.target_col in row:
-                y = row[self.target_col]
-                x = row.drop(self.target_col).values
-                
-                # Convert to tensors
-                x = torch.tensor(x, dtype=torch.float32)
-                # Auto-detect target type (scalar vs class index)
-                if isinstance(y, (int, float)):
-                     y = torch.tensor(y, dtype=torch.float32) # Regression/Binary
-                else:
-                    # TODO: Label encoding if string
-                    pass
-                
-                sample = (x, y)
-            else:
-                # Unsupervised
-                x = torch.tensor(row.values, dtype=torch.float32)
-                sample = x
-
-            if self.transform:
-                sample = self.transform(sample)
-
-            return sample
-
-except ImportError:
-    class VesperPyTorchDataset:
-        def __init__(self, *args, **kwargs):
-            raise ImportError("PyTorch or Polars not installed.")
-
-# --- HuggingFace Adapter ---
-try:
-    from datasets import load_dataset as hf_load_dataset
-    
-    def load_vesper_dataset(file_path):
-        """
-        Loads a Vesper export into a Hugging Face Dataset.
-        Supported: Parquet, CSV, JSONL, Arrow.
-        """
-        output_format = "parquet" # Default fallback
-        if file_path.endswith(".csv"): output_format = "csv"
-        elif file_path.endswith(".jsonl"): output_format = "json"
-        elif file_path.endswith(".arrow"): output_format = "arrow"
-        
-        # 'arrow' format in HF might need custom script, but usually parquet/csv/json are native
-        if output_format == "arrow":
-             # Use pandas/polars to read then convert to HF dataset
-             import polars as pl
-             from datasets import Dataset
-             df = pl.read_ipc(file_path).to_pandas()
-             return Dataset.from_pandas(df)
-        
-        return hf_load_dataset(output_format, data_files=file_path, split="train")
-
-except ImportError:
-    def load_vesper_dataset(*args, **kwargs):
-        raise ImportError("HuggingFace 'datasets' library not installed.")
-
-if __name__ == "__main__":
-    print("Vesper Framework Adapters Library")
-    print("Usage: import this module in your training script.")