vesper-wizard 2.3.0 → 2.3.2

package/src/python/asset_downloader_engine.py

@@ -1,94 +0,0 @@
-import argparse
-import asyncio
-import json
-import os
-import sys
-import warnings
-from pathlib import Path
-from typing import Any, Dict
-
-# Suppress noisy HF warnings
-warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
-warnings.filterwarnings("ignore", message=".*legacy.*")
-
-CURRENT_DIR = Path(__file__).resolve().parent
-if str(CURRENT_DIR) not in sys.path:
-    sys.path.insert(0, str(CURRENT_DIR))
-
-from vesper.core.asset_downloader import AssetDownloader
-from vesper.core.download_recipe import build_download_recipe, save_recipe, get_download_recipe
-
-
-def _print(payload: Dict[str, Any]) -> None:
-    print(json.dumps(payload, ensure_ascii=False))
-
-
-async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
-    payload = json.loads(args.payload)
-    output_root = payload.get("output_root") or str(Path.home() / ".vesper" / "data" / "assets")
-    output_dir = payload.get("output_dir")
-    workers = int(payload.get("workers") or 8)
-    recipes_dir = payload.get("recipes_dir")
-
-    # Auto-set HF token from payload if provided
-    token = payload.get("token") or payload.get("hf_token")
-    if token:
-        os.environ["HF_TOKEN"] = str(token)
-
-    downloader = AssetDownloader(output_root=output_root, workers=workers, recipes_dir=recipes_dir)
-
-    result = await downloader.download_assets(
-        dataset_id=str(payload.get("dataset_id")),
-        source=payload.get("source"),
-        repo_id=payload.get("repo_id"),
-        kaggle_ref=payload.get("kaggle_ref"),
-        urls=payload.get("urls"),
-        output_format=payload.get("output_format", "webdataset"),
-        output_dir=str(output_dir) if output_dir else None,
-        max_items=payload.get("max_items"),
-        image_column=payload.get("image_column"),
-    )
-    return {"ok": True, "result": result}
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Vesper Smart Asset Downloader Engine")
-    parser.add_argument("action", choices=["download", "build_recipe", "get_recipe"])
-    parser.add_argument("payload", help="JSON payload")
-    args = parser.parse_args()
-
-    try:
-        if args.action == "download":
-            response = asyncio.run(_run_download(args))
-            _print(response)
-            return
-
-        payload = json.loads(args.payload)
-        if args.action == "build_recipe":
-            recipe = build_download_recipe(payload)
-            saved = save_recipe(recipe, payload.get("recipes_dir"))
-            _print({"ok": True, "recipe": recipe, "saved_to": saved})
-            return
-
-        if args.action == "get_recipe":
-            dataset_id = str(payload.get("dataset_id"))
-            recipe = get_download_recipe(dataset_id, payload.get("recipes_dir"))
-            _print({"ok": True, "recipe": recipe})
-            return
-
-        _print({"ok": False, "error": f"Unknown action: {args.action}"})
-    except Exception as e:
-        error_msg = str(e)
-        # Provide actionable error messages
-        if "401" in error_msg or "403" in error_msg or "Unauthorized" in error_msg:
-            error_msg = (
-                "Authentication required. This dataset may be gated/private. "
-                "Use configure_keys tool to set HF_TOKEN, then retry."
-            )
-        elif "No image column" in error_msg:
-            error_msg += " Hint: specify image_column parameter with the name of the column containing images."
-        _print({"ok": False, "error": error_msg})
-
-
-if __name__ == "__main__":
-    main()

package/src/python/cleaner.py

@@ -1,226 +0,0 @@
-import sys
-import json
-import polars as pl
-import numpy as np
-
-# --- Operations Library ---
-
-def op_remove_duplicates(df, params):
-    subset = params.get("subset", None) # List of cols or None
-    before = len(df)
-    if subset:
-        df = df.unique(subset=subset)
-    else:
-        df = df.unique()
-    return df, {"rows_removed": before - len(df)}
-
-def op_drop_columns(df, params):
-    cols = params.get("columns", [])
-    before = len(df.columns)
-    # Filter only existing cols to avoid errors
-    cols_to_drop = [c for c in cols if c in df.columns]
-    df = df.drop(cols_to_drop)
-    return df, {"columns_dropped": len(cols_to_drop)}
-
-def op_fill_missing(df, params):
-    col = params["column"]
-    method = params.get("method", "mean") # mean, median, mode, constant
-    value = params.get("value", None)
-    
-    if col not in df.columns:
-        return df, {"error": f"Column {col} not found"}
-
-    affected = df[col].null_count()
-    
-    if method == "constant":
-        df = df.with_columns(pl.col(col).fill_null(value))
-    elif method == "mean":
-        mean_val = df[col].mean()
-        df = df.with_columns(pl.col(col).fill_null(mean_val))
-    elif method == "median":
-        median_val = df[col].median()
-        df = df.with_columns(pl.col(col).fill_null(median_val))
-    
-    return df, {"rows_imputed": affected}
-
-def op_fix_types(df, params):
-    col = params["column"]
-    target_type = params["type"] # "int", "float", "string", "date"
-    
-    if col not in df.columns:
-        return df, {"error": f"Column {col} not found"}
-
-    try:
-        if target_type == "int":
-            df = df.with_columns(pl.col(col).cast(pl.Int64, strict=False))
-        elif target_type == "float":
-            df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
-        elif target_type == "string":
-            df = df.with_columns(pl.col(col).cast(pl.Utf8))
-        elif target_type == "date":
-             df = df.with_columns(pl.col(col).str.to_date(strict=False))
-        
-        return df, {"status": "Converted"}
-    except Exception as e:
-        return df, {"error": str(e)}
-
-def op_remove_outliers(df, params):
-    col = params["column"]
-    method = params.get("method", "iqr")
-    threshold = params.get("threshold", 1.5)
-    
-    if col not in df.columns:
-        return df, {"error": f"Column {col} not found"}
-
-    before = len(df)
-    
-    if method == "iqr":
-        q1 = df[col].quantile(0.25)
-        q3 = df[col].quantile(0.75)
-        iqr = q3 - q1
-        lower = q1 - (threshold * iqr)
-        upper = q3 + (threshold * iqr)
-        
-        df = df.filter((pl.col(col) >= lower) & (pl.col(col) <= upper))
-    
-    return df, {"rows_removed": before - len(df)}
-
-def op_encode_categories(df, params):
-    col = params["column"]
-    method = params.get("method", "label") # label, onehot
-    
-    if col not in df.columns:
-        return df, {"error": f"Column {col} not found"}
-
-    if method == "label":
-        # Polars dense_rank acts similar to label encoding
-        df = df.with_columns(pl.col(col).rank("dense").alias(f"{col}_encoded"))
-    elif method == "onehot":
-        dummies = df[col].to_dummies()
-        df = pl.concat([df, dummies], how="horizontal")
-    
-    return df, {"status": f"Encoded using {method}"}
-
-# --- Registry ---
-
-OPERATIONS = {
-    "RemoveDuplicates": op_remove_duplicates,
-    "DropColumns": op_drop_columns,
-    "FillMissing": op_fill_missing,
-    "FixTypes": op_fix_types,
-    "RemoveOutliers": op_remove_outliers,
-    "EncodeCategories": op_encode_categories
-}
-
-def main():
-    if len(sys.argv) < 3:
-        print(json.dumps({"error": "Usage: cleaner.py <file_path> <operations_json>"}), file=sys.stderr)
-        sys.exit(1)
-
-    file_path = sys.argv[1]
-    ops_json = sys.argv[2]
-    
-    try:
-        operations = json.loads(ops_json)
-        
-        # Load Data
-        file_path_lower = file_path.lower()
-        if file_path_lower.endswith(".csv"):
-            df = pl.read_csv(file_path, ignore_errors=True)
-        elif file_path_lower.endswith(".parquet"):
-            df = pl.read_parquet(file_path)
-        elif file_path_lower.endswith(".jsonl") or file_path_lower.endswith(".ndjson"):
-            # Explicit NDJSON
-            df = pl.read_ndjson(file_path)
-        elif file_path_lower.endswith(".json"):
-            # Ambiguous .json
-            try:
-                df = pl.read_json(file_path)
-            except Exception:
-                try:
-                     df = pl.read_ndjson(file_path)
-                except Exception as e:
-                     raise ValueError(f"Failed to read JSON: {str(e)}")
-        else:
-            raise ValueError(f"Unsupported format: {file_path}")
-
-        logs = []
-        total_rows_affected = 0
-
-        # Execute Pipeline
-        for op in operations:
-            op_type = op["type"]
-            params = op.get("params", {})
-            
-            if op_type == "RenameTarget":
-                old_name = params.get("old_name")
-                new_name = params.get("new_name", "target")
-                if old_name and old_name in df.columns:
-                    df = df.rename({old_name: new_name})
-                    logs.append(f"Renamed column '{old_name}' to '{new_name}'")
-                else:
-                    logs.append(f"Failed RenameTarget: Column '{old_name}' not found or not specified.")
-            elif op_type in OPERATIONS:
-                try:
-                    df, stats = OPERATIONS[op_type](df, params)
-                    logs.append(f"Executed {op_type}: {stats}")
-                    total_rows_affected += stats.get("rows_removed", 0)
-                except Exception as e:
-                    logs.append(f"Failed {op_type}: {str(e)}")
-            else:
-                logs.append(f"Unknown operation: {op_type}")
-
-        # Save Result (overwrite or new file)
-        # Save Result (overwrite or new file)
-        output_format = sys.argv[3] if len(sys.argv) > 3 else None
-        
-        if not output_format:
-            # Legacy logic: preserve CSV or default to parquet
-            if file_path_lower.endswith(".csv"):
-                output_format = "csv"
-            else:
-                output_format = "parquet"
-
-        base_name = file_path.rsplit(".", 1)[0]
-        if output_format == "csv":
-            output_path = f"{base_name}_cleaned.csv"
-            # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)
-            for col in df.columns:
-                dtype = df.schema[col]
-                # Only keep simple types; stringify everything else for CSV
-                is_simple = (
-                    dtype.is_numeric() or 
-                    dtype.is_temporal() or 
-                    str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
-                )
-                if not is_simple:
-                    # Use a robust helper for clean JSON serialization
-                    def safe_serialize(val):
-                        try:
-                            # Handle Polars nested types (convert to Python list/dict first)
-                            if hasattr(val, "to_list"):
-                                return json.dumps(val.to_list())
-                            if hasattr(val, "to_dict"):
-                                return json.dumps(val.to_dict())
-                            return json.dumps(val)
-                        except:
-                            return str(val)
-                    df = df.with_columns(pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8))
-            df.write_csv(output_path)
-        else:
-            output_path = f"{base_name}_cleaned.parquet"
-            df.write_parquet(output_path)
-
-        print(json.dumps({
-            "success": True,
-            "output_path": output_path,
-            "rows_affected": total_rows_affected,
-            "logs": logs
-        }, default=str))
-
-    except Exception as e:
-        print(json.dumps({"success": False, "error": str(e)}, default=str))
-        sys.exit(1)
-
-if __name__ == "__main__":
-    main()

package/src/python/config.py

@@ -1,263 +0,0 @@
-import os
-import sys
-import json
-import base64
-import hashlib
-import secrets
-from pathlib import Path
-from typing import Dict, Optional
-
-SERVICE_NAME = "vesper"
-
-KEY_ALIASES = {
-    "hf_token": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
-    "kaggle_username": ["KAGGLE_USERNAME"],
-    "kaggle_key": ["KAGGLE_KEY"],
-    "dataworld_token": ["DW_AUTH_TOKEN"],
-}
-
-try:
-    import keyring  # type: ignore
-    HAS_KEYRING = True
-except Exception:
-    HAS_KEYRING = False
-
-try:
-    from cryptography.fernet import Fernet, InvalidToken  # type: ignore
-    HAS_FERNET = True
-except Exception:
-    HAS_FERNET = False
-
-
-def _config_path() -> Path:
-    return Path.home() / ".vesper" / "config.toml"
-
-
-def _secret_path() -> Path:
-    return Path.home() / ".vesper" / ".config_key"
-
-
-def _ensure_parent(path: Path) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-
-
-def _read_fallback_toml() -> Dict[str, str]:
-    path = _config_path()
-    if not path.exists():
-        return {}
-
-    values: Dict[str, str] = {}
-    in_keys = False
-    method = ""
-
-    for raw in path.read_text(encoding="utf-8").splitlines():
-        line = raw.strip()
-        if not line or line.startswith("#"):
-            continue
-        if line.startswith("[") and line.endswith("]"):
-            in_keys = (line == "[keys]")
-            continue
-        if line.startswith("method") and "=" in line:
-            method = line.split("=", 1)[1].strip().strip('"').strip("'")
-            continue
-        if not in_keys or "=" not in line:
-            continue
-
-        key, val = line.split("=", 1)
-        key = key.strip()
-        val = val.strip().strip('"').strip("'")
-        values[key] = val
-
-    if method:
-        values["__method__"] = method
-
-    return values
-
-
-def _get_or_create_local_secret() -> str:
-    secret_file = _secret_path()
-    _ensure_parent(secret_file)
-
-    if secret_file.exists():
-        return secret_file.read_text(encoding="utf-8").strip()
-
-    secret = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8")
-    secret_file.write_text(secret, encoding="utf-8")
-    try:
-        os.chmod(secret_file, 0o600)
-    except Exception:
-        pass
-    return secret
-
-
-def _xor_encrypt(plain: str, secret: str) -> str:
-    key = hashlib.sha256(secret.encode("utf-8")).digest()
-    data = plain.encode("utf-8")
-    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
-    return base64.urlsafe_b64encode(out).decode("utf-8")
-
-
-def _xor_decrypt(cipher_text: str, secret: str) -> str:
-    key = hashlib.sha256(secret.encode("utf-8")).digest()
-    data = base64.urlsafe_b64decode(cipher_text.encode("utf-8"))
-    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
-    return out.decode("utf-8")
-
-
-def _encrypt_value(value: str, secret: str) -> Dict[str, str]:
-    if HAS_FERNET:
-        token = Fernet(secret.encode("utf-8")).encrypt(value.encode("utf-8")).decode("utf-8")
-        return {"method": "fernet", "value": token}
-    # fallback encryption (weaker than fernet, but still not plaintext)
-    return {"method": "xor", "value": _xor_encrypt(value, secret)}
-
-
-def _decrypt_value(value: str, method: str, secret: str) -> Optional[str]:
-    try:
-        if method == "fernet" and HAS_FERNET:
-            return Fernet(secret.encode("utf-8")).decrypt(value.encode("utf-8")).decode("utf-8")
-        if method == "xor":
-            return _xor_decrypt(value, secret)
-        return None
-    except InvalidToken:
-        return None
-    except Exception:
-        return None
-
-
-def _write_fallback_toml(values: Dict[str, str]) -> None:
-    path = _config_path()
-    _ensure_parent(path)
-
-    method = values.get("__method__", "fernet" if HAS_FERNET else "xor")
-    lines = [
-        "# Vesper optional API keys fallback storage",
-        "# Encrypted fallback (keyring is preferred)",
-        "[meta]",
-        f'method = "{method}"',
-        "[keys]",
-    ]
-    for key in sorted(values.keys()):
-        if key.startswith("__"):
-            continue
-        val = str(values[key]).replace('"', '\\"')
-        lines.append(f'{key} = "{val}"')
-
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-
-def _get_from_env(name: str) -> Optional[str]:
-    for env_key in KEY_ALIASES.get(name, []):
-        val = os.getenv(env_key)
-        if val:
-            return val
-    return None
-
-
-def get_key(name: str) -> Optional[str]:
-    # 1) keyring (secure)
-    if HAS_KEYRING:
-        try:
-            val = keyring.get_password(SERVICE_NAME, name)
-            if val:
-                return val
-        except Exception:
-            pass
-
-    # 2) encrypted fallback config.toml
-    fallback = _read_fallback_toml()
-    enc = fallback.get(name)
-    if enc:
-        secret = _get_or_create_local_secret()
-        method = fallback.get("__method__", "fernet" if HAS_FERNET else "xor")
-        dec = _decrypt_value(enc, method, secret)
-        if dec:
-            return dec
-
-    # 3) env vars (fallback only)
-    env_val = _get_from_env(name)
-    if env_val:
-        return env_val
-    return None
-
-
-def set_key(name: str, value: str) -> Dict[str, str]:
-    if not value:
-        return {"ok": "false", "method": "none", "error": "Empty value"}
-
-    if HAS_KEYRING:
-        try:
-            keyring.set_password(SERVICE_NAME, name, value)
-            return {"ok": "true", "method": "keyring"}
-        except Exception:
-            pass
-
-    fallback = _read_fallback_toml()
-    secret = _get_or_create_local_secret()
-    enc = _encrypt_value(value, secret)
-    fallback["__method__"] = enc["method"]
-    fallback[name] = enc["value"]
-    _write_fallback_toml(fallback)
-    return {"ok": "true", "method": f'toml:{enc["method"]}'}
-
-
-def has_key(name: str) -> bool:
-    return bool(get_key(name))
-
-
-def get_all() -> Dict[str, Optional[str]]:
-    return {
-        "hf_token": get_key("hf_token"),
-        "kaggle_username": get_key("kaggle_username"),
-        "kaggle_key": get_key("kaggle_key"),
-        "dataworld_token": get_key("dataworld_token"),
-    }
-
-
-def _print_json(data):
-    print(json.dumps(data))
-
-
-def main() -> None:
-    if len(sys.argv) < 2:
-        _print_json({
-            "ok": False,
-            "error": "Usage: config.py <get|set|has|all> [name] [value]",
-        })
-        sys.exit(1)
-
-    cmd = sys.argv[1].lower()
-
-    if cmd == "all":
-        _print_json({"ok": True, "data": get_all()})
-        return
-
-    if len(sys.argv) < 3:
-        _print_json({"ok": False, "error": "Missing key name"})
-        sys.exit(1)
-
-    name = sys.argv[2]
-
-    if cmd == "get":
-        _print_json({"ok": True, "name": name, "value": get_key(name)})
-        return
-
-    if cmd == "has":
-        _print_json({"ok": True, "name": name, "value": has_key(name)})
-        return
-
-    if cmd == "set":
-        if len(sys.argv) < 4:
-            _print_json({"ok": False, "error": "Missing value for set"})
-            sys.exit(1)
-        value = sys.argv[3]
-        result = set_key(name, value)
-        _print_json({"ok": result.get("ok") == "true", "name": name, "method": result.get("method"), "error": result.get("error")})
-        return
-
-    _print_json({"ok": False, "error": f"Unknown command: {cmd}"})
-    sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/src/python/convert_engine.py

@@ -1,92 +0,0 @@
-"""
-Convert a dataset file between formats (CSV, Parquet, JSON, JSONL).
-Usage: convert_engine.py <input_path> <output_path>
-Outputs JSON: {"ok": true, "output_path": "...", "rows": N, "columns": N} or {"ok": false, "error": "..."}
-"""
-import sys
-import json
-import os
-
-try:
-    import polars as pl
-except Exception:
-    print(json.dumps({"ok": False, "error": "polars is required. Install with: pip install polars"}))
-    sys.exit(1)
-
-
-def _load(src: str) -> pl.DataFrame:
-    ext = os.path.splitext(src)[1].lower()
-    if ext == ".csv":
-        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
-    if ext in (".tsv", ".tab"):
-        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
-    if ext in (".parquet", ".pq"):
-        return pl.read_parquet(src)
-    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
-        return pl.read_ipc(src)
-    if ext in (".jsonl", ".ndjson"):
-        return pl.read_ndjson(src)
-    if ext == ".json":
-        raw = open(src, "r", encoding="utf-8").read().strip()
-        if raw.startswith("["):
-            return pl.read_json(src)
-        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
-            return pl.read_ndjson(src)
-        obj = json.loads(raw)
-        if isinstance(obj, dict):
-            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
-                if key in obj and isinstance(obj[key], list):
-                    return pl.DataFrame(obj[key])
-            for v in obj.values():
-                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
-                    return pl.DataFrame(v)
-        return pl.read_json(src)
-    # Fallback: try csv
-    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
-
-
-def _write(df: pl.DataFrame, dst: str) -> None:
-    ext = os.path.splitext(dst)[1].lower()
-    os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
-    if ext in (".parquet", ".pq"):
-        df.write_parquet(dst)
-    elif ext == ".csv":
-        df.write_csv(dst)
-    elif ext == ".json":
-        df.write_json(dst, row_oriented=True)
-    elif ext in (".jsonl", ".ndjson"):
-        df.write_ndjson(dst)
-    else:
-        raise ValueError(f"Unsupported output format: {ext}")
-
-
-def main():
-    if len(sys.argv) < 3:
-        print(json.dumps({"ok": False, "error": "Usage: convert_engine.py <input> <output>"}))
-        sys.exit(1)
-
-    input_path = sys.argv[1]
-    output_path = sys.argv[2]
-
-    if not os.path.exists(input_path):
-        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
-        sys.exit(1)
-
-    try:
-        df = _load(input_path)
-        _write(df, output_path)
-        size_mb = round(os.path.getsize(output_path) / (1024 * 1024), 2)
-        print(json.dumps({
-            "ok": True,
-            "output_path": output_path,
-            "rows": df.height,
-            "columns": df.width,
-            "size_mb": size_mb,
-        }))
-    except Exception as e:
-        print(json.dumps({"ok": False, "error": str(e)}))
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()