vesper-wizard 2.0.4 → 2.0.6

package/src/python/cleaner.py

@@ -0,0 +1,226 @@
+import sys
+import json
+import polars as pl
+import numpy as np
+
+# --- Operations Library ---
+
+def op_remove_duplicates(df, params):
+    subset = params.get("subset", None) # List of cols or None
+    before = len(df)
+    if subset:
+        df = df.unique(subset=subset)
+    else:
+        df = df.unique()
+    return df, {"rows_removed": before - len(df)}
+
+def op_drop_columns(df, params):
+    cols = params.get("columns", [])
+    before = len(df.columns)
+    # Filter only existing cols to avoid errors
+    cols_to_drop = [c for c in cols if c in df.columns]
+    df = df.drop(cols_to_drop)
+    return df, {"columns_dropped": len(cols_to_drop)}
+
+def op_fill_missing(df, params):
+    col = params["column"]
+    method = params.get("method", "mean") # mean, median, mode, constant
+    value = params.get("value", None)
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    affected = df[col].null_count()
+    
+    if method == "constant":
+        df = df.with_columns(pl.col(col).fill_null(value))
+    elif method == "mean":
+        mean_val = df[col].mean()
+        df = df.with_columns(pl.col(col).fill_null(mean_val))
+    elif method == "median":
+        median_val = df[col].median()
+        df = df.with_columns(pl.col(col).fill_null(median_val))
+    
+    return df, {"rows_imputed": affected}
+
+def op_fix_types(df, params):
+    col = params["column"]
+    target_type = params["type"] # "int", "float", "string", "date"
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    try:
+        if target_type == "int":
+            df = df.with_columns(pl.col(col).cast(pl.Int64, strict=False))
+        elif target_type == "float":
+            df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
+        elif target_type == "string":
+            df = df.with_columns(pl.col(col).cast(pl.Utf8))
+        elif target_type == "date":
+             df = df.with_columns(pl.col(col).str.to_date(strict=False))
+        
+        return df, {"status": "Converted"}
+    except Exception as e:
+        return df, {"error": str(e)}
+
+def op_remove_outliers(df, params):
+    col = params["column"]
+    method = params.get("method", "iqr")
+    threshold = params.get("threshold", 1.5)
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    before = len(df)
+    
+    if method == "iqr":
+        q1 = df[col].quantile(0.25)
+        q3 = df[col].quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - (threshold * iqr)
+        upper = q3 + (threshold * iqr)
+        
+        df = df.filter((pl.col(col) >= lower) & (pl.col(col) <= upper))
+    
+    return df, {"rows_removed": before - len(df)}
+
+def op_encode_categories(df, params):
+    col = params["column"]
+    method = params.get("method", "label") # label, onehot
+    
+    if col not in df.columns:
+        return df, {"error": f"Column {col} not found"}
+
+    if method == "label":
+        # Polars dense_rank acts similar to label encoding
+        df = df.with_columns(pl.col(col).rank("dense").alias(f"{col}_encoded"))
+    elif method == "onehot":
+        dummies = df[col].to_dummies()
+        df = pl.concat([df, dummies], how="horizontal")
+    
+    return df, {"status": f"Encoded using {method}"}
+
+# --- Registry ---
+
+OPERATIONS = {
+    "RemoveDuplicates": op_remove_duplicates,
+    "DropColumns": op_drop_columns,
+    "FillMissing": op_fill_missing,
+    "FixTypes": op_fix_types,
+    "RemoveOutliers": op_remove_outliers,
+    "EncodeCategories": op_encode_categories
+}
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"error": "Usage: cleaner.py <file_path> <operations_json>"}), file=sys.stderr)
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+    ops_json = sys.argv[2]
+    
+    try:
+        operations = json.loads(ops_json)
+        
+        # Load Data
+        file_path_lower = file_path.lower()
+        if file_path_lower.endswith(".csv"):
+            df = pl.read_csv(file_path, ignore_errors=True)
+        elif file_path_lower.endswith(".parquet"):
+            df = pl.read_parquet(file_path)
+        elif file_path_lower.endswith(".jsonl") or file_path_lower.endswith(".ndjson"):
+            # Explicit NDJSON
+            df = pl.read_ndjson(file_path)
+        elif file_path_lower.endswith(".json"):
+            # Ambiguous .json
+            try:
+                df = pl.read_json(file_path)
+            except Exception:
+                try:
+                     df = pl.read_ndjson(file_path)
+                except Exception as e:
+                     raise ValueError(f"Failed to read JSON: {str(e)}")
+        else:
+            raise ValueError(f"Unsupported format: {file_path}")
+
+        logs = []
+        total_rows_affected = 0
+
+        # Execute Pipeline
+        for op in operations:
+            op_type = op["type"]
+            params = op.get("params", {})
+            
+            if op_type == "RenameTarget":
+                old_name = params.get("old_name")
+                new_name = params.get("new_name", "target")
+                if old_name and old_name in df.columns:
+                    df = df.rename({old_name: new_name})
+                    logs.append(f"Renamed column '{old_name}' to '{new_name}'")
+                else:
+                    logs.append(f"Failed RenameTarget: Column '{old_name}' not found or not specified.")
+            elif op_type in OPERATIONS:
+                try:
+                    df, stats = OPERATIONS[op_type](df, params)
+                    logs.append(f"Executed {op_type}: {stats}")
+                    total_rows_affected += stats.get("rows_removed", 0)
+                except Exception as e:
+                    logs.append(f"Failed {op_type}: {str(e)}")
+            else:
+                logs.append(f"Unknown operation: {op_type}")
+
+        # Save Result (overwrite or new file)
+        # Save Result (overwrite or new file)
+        output_format = sys.argv[3] if len(sys.argv) > 3 else None
+        
+        if not output_format:
+            # Legacy logic: preserve CSV or default to parquet
+            if file_path_lower.endswith(".csv"):
+                output_format = "csv"
+            else:
+                output_format = "parquet"
+
+        base_name = file_path.rsplit(".", 1)[0]
+        if output_format == "csv":
+            output_path = f"{base_name}_cleaned.csv"
+            # Stringify ANY column that might not be CSV-safe (List, Struct, Object, etc.)
+            for col in df.columns:
+                dtype = df.schema[col]
+                # Only keep simple types; stringify everything else for CSV
+                is_simple = (
+                    dtype.is_numeric() or 
+                    dtype.is_temporal() or 
+                    str(dtype).lower() in ["string", "utf8", "boolean", "bool"]
+                )
+                if not is_simple:
+                    # Use a robust helper for clean JSON serialization
+                    def safe_serialize(val):
+                        try:
+                            # Handle Polars nested types (convert to Python list/dict first)
+                            if hasattr(val, "to_list"):
+                                return json.dumps(val.to_list())
+                            if hasattr(val, "to_dict"):
+                                return json.dumps(val.to_dict())
+                            return json.dumps(val)
+                        except:
+                            return str(val)
+                    df = df.with_columns(pl.col(col).map_elements(safe_serialize, return_dtype=pl.Utf8))
+            df.write_csv(output_path)
+        else:
+            output_path = f"{base_name}_cleaned.parquet"
+            df.write_parquet(output_path)
+
+        print(json.dumps({
+            "success": True,
+            "output_path": output_path,
+            "rows_affected": total_rows_affected,
+            "logs": logs
+        }, default=str))
+
+    except Exception as e:
+        print(json.dumps({"success": False, "error": str(e)}, default=str))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

package/src/python/config.py

@@ -0,0 +1,263 @@
+import os
+import sys
+import json
+import base64
+import hashlib
+import secrets
+from pathlib import Path
+from typing import Dict, Optional
+
+SERVICE_NAME = "vesper"
+
+KEY_ALIASES = {
+    "hf_token": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
+    "kaggle_username": ["KAGGLE_USERNAME"],
+    "kaggle_key": ["KAGGLE_KEY"],
+    "dataworld_token": ["DW_AUTH_TOKEN"],
+}
+
+try:
+    import keyring  # type: ignore
+    HAS_KEYRING = True
+except Exception:
+    HAS_KEYRING = False
+
+try:
+    from cryptography.fernet import Fernet, InvalidToken  # type: ignore
+    HAS_FERNET = True
+except Exception:
+    HAS_FERNET = False
+
+
+def _config_path() -> Path:
+    return Path.home() / ".vesper" / "config.toml"
+
+
+def _secret_path() -> Path:
+    return Path.home() / ".vesper" / ".config_key"
+
+
+def _ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def _read_fallback_toml() -> Dict[str, str]:
+    path = _config_path()
+    if not path.exists():
+        return {}
+
+    values: Dict[str, str] = {}
+    in_keys = False
+    method = ""
+
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("[") and line.endswith("]"):
+            in_keys = (line == "[keys]")
+            continue
+        if line.startswith("method") and "=" in line:
+            method = line.split("=", 1)[1].strip().strip('"').strip("'")
+            continue
+        if not in_keys or "=" not in line:
+            continue
+
+        key, val = line.split("=", 1)
+        key = key.strip()
+        val = val.strip().strip('"').strip("'")
+        values[key] = val
+
+    if method:
+        values["__method__"] = method
+
+    return values
+
+
+def _get_or_create_local_secret() -> str:
+    secret_file = _secret_path()
+    _ensure_parent(secret_file)
+
+    if secret_file.exists():
+        return secret_file.read_text(encoding="utf-8").strip()
+
+    secret = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8")
+    secret_file.write_text(secret, encoding="utf-8")
+    try:
+        os.chmod(secret_file, 0o600)
+    except Exception:
+        pass
+    return secret
+
+
+def _xor_encrypt(plain: str, secret: str) -> str:
+    key = hashlib.sha256(secret.encode("utf-8")).digest()
+    data = plain.encode("utf-8")
+    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
+    return base64.urlsafe_b64encode(out).decode("utf-8")
+
+
+def _xor_decrypt(cipher_text: str, secret: str) -> str:
+    key = hashlib.sha256(secret.encode("utf-8")).digest()
+    data = base64.urlsafe_b64decode(cipher_text.encode("utf-8"))
+    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
+    return out.decode("utf-8")
+
+
+def _encrypt_value(value: str, secret: str) -> Dict[str, str]:
+    if HAS_FERNET:
+        token = Fernet(secret.encode("utf-8")).encrypt(value.encode("utf-8")).decode("utf-8")
+        return {"method": "fernet", "value": token}
+    # fallback encryption (weaker than fernet, but still not plaintext)
+    return {"method": "xor", "value": _xor_encrypt(value, secret)}
+
+
+def _decrypt_value(value: str, method: str, secret: str) -> Optional[str]:
+    try:
+        if method == "fernet" and HAS_FERNET:
+            return Fernet(secret.encode("utf-8")).decrypt(value.encode("utf-8")).decode("utf-8")
+        if method == "xor":
+            return _xor_decrypt(value, secret)
+        return None
+    except InvalidToken:
+        return None
+    except Exception:
+        return None
+
+
+def _write_fallback_toml(values: Dict[str, str]) -> None:
+    path = _config_path()
+    _ensure_parent(path)
+
+    method = values.get("__method__", "fernet" if HAS_FERNET else "xor")
+    lines = [
+        "# Vesper optional API keys fallback storage",
+        "# Encrypted fallback (keyring is preferred)",
+        "[meta]",
+        f'method = "{method}"',
+        "[keys]",
+    ]
+    for key in sorted(values.keys()):
+        if key.startswith("__"):
+            continue
+        val = str(values[key]).replace('"', '\\"')
+        lines.append(f'{key} = "{val}"')
+
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _get_from_env(name: str) -> Optional[str]:
+    for env_key in KEY_ALIASES.get(name, []):
+        val = os.getenv(env_key)
+        if val:
+            return val
+    return None
+
+
+def get_key(name: str) -> Optional[str]:
+    # 1) keyring (secure)
+    if HAS_KEYRING:
+        try:
+            val = keyring.get_password(SERVICE_NAME, name)
+            if val:
+                return val
+        except Exception:
+            pass
+
+    # 2) encrypted fallback config.toml
+    fallback = _read_fallback_toml()
+    enc = fallback.get(name)
+    if enc:
+        secret = _get_or_create_local_secret()
+        method = fallback.get("__method__", "fernet" if HAS_FERNET else "xor")
+        dec = _decrypt_value(enc, method, secret)
+        if dec:
+            return dec
+
+    # 3) env vars (fallback only)
+    env_val = _get_from_env(name)
+    if env_val:
+        return env_val
+    return None
+
+
+def set_key(name: str, value: str) -> Dict[str, str]:
+    if not value:
+        return {"ok": "false", "method": "none", "error": "Empty value"}
+
+    if HAS_KEYRING:
+        try:
+            keyring.set_password(SERVICE_NAME, name, value)
+            return {"ok": "true", "method": "keyring"}
+        except Exception:
+            pass
+
+    fallback = _read_fallback_toml()
+    secret = _get_or_create_local_secret()
+    enc = _encrypt_value(value, secret)
+    fallback["__method__"] = enc["method"]
+    fallback[name] = enc["value"]
+    _write_fallback_toml(fallback)
+    return {"ok": "true", "method": f'toml:{enc["method"]}'}
+
+
+def has_key(name: str) -> bool:
+    return bool(get_key(name))
+
+
+def get_all() -> Dict[str, Optional[str]]:
+    return {
+        "hf_token": get_key("hf_token"),
+        "kaggle_username": get_key("kaggle_username"),
+        "kaggle_key": get_key("kaggle_key"),
+        "dataworld_token": get_key("dataworld_token"),
+    }
+
+
+def _print_json(data):
+    print(json.dumps(data))
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        _print_json({
+            "ok": False,
+            "error": "Usage: config.py <get|set|has|all> [name] [value]",
+        })
+        sys.exit(1)
+
+    cmd = sys.argv[1].lower()
+
+    if cmd == "all":
+        _print_json({"ok": True, "data": get_all()})
+        return
+
+    if len(sys.argv) < 3:
+        _print_json({"ok": False, "error": "Missing key name"})
+        sys.exit(1)
+
+    name = sys.argv[2]
+
+    if cmd == "get":
+        _print_json({"ok": True, "name": name, "value": get_key(name)})
+        return
+
+    if cmd == "has":
+        _print_json({"ok": True, "name": name, "value": has_key(name)})
+        return
+
+    if cmd == "set":
+        if len(sys.argv) < 4:
+            _print_json({"ok": False, "error": "Missing value for set"})
+            sys.exit(1)
+        value = sys.argv[3]
+        result = set_key(name, value)
+        _print_json({"ok": result.get("ok") == "true", "name": name, "method": result.get("method"), "error": result.get("error")})
+        return
+
+    _print_json({"ok": False, "error": f"Unknown command: {cmd}"})
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/src/python/dataworld_engine.py

@@ -0,0 +1,208 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+import urllib.request
+import urllib.error
+import urllib.parse
+from typing import Dict, Any, List
+
+def _get_token() -> str:
+    token = os.environ.get("DW_AUTH_TOKEN")
+    if not token:
+        raise ValueError("DW_AUTH_TOKEN environment variable is required for data.world")
+    return token
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    owner_field = ds.get("owner", "")
+    if isinstance(owner_field, dict):
+        owner = owner_field.get("id") or owner_field.get("name") or ""
+    else:
+        owner = owner_field or ""
+
+    id_str = ds.get("id", "")
+    title = ds.get("title", "")
+
+    if (not owner or not id_str) and isinstance(ds.get("resourceLink"), str):
+        # Expected format includes /<owner>/<dataset-id>
+        parts = ds["resourceLink"].strip("/").split("/")
+        if len(parts) >= 2:
+            owner = owner or parts[-2]
+            id_str = id_str or parts[-1]
+
+    if isinstance(id_str, str) and "/" in id_str and not owner:
+        split_ref = id_str.split("/", 1)
+        owner = split_ref[0]
+        id_str = split_ref[1]
+
+    if not owner and not id_str:
+        owner = "unknown"
+        id_str = "unknown"
+
+    if not title:
+        title = f"{owner}/{id_str}"
+    
+    return {
+        "id": f"dataworld:{owner}/{id_str}",
+        "name": title,
+        "source": "dataworld",
+        "description": ds.get("description", f"data.world dataset {title}"),
+        "author": owner,
+        "license": {
+            "id": "Unknown",
+            "category": "unknown",
+            "commercial_use": None,
+            "warnings": []
+        },
+        "tags": ds.get("tags", []) + ["dataworld"],
+        "downloads": 0,
+        "likes": 0,
+        "created_at": ds.get("created", ""),
+        "updated_at": ds.get("updated", ""),
+        "size_bytes": 0,
+        "quality_score": 0.8,
+        "domain": "general",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": len(ds.get("description", "")),
+        "has_readme": False,
+        "download_url": f"https://data.world/{owner}/{id_str}",
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # data.world simple search API
+        url = f"https://api.data.world/v0/search/resources?size={limit}"
+        
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/json"
+        }
+        
+        # Search datasets and include community results to improve recall
+        body = {
+            "query": query,
+            "category": ["dataset"],
+            "includeCommunityResults": True,
+        }
+        
+        req = urllib.request.Request(url, data=json.dumps(body).encode('utf-8'), headers=headers, method="POST")
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode('utf-8'))
+            
+        records = data.get("records", [])
+
+        # Fallback to advanced endpoint if simple search returns nothing
+        if not records:
+            adv_url = f"https://api.data.world/v0/search?size={limit}"
+            adv_body = {
+                "query": query,
+                "category": ["dataset"],
+            }
+            adv_req = urllib.request.Request(
+                adv_url,
+                data=json.dumps(adv_body).encode("utf-8"),
+                headers=headers,
+                method="POST",
+            )
+            with urllib.request.urlopen(adv_req) as response:
+                adv_data = json.loads(response.read().decode("utf-8"))
+            records = adv_data.get("records", [])
+
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"data.world discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # dataset_ref is expected to be "dataworld:owner/id"
+        if dataset_ref.startswith("dataworld:"):
+            ref = dataset_ref.split(":", 1)[1]
+        else:
+            ref = dataset_ref
+            
+        parts = ref.split("/")
+        if len(parts) != 2:
+            return {"ok": False, "error": f"Invalid data.world dataset ID format. Expected owner/id, got {ref}"}
+            
+        owner, dataset_id = parts
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_dataworld_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # First, get the dataset metadata to find the files
+        url = f"https://api.data.world/v0/datasets/{owner}/{dataset_id}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json"
+        }
+        
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req) as response:
+            dataset_meta = json.loads(response.read().decode('utf-8'))
+            
+        files = dataset_meta.get("files", [])
+        if not files:
+            return {"ok": False, "error": "No files found in this dataset"}
+            
+        # Find the best file to download (prefer csv, parquet, jsonl)
+        best_file = None
+        for ext in [".parquet", ".csv", ".jsonl", ".json"]:
+            for f in files:
+                if f.get("name", "").lower().endswith(ext):
+                    best_file = f
+                    break
+            if best_file:
+                break
+                
+        if not best_file:
+            best_file = files[0] # Just take the first one if no preferred format
+            
+        filename = best_file.get("name")
+        
+        # Download the file
+        download_url = f"https://api.data.world/v0/file_download/{owner}/{dataset_id}/{urllib.parse.quote(filename)}"
+        
+        file_path = os.path.join(target_dir, filename)
+        
+        download_req = urllib.request.Request(download_url, headers=headers)
+        with urllib.request.urlopen(download_req) as response, open(file_path, 'wb') as out_file:
+            out_file.write(response.read())
+            
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"data.world download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper data.world Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()