vesper-wizard 2.0.5 → 2.0.7

package/src/python/config.py

@@ -0,0 +1,263 @@
+import os
+import sys
+import json
+import base64
+import hashlib
+import secrets
+from pathlib import Path
+from typing import Dict, Optional
+
+SERVICE_NAME = "vesper"
+
+KEY_ALIASES = {
+    "hf_token": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
+    "kaggle_username": ["KAGGLE_USERNAME"],
+    "kaggle_key": ["KAGGLE_KEY"],
+    "dataworld_token": ["DW_AUTH_TOKEN"],
+}
+
+try:
+    import keyring  # type: ignore
+    HAS_KEYRING = True
+except Exception:
+    HAS_KEYRING = False
+
+try:
+    from cryptography.fernet import Fernet, InvalidToken  # type: ignore
+    HAS_FERNET = True
+except Exception:
+    HAS_FERNET = False
+
+
+def _config_path() -> Path:
+    return Path.home() / ".vesper" / "config.toml"
+
+
+def _secret_path() -> Path:
+    return Path.home() / ".vesper" / ".config_key"
+
+
+def _ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def _read_fallback_toml() -> Dict[str, str]:
+    path = _config_path()
+    if not path.exists():
+        return {}
+
+    values: Dict[str, str] = {}
+    in_keys = False
+    method = ""
+
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("[") and line.endswith("]"):
+            in_keys = (line == "[keys]")
+            continue
+        if line.startswith("method") and "=" in line:
+            method = line.split("=", 1)[1].strip().strip('"').strip("'")
+            continue
+        if not in_keys or "=" not in line:
+            continue
+
+        key, val = line.split("=", 1)
+        key = key.strip()
+        val = val.strip().strip('"').strip("'")
+        values[key] = val
+
+    if method:
+        values["__method__"] = method
+
+    return values
+
+
+def _get_or_create_local_secret() -> str:
+    secret_file = _secret_path()
+    _ensure_parent(secret_file)
+
+    if secret_file.exists():
+        return secret_file.read_text(encoding="utf-8").strip()
+
+    secret = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8")
+    secret_file.write_text(secret, encoding="utf-8")
+    try:
+        os.chmod(secret_file, 0o600)
+    except Exception:
+        pass
+    return secret
+
+
+def _xor_encrypt(plain: str, secret: str) -> str:
+    key = hashlib.sha256(secret.encode("utf-8")).digest()
+    data = plain.encode("utf-8")
+    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
+    return base64.urlsafe_b64encode(out).decode("utf-8")
+
+
+def _xor_decrypt(cipher_text: str, secret: str) -> str:
+    key = hashlib.sha256(secret.encode("utf-8")).digest()
+    data = base64.urlsafe_b64decode(cipher_text.encode("utf-8"))
+    out = bytes([data[i] ^ key[i % len(key)] for i in range(len(data))])
+    return out.decode("utf-8")
+
+
+def _encrypt_value(value: str, secret: str) -> Dict[str, str]:
+    if HAS_FERNET:
+        token = Fernet(secret.encode("utf-8")).encrypt(value.encode("utf-8")).decode("utf-8")
+        return {"method": "fernet", "value": token}
+    # fallback encryption (weaker than fernet, but still not plaintext)
+    return {"method": "xor", "value": _xor_encrypt(value, secret)}
+
+
+def _decrypt_value(value: str, method: str, secret: str) -> Optional[str]:
+    try:
+        if method == "fernet" and HAS_FERNET:
+            return Fernet(secret.encode("utf-8")).decrypt(value.encode("utf-8")).decode("utf-8")
+        if method == "xor":
+            return _xor_decrypt(value, secret)
+        return None
+    except InvalidToken:
+        return None
+    except Exception:
+        return None
+
+
+def _write_fallback_toml(values: Dict[str, str]) -> None:
+    path = _config_path()
+    _ensure_parent(path)
+
+    method = values.get("__method__", "fernet" if HAS_FERNET else "xor")
+    lines = [
+        "# Vesper optional API keys fallback storage",
+        "# Encrypted fallback (keyring is preferred)",
+        "[meta]",
+        f'method = "{method}"',
+        "[keys]",
+    ]
+    for key in sorted(values.keys()):
+        if key.startswith("__"):
+            continue
+        val = str(values[key]).replace('"', '\\"')
+        lines.append(f'{key} = "{val}"')
+
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _get_from_env(name: str) -> Optional[str]:
+    for env_key in KEY_ALIASES.get(name, []):
+        val = os.getenv(env_key)
+        if val:
+            return val
+    return None
+
+
+def get_key(name: str) -> Optional[str]:
+    # 1) keyring (secure)
+    if HAS_KEYRING:
+        try:
+            val = keyring.get_password(SERVICE_NAME, name)
+            if val:
+                return val
+        except Exception:
+            pass
+
+    # 2) encrypted fallback config.toml
+    fallback = _read_fallback_toml()
+    enc = fallback.get(name)
+    if enc:
+        secret = _get_or_create_local_secret()
+        method = fallback.get("__method__", "fernet" if HAS_FERNET else "xor")
+        dec = _decrypt_value(enc, method, secret)
+        if dec:
+            return dec
+
+    # 3) env vars (fallback only)
+    env_val = _get_from_env(name)
+    if env_val:
+        return env_val
+    return None
+
+
+def set_key(name: str, value: str) -> Dict[str, str]:
+    if not value:
+        return {"ok": "false", "method": "none", "error": "Empty value"}
+
+    if HAS_KEYRING:
+        try:
+            keyring.set_password(SERVICE_NAME, name, value)
+            return {"ok": "true", "method": "keyring"}
+        except Exception:
+            pass
+
+    fallback = _read_fallback_toml()
+    secret = _get_or_create_local_secret()
+    enc = _encrypt_value(value, secret)
+    fallback["__method__"] = enc["method"]
+    fallback[name] = enc["value"]
+    _write_fallback_toml(fallback)
+    return {"ok": "true", "method": f'toml:{enc["method"]}'}
+
+
+def has_key(name: str) -> bool:
+    return bool(get_key(name))
+
+
+def get_all() -> Dict[str, Optional[str]]:
+    return {
+        "hf_token": get_key("hf_token"),
+        "kaggle_username": get_key("kaggle_username"),
+        "kaggle_key": get_key("kaggle_key"),
+        "dataworld_token": get_key("dataworld_token"),
+    }
+
+
+def _print_json(data):
+    print(json.dumps(data))
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        _print_json({
+            "ok": False,
+            "error": "Usage: config.py <get|set|has|all> [name] [value]",
+        })
+        sys.exit(1)
+
+    cmd = sys.argv[1].lower()
+
+    if cmd == "all":
+        _print_json({"ok": True, "data": get_all()})
+        return
+
+    if len(sys.argv) < 3:
+        _print_json({"ok": False, "error": "Missing key name"})
+        sys.exit(1)
+
+    name = sys.argv[2]
+
+    if cmd == "get":
+        _print_json({"ok": True, "name": name, "value": get_key(name)})
+        return
+
+    if cmd == "has":
+        _print_json({"ok": True, "name": name, "value": has_key(name)})
+        return
+
+    if cmd == "set":
+        if len(sys.argv) < 4:
+            _print_json({"ok": False, "error": "Missing value for set"})
+            sys.exit(1)
+        value = sys.argv[3]
+        result = set_key(name, value)
+        _print_json({"ok": result.get("ok") == "true", "name": name, "method": result.get("method"), "error": result.get("error")})
+        return
+
+    _print_json({"ok": False, "error": f"Unknown command: {cmd}"})
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/src/python/dataworld_engine.py

@@ -0,0 +1,208 @@
+import sys
+import json
+import argparse
+import tempfile
+import os
+import urllib.request
+import urllib.error
+import urllib.parse
+from typing import Dict, Any, List
+
+def _get_token() -> str:
+    token = os.environ.get("DW_AUTH_TOKEN")
+    if not token:
+        raise ValueError("DW_AUTH_TOKEN environment variable is required for data.world")
+    return token
+
+def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
+    owner_field = ds.get("owner", "")
+    if isinstance(owner_field, dict):
+        owner = owner_field.get("id") or owner_field.get("name") or ""
+    else:
+        owner = owner_field or ""
+
+    id_str = ds.get("id", "")
+    title = ds.get("title", "")
+
+    if (not owner or not id_str) and isinstance(ds.get("resourceLink"), str):
+        # Expected format includes /<owner>/<dataset-id>
+        parts = ds["resourceLink"].strip("/").split("/")
+        if len(parts) >= 2:
+            owner = owner or parts[-2]
+            id_str = id_str or parts[-1]
+
+    if isinstance(id_str, str) and "/" in id_str and not owner:
+        split_ref = id_str.split("/", 1)
+        owner = split_ref[0]
+        id_str = split_ref[1]
+
+    if not owner and not id_str:
+        owner = "unknown"
+        id_str = "unknown"
+
+    if not title:
+        title = f"{owner}/{id_str}"
+    
+    return {
+        "id": f"dataworld:{owner}/{id_str}",
+        "name": title,
+        "source": "dataworld",
+        "description": ds.get("description", f"data.world dataset {title}"),
+        "author": owner,
+        "license": {
+            "id": "Unknown",
+            "category": "unknown",
+            "commercial_use": None,
+            "warnings": []
+        },
+        "tags": ds.get("tags", []) + ["dataworld"],
+        "downloads": 0,
+        "likes": 0,
+        "created_at": ds.get("created", ""),
+        "updated_at": ds.get("updated", ""),
+        "size_bytes": 0,
+        "quality_score": 0.8,
+        "domain": "general",
+        "is_gated": False,
+        "is_nsfw": False,
+        "description_length": len(ds.get("description", "")),
+        "has_readme": False,
+        "download_url": f"https://data.world/{owner}/{id_str}",
+    }
+
+def discover(query: str, limit: int = 20) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # data.world simple search API
+        url = f"https://api.data.world/v0/search/resources?size={limit}"
+        
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/json"
+        }
+        
+        # Search datasets and include community results to improve recall
+        body = {
+            "query": query,
+            "category": ["dataset"],
+            "includeCommunityResults": True,
+        }
+        
+        req = urllib.request.Request(url, data=json.dumps(body).encode('utf-8'), headers=headers, method="POST")
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode('utf-8'))
+            
+        records = data.get("records", [])
+
+        # Fallback to advanced endpoint if simple search returns nothing
+        if not records:
+            adv_url = f"https://api.data.world/v0/search?size={limit}"
+            adv_body = {
+                "query": query,
+                "category": ["dataset"],
+            }
+            adv_req = urllib.request.Request(
+                adv_url,
+                data=json.dumps(adv_body).encode("utf-8"),
+                headers=headers,
+                method="POST",
+            )
+            with urllib.request.urlopen(adv_req) as response:
+                adv_data = json.loads(response.read().decode("utf-8"))
+            records = adv_data.get("records", [])
+
+        items = [_dataset_to_dict(r) for r in records]
+        
+        return {"ok": True, "results": items, "count": len(items)}
+    except Exception as e:
+        return {"ok": False, "error": f"data.world discover failed: {str(e)}"}
+
+def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
+    try:
+        token = _get_token()
+        
+        # dataset_ref is expected to be "dataworld:owner/id"
+        if dataset_ref.startswith("dataworld:"):
+            ref = dataset_ref.split(":", 1)[1]
+        else:
+            ref = dataset_ref
+            
+        parts = ref.split("/")
+        if len(parts) != 2:
+            return {"ok": False, "error": f"Invalid data.world dataset ID format. Expected owner/id, got {ref}"}
+            
+        owner, dataset_id = parts
+        
+        if not target_dir:
+            target_dir = tempfile.mkdtemp(prefix="vesper_dataworld_")
+            
+        os.makedirs(target_dir, exist_ok=True)
+        
+        # First, get the dataset metadata to find the files
+        url = f"https://api.data.world/v0/datasets/{owner}/{dataset_id}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json"
+        }
+        
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req) as response:
+            dataset_meta = json.loads(response.read().decode('utf-8'))
+            
+        files = dataset_meta.get("files", [])
+        if not files:
+            return {"ok": False, "error": "No files found in this dataset"}
+            
+        # Find the best file to download (prefer csv, parquet, jsonl)
+        best_file = None
+        for ext in [".parquet", ".csv", ".jsonl", ".json"]:
+            for f in files:
+                if f.get("name", "").lower().endswith(ext):
+                    best_file = f
+                    break
+            if best_file:
+                break
+                
+        if not best_file:
+            best_file = files[0] # Just take the first one if no preferred format
+            
+        filename = best_file.get("name")
+        
+        # Download the file
+        download_url = f"https://api.data.world/v0/file_download/{owner}/{dataset_id}/{urllib.parse.quote(filename)}"
+        
+        file_path = os.path.join(target_dir, filename)
+        
+        download_req = urllib.request.Request(download_url, headers=headers)
+        with urllib.request.urlopen(download_req) as response, open(file_path, 'wb') as out_file:
+            out_file.write(response.read())
+            
+        return {
+            "ok": True, 
+            "local_path": file_path, 
+            "target_dir": target_dir
+        }
+    except Exception as e:
+        return {"ok": False, "error": f"data.world download failed: {str(e)}"}
+
+def main():
+    parser = argparse.ArgumentParser(description="Vesper data.world Engine")
+    parser.add_argument("action", choices=["discover", "download"])
+    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
+    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
+
+    args = parser.parse_args()
+
+    if args.action == "discover":
+        limit = int(args.arg2) if args.arg2 else 20
+        result = discover(args.arg1, limit)
+        print(json.dumps(result))
+    elif args.action == "download":
+        result = download(args.arg1, args.arg2)
+        print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()